blob: 03ee684cd8e020b9b0ee00311d6419d7de220981 [file] [log] [blame]
perkj26091b12016-09-01 01:17:40 -07001/*
2 * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Erik Språng4529fbc2018-10-12 10:30:31 +020011#include "video/video_stream_encoder.h"
12
sprangfe627f32017-03-29 08:24:59 -070013#include <algorithm>
perkj803d97f2016-11-01 11:45:46 -070014#include <limits>
Danil Chapovalovd3ba2362019-04-10 17:01:23 +020015#include <memory>
Henrik Boström56db9ff2021-03-24 09:06:45 +010016#include <tuple>
Per512ecb32016-09-23 15:52:06 +020017#include <utility>
18
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020019#include "absl/memory/memory.h"
Markus Handell9a478b52021-11-18 16:07:01 +010020#include "api/rtp_parameters.h"
Danil Chapovalovd3ba2362019-04-10 17:01:23 +020021#include "api/task_queue/default_task_queue_factory.h"
Markus Handell818e7fb2021-12-30 13:01:33 +010022#include "api/task_queue/task_queue_base.h"
Markus Handellb4e96d42021-11-05 12:00:55 +010023#include "api/task_queue/task_queue_factory.h"
Elad Alon45befc52019-07-02 11:20:09 +020024#include "api/test/mock_fec_controller_override.h"
philipel9b058032020-02-10 11:30:00 +010025#include "api/test/mock_video_encoder.h"
Henrik Boström56db9ff2021-03-24 09:06:45 +010026#include "api/test/mock_video_encoder_factory.h"
Markus Handell8d87c462021-12-16 11:37:16 +010027#include "api/units/data_rate.h"
Jiawei Ouc2ebe212018-11-08 10:02:56 -080028#include "api/video/builtin_video_bitrate_allocator_factory.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020029#include "api/video/i420_buffer.h"
Evan Shrubsole895556e2020-10-05 09:15:13 +020030#include "api/video/nv12_buffer.h"
Evan Shrubsolece0a11d2020-04-16 11:36:55 +020031#include "api/video/video_adaptation_reason.h"
Erik Språngf93eda12019-01-16 17:10:57 +010032#include "api/video/video_bitrate_allocation.h"
Henrik Boström56db9ff2021-03-24 09:06:45 +010033#include "api/video_codecs/sdp_video_format.h"
Elad Alon370f93a2019-06-11 14:57:57 +020034#include "api/video_codecs/video_encoder.h"
Erik Språng4529fbc2018-10-12 10:30:31 +020035#include "api/video_codecs/vp8_temporal_layers.h"
Elad Aloncde8ab22019-03-20 11:56:20 +010036#include "api/video_codecs/vp8_temporal_layers_factory.h"
Henrik Boström0f0aa9c2020-06-02 13:02:36 +020037#include "call/adaptation/test/fake_adaptation_constraint.h"
Evan Shrubsoleaa6fbc12020-02-25 16:26:01 +010038#include "call/adaptation/test/fake_resource.h"
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020039#include "common_video/h264/h264_common.h"
Noah Richards51db4212019-06-12 06:59:12 -070040#include "common_video/include/video_frame_buffer.h"
Steve Anton10542f22019-01-11 09:11:00 -080041#include "media/base/video_adapter.h"
Åsa Perssonc5a74ff2020-09-20 17:50:00 +020042#include "media/engine/webrtc_video_engine.h"
philipel95701502022-01-18 18:47:52 +010043#include "modules/video_coding/codecs/av1/libaom_av1_encoder_supported.h"
Henrik Boström56db9ff2021-03-24 09:06:45 +010044#include "modules/video_coding/codecs/h264/include/h264.h"
45#include "modules/video_coding/codecs/multiplex/include/multiplex_encoder_adapter.h"
46#include "modules/video_coding/codecs/vp8/include/vp8.h"
47#include "modules/video_coding/codecs/vp9/include/vp9.h"
Sergey Silkin86684962018-03-28 19:32:37 +020048#include "modules/video_coding/codecs/vp9/include/vp9_globals.h"
Erik Språng7444b192021-06-02 14:02:13 +020049#include "modules/video_coding/codecs/vp9/svc_config.h"
Henrik Boström91aa7322020-04-28 12:24:33 +020050#include "modules/video_coding/utility/quality_scaler.h"
Åsa Perssonc29cb2c2019-03-25 12:06:59 +010051#include "modules/video_coding/utility/simulcast_rate_allocator.h"
Tomas Gunnarsson612445e2020-09-21 14:31:23 +020052#include "rtc_base/event.h"
Åsa Persson258e9892021-02-25 10:39:51 +010053#include "rtc_base/experiments/encoder_info_settings.h"
Henrik Boström2671dac2020-05-19 16:29:09 +020054#include "rtc_base/gunit.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020055#include "rtc_base/logging.h"
Steve Anton10542f22019-01-11 09:11:00 -080056#include "rtc_base/ref_counted_object.h"
Markus Handella3765182020-07-08 13:13:32 +020057#include "rtc_base/synchronization/mutex.h"
Mirko Bonadei17f48782018-09-28 08:51:10 +020058#include "system_wrappers/include/metrics.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020059#include "test/encoder_settings.h"
60#include "test/fake_encoder.h"
Artem Titov33f9d2b2019-12-05 15:59:00 +010061#include "test/frame_forwarder.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020062#include "test/gmock.h"
63#include "test/gtest.h"
Henrik Boström56db9ff2021-03-24 09:06:45 +010064#include "test/mappable_native_buffer.h"
Jonas Orelandc7f691a2022-03-09 15:12:07 +010065#include "test/scoped_key_value_config.h"
Tomas Gunnarsson612445e2020-09-21 14:31:23 +020066#include "test/time_controller/simulated_time_controller.h"
Niels Möllercbcbc222018-09-28 09:07:24 +020067#include "test/video_encoder_proxy_factory.h"
Markus Handellb4e96d42021-11-05 12:00:55 +010068#include "video/frame_cadence_adapter.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020069#include "video/send_statistics_proxy.h"
perkj26091b12016-09-01 01:17:40 -070070
71namespace webrtc {
72
sprang57c2fff2017-01-16 06:24:02 -080073using ::testing::_;
philipeld9cc8c02019-09-16 14:53:40 +020074using ::testing::AllOf;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +020075using ::testing::Eq;
philipeld9cc8c02019-09-16 14:53:40 +020076using ::testing::Field;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +020077using ::testing::Ge;
78using ::testing::Gt;
Markus Handellee225432021-11-29 12:35:12 +010079using ::testing::Invoke;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +020080using ::testing::Le;
81using ::testing::Lt;
philipel9b058032020-02-10 11:30:00 +010082using ::testing::Matcher;
Markus Handellb4e96d42021-11-05 12:00:55 +010083using ::testing::Mock;
philipel9b058032020-02-10 11:30:00 +010084using ::testing::NiceMock;
Markus Handellb4e96d42021-11-05 12:00:55 +010085using ::testing::Optional;
philipel9b058032020-02-10 11:30:00 +010086using ::testing::Return;
Per Kjellander4190ce92020-12-15 17:24:55 +010087using ::testing::SizeIs;
philipeld9cc8c02019-09-16 14:53:40 +020088using ::testing::StrictMock;
kthelgason876222f2016-11-29 01:44:11 -080089
perkj803d97f2016-11-01 11:45:46 -070090namespace {
Åsa Persson8c1bf952018-09-13 10:42:19 +020091const int kMinPixelsPerFrame = 320 * 180;
Åsa Perssone644a032019-11-08 15:56:00 +010092const int kQpLow = 1;
93const int kQpHigh = 2;
Åsa Persson8c1bf952018-09-13 10:42:19 +020094const int kMinFramerateFps = 2;
95const int kMinBalancedFramerateFps = 7;
96const int64_t kFrameTimeoutMs = 100;
asapersson5f7226f2016-11-25 04:37:00 -080097const size_t kMaxPayloadLength = 1440;
Asa Persson606d3cb2021-10-04 10:07:11 +020098const DataRate kTargetBitrate = DataRate::KilobitsPerSec(1000);
99const DataRate kLowTargetBitrate = DataRate::KilobitsPerSec(100);
100const DataRate kStartBitrate = DataRate::KilobitsPerSec(600);
101const DataRate kSimulcastTargetBitrate = DataRate::KilobitsPerSec(3150);
kthelgason2bc68642017-02-07 07:02:22 -0800102const int kMaxInitialFramedrop = 4;
sprangfda496a2017-06-15 04:21:07 -0700103const int kDefaultFramerate = 30;
Åsa Persson8c1bf952018-09-13 10:42:19 +0200104const int64_t kFrameIntervalMs = rtc::kNumMillisecsPerSec / kDefaultFramerate;
Niels Möllerfe407b72019-09-10 10:48:48 +0200105const int64_t kProcessIntervalMs = 1000;
Sergey Silkin41c650b2019-10-14 13:12:19 +0200106const VideoEncoder::ResolutionBitrateLimits
107 kEncoderBitrateLimits540p(960 * 540, 100 * 1000, 100 * 1000, 2000 * 1000);
108const VideoEncoder::ResolutionBitrateLimits
109 kEncoderBitrateLimits720p(1280 * 720, 200 * 1000, 200 * 1000, 4000 * 1000);
asapersson5f7226f2016-11-25 04:37:00 -0800110
Asa Persson606d3cb2021-10-04 10:07:11 +0200111uint8_t kOptimalSps[] = {0, 0, 0, 1, H264::NaluType::kSps,
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +0200112 0x00, 0x00, 0x03, 0x03, 0xF4,
113 0x05, 0x03, 0xC7, 0xE0, 0x1B,
114 0x41, 0x10, 0x8D, 0x00};
115
Sergey Silkin0e42cf72021-03-15 10:12:57 +0100116const uint8_t kCodedFrameVp8Qp25[] = {
117 0x10, 0x02, 0x00, 0x9d, 0x01, 0x2a, 0x10, 0x00, 0x10, 0x00,
118 0x02, 0x47, 0x08, 0x85, 0x85, 0x88, 0x85, 0x84, 0x88, 0x0c,
119 0x82, 0x00, 0x0c, 0x0d, 0x60, 0x00, 0xfe, 0xfc, 0x5c, 0xd0};
120
Markus Handell818e7fb2021-12-30 13:01:33 +0100121VideoFrame CreateSimpleNV12Frame() {
122 return VideoFrame::Builder()
123 .set_video_frame_buffer(rtc::make_ref_counted<NV12Buffer>(
124 /*width=*/16, /*height=*/16))
125 .build();
126}
127
Markus Handell8d87c462021-12-16 11:37:16 +0100128void PassAFrame(
129 TaskQueueBase* encoder_queue,
130 FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback,
131 int64_t ntp_time_ms) {
132 encoder_queue->PostTask(
133 ToQueuedTask([video_stream_encoder_callback, ntp_time_ms] {
Markus Handell818e7fb2021-12-30 13:01:33 +0100134 video_stream_encoder_callback->OnFrame(Timestamp::Millis(ntp_time_ms),
135 1, CreateSimpleNV12Frame());
Markus Handell8d87c462021-12-16 11:37:16 +0100136 }));
137}
138
perkj803d97f2016-11-01 11:45:46 -0700139class TestBuffer : public webrtc::I420Buffer {
140 public:
141 TestBuffer(rtc::Event* event, int width, int height)
142 : I420Buffer(width, height), event_(event) {}
143
144 private:
145 friend class rtc::RefCountedObject<TestBuffer>;
146 ~TestBuffer() override {
147 if (event_)
148 event_->Set();
149 }
150 rtc::Event* const event_;
151};
152
Henrik Boström56db9ff2021-03-24 09:06:45 +0100153// A fake native buffer that can't be converted to I420. Upon scaling, it
154// produces another FakeNativeBuffer.
Noah Richards51db4212019-06-12 06:59:12 -0700155class FakeNativeBuffer : public webrtc::VideoFrameBuffer {
156 public:
157 FakeNativeBuffer(rtc::Event* event, int width, int height)
158 : event_(event), width_(width), height_(height) {}
159 webrtc::VideoFrameBuffer::Type type() const override { return Type::kNative; }
160 int width() const override { return width_; }
161 int height() const override { return height_; }
162 rtc::scoped_refptr<webrtc::I420BufferInterface> ToI420() override {
163 return nullptr;
164 }
Henrik Boström56db9ff2021-03-24 09:06:45 +0100165 rtc::scoped_refptr<VideoFrameBuffer> CropAndScale(
166 int offset_x,
167 int offset_y,
168 int crop_width,
169 int crop_height,
170 int scaled_width,
171 int scaled_height) override {
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200172 return rtc::make_ref_counted<FakeNativeBuffer>(nullptr, scaled_width,
173 scaled_height);
Henrik Boström56db9ff2021-03-24 09:06:45 +0100174 }
Noah Richards51db4212019-06-12 06:59:12 -0700175
176 private:
177 friend class rtc::RefCountedObject<FakeNativeBuffer>;
178 ~FakeNativeBuffer() override {
179 if (event_)
180 event_->Set();
181 }
182 rtc::Event* const event_;
183 const int width_;
184 const int height_;
185};
186
Evan Shrubsole895556e2020-10-05 09:15:13 +0200187// A fake native buffer that is backed by an NV12 buffer.
188class FakeNV12NativeBuffer : public webrtc::VideoFrameBuffer {
189 public:
190 FakeNV12NativeBuffer(rtc::Event* event, int width, int height)
191 : nv12_buffer_(NV12Buffer::Create(width, height)), event_(event) {}
192
193 webrtc::VideoFrameBuffer::Type type() const override { return Type::kNative; }
194 int width() const override { return nv12_buffer_->width(); }
195 int height() const override { return nv12_buffer_->height(); }
196 rtc::scoped_refptr<webrtc::I420BufferInterface> ToI420() override {
197 return nv12_buffer_->ToI420();
198 }
Evan Shrubsoleb556b082020-10-08 14:56:45 +0200199 rtc::scoped_refptr<VideoFrameBuffer> GetMappedFrameBuffer(
200 rtc::ArrayView<VideoFrameBuffer::Type> types) override {
201 if (absl::c_find(types, Type::kNV12) != types.end()) {
202 return nv12_buffer_;
203 }
204 return nullptr;
205 }
Evan Shrubsole895556e2020-10-05 09:15:13 +0200206 const NV12BufferInterface* GetNV12() const { return nv12_buffer_; }
207
208 private:
209 friend class rtc::RefCountedObject<FakeNV12NativeBuffer>;
210 ~FakeNV12NativeBuffer() override {
211 if (event_)
212 event_->Set();
213 }
214 rtc::scoped_refptr<NV12Buffer> nv12_buffer_;
215 rtc::Event* const event_;
216};
217
Niels Möller7dc26b72017-12-06 10:27:48 +0100218class CpuOveruseDetectorProxy : public OveruseFrameDetector {
219 public:
Niels Möllerd1f7eb62018-03-28 16:40:58 +0200220 explicit CpuOveruseDetectorProxy(CpuOveruseMetricsObserver* metrics_observer)
221 : OveruseFrameDetector(metrics_observer),
Henrik Boström381d1092020-05-12 18:49:07 +0200222 last_target_framerate_fps_(-1),
223 framerate_updated_event_(true /* manual_reset */,
224 false /* initially_signaled */) {}
Niels Möller7dc26b72017-12-06 10:27:48 +0100225 virtual ~CpuOveruseDetectorProxy() {}
226
227 void OnTargetFramerateUpdated(int framerate_fps) override {
Markus Handella3765182020-07-08 13:13:32 +0200228 MutexLock lock(&lock_);
Niels Möller7dc26b72017-12-06 10:27:48 +0100229 last_target_framerate_fps_ = framerate_fps;
230 OveruseFrameDetector::OnTargetFramerateUpdated(framerate_fps);
Henrik Boström381d1092020-05-12 18:49:07 +0200231 framerate_updated_event_.Set();
Niels Möller7dc26b72017-12-06 10:27:48 +0100232 }
233
234 int GetLastTargetFramerate() {
Markus Handella3765182020-07-08 13:13:32 +0200235 MutexLock lock(&lock_);
Niels Möller7dc26b72017-12-06 10:27:48 +0100236 return last_target_framerate_fps_;
237 }
238
Niels Möller4db138e2018-04-19 09:04:13 +0200239 CpuOveruseOptions GetOptions() { return options_; }
240
Henrik Boström381d1092020-05-12 18:49:07 +0200241 rtc::Event* framerate_updated_event() { return &framerate_updated_event_; }
242
Niels Möller7dc26b72017-12-06 10:27:48 +0100243 private:
Markus Handella3765182020-07-08 13:13:32 +0200244 Mutex lock_;
Niels Möller7dc26b72017-12-06 10:27:48 +0100245 int last_target_framerate_fps_ RTC_GUARDED_BY(lock_);
Henrik Boström381d1092020-05-12 18:49:07 +0200246 rtc::Event framerate_updated_event_;
Niels Möller7dc26b72017-12-06 10:27:48 +0100247};
248
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200249class FakeVideoSourceRestrictionsListener
250 : public VideoSourceRestrictionsListener {
Henrik Boström381d1092020-05-12 18:49:07 +0200251 public:
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200252 FakeVideoSourceRestrictionsListener()
Henrik Boström381d1092020-05-12 18:49:07 +0200253 : was_restrictions_updated_(false), restrictions_updated_event_() {}
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200254 ~FakeVideoSourceRestrictionsListener() override {
Henrik Boström381d1092020-05-12 18:49:07 +0200255 RTC_DCHECK(was_restrictions_updated_);
256 }
257
258 rtc::Event* restrictions_updated_event() {
259 return &restrictions_updated_event_;
260 }
261
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200262 // VideoSourceRestrictionsListener implementation.
Henrik Boström381d1092020-05-12 18:49:07 +0200263 void OnVideoSourceRestrictionsUpdated(
264 VideoSourceRestrictions restrictions,
265 const VideoAdaptationCounters& adaptation_counters,
Evan Shrubsoleec0af262020-07-01 11:47:46 +0200266 rtc::scoped_refptr<Resource> reason,
267 const VideoSourceRestrictions& unfiltered_restrictions) override {
Henrik Boström381d1092020-05-12 18:49:07 +0200268 was_restrictions_updated_ = true;
269 restrictions_updated_event_.Set();
270 }
271
272 private:
273 bool was_restrictions_updated_;
274 rtc::Event restrictions_updated_event_;
275};
276
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200277auto WantsFps(Matcher<int> fps_matcher) {
278 return Field("max_framerate_fps", &rtc::VideoSinkWants::max_framerate_fps,
279 fps_matcher);
280}
281
282auto WantsMaxPixels(Matcher<int> max_pixel_matcher) {
283 return Field("max_pixel_count", &rtc::VideoSinkWants::max_pixel_count,
284 AllOf(max_pixel_matcher, Gt(0)));
285}
286
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200287auto ResolutionMax() {
288 return AllOf(
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200289 WantsMaxPixels(Eq(std::numeric_limits<int>::max())),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200290 Field("target_pixel_count", &rtc::VideoSinkWants::target_pixel_count,
291 Eq(absl::nullopt)));
292}
293
294auto FpsMax() {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200295 return WantsFps(Eq(kDefaultFramerate));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200296}
297
298auto FpsUnlimited() {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200299 return WantsFps(Eq(std::numeric_limits<int>::max()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200300}
301
302auto FpsMatchesResolutionMax(Matcher<int> fps_matcher) {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200303 return AllOf(WantsFps(fps_matcher), ResolutionMax());
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200304}
305
306auto FpsMaxResolutionMatches(Matcher<int> pixel_matcher) {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200307 return AllOf(FpsMax(), WantsMaxPixels(pixel_matcher));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200308}
309
310auto FpsMaxResolutionMax() {
311 return AllOf(FpsMax(), ResolutionMax());
312}
313
314auto UnlimitedSinkWants() {
315 return AllOf(FpsUnlimited(), ResolutionMax());
316}
317
318auto FpsInRangeForPixelsInBalanced(int last_frame_pixels) {
319 Matcher<int> fps_range_matcher;
320
321 if (last_frame_pixels <= 320 * 240) {
322 fps_range_matcher = AllOf(Ge(7), Le(10));
Åsa Perssonc5a74ff2020-09-20 17:50:00 +0200323 } else if (last_frame_pixels <= 480 * 360) {
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200324 fps_range_matcher = AllOf(Ge(10), Le(15));
325 } else if (last_frame_pixels <= 640 * 480) {
326 fps_range_matcher = Ge(15);
327 } else {
328 fps_range_matcher = Eq(kDefaultFramerate);
329 }
330 return Field("max_framerate_fps", &rtc::VideoSinkWants::max_framerate_fps,
331 fps_range_matcher);
332}
333
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200334auto FpsEqResolutionEqTo(const rtc::VideoSinkWants& other_wants) {
335 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
336 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
337}
338
339auto FpsMaxResolutionLt(const rtc::VideoSinkWants& other_wants) {
340 return AllOf(FpsMax(), WantsMaxPixels(Lt(other_wants.max_pixel_count)));
341}
342
343auto FpsMaxResolutionGt(const rtc::VideoSinkWants& other_wants) {
344 return AllOf(FpsMax(), WantsMaxPixels(Gt(other_wants.max_pixel_count)));
345}
346
347auto FpsLtResolutionEq(const rtc::VideoSinkWants& other_wants) {
348 return AllOf(WantsFps(Lt(other_wants.max_framerate_fps)),
349 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
350}
351
352auto FpsGtResolutionEq(const rtc::VideoSinkWants& other_wants) {
353 return AllOf(WantsFps(Gt(other_wants.max_framerate_fps)),
354 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
355}
356
357auto FpsEqResolutionLt(const rtc::VideoSinkWants& other_wants) {
358 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
359 WantsMaxPixels(Lt(other_wants.max_pixel_count)));
360}
361
362auto FpsEqResolutionGt(const rtc::VideoSinkWants& other_wants) {
363 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
364 WantsMaxPixels(Gt(other_wants.max_pixel_count)));
365}
366
mflodmancc3d4422017-08-03 08:27:51 -0700367class VideoStreamEncoderUnderTest : public VideoStreamEncoder {
perkj803d97f2016-11-01 11:45:46 -0700368 public:
Markus Handell9a478b52021-11-18 16:07:01 +0100369 VideoStreamEncoderUnderTest(
370 TimeController* time_controller,
371 std::unique_ptr<FrameCadenceAdapterInterface> cadence_adapter,
372 std::unique_ptr<webrtc::TaskQueueBase, webrtc::TaskQueueDeleter>
373 encoder_queue,
374 SendStatisticsProxy* stats_proxy,
375 const VideoStreamEncoderSettings& settings,
376 VideoStreamEncoder::BitrateAllocationCallbackType
Jonas Orelandc7f691a2022-03-09 15:12:07 +0100377 allocation_callback_type,
378 const WebRtcKeyValueConfig& field_trials)
Markus Handell9a478b52021-11-18 16:07:01 +0100379 : VideoStreamEncoder(time_controller->GetClock(),
380 1 /* number_of_cores */,
381 stats_proxy,
382 settings,
383 std::unique_ptr<OveruseFrameDetector>(
384 overuse_detector_proxy_ =
385 new CpuOveruseDetectorProxy(stats_proxy)),
386 std::move(cadence_adapter),
387 std::move(encoder_queue),
Jonas Orelandc7f691a2022-03-09 15:12:07 +0100388 allocation_callback_type,
389 field_trials),
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200390 time_controller_(time_controller),
Henrik Boström5cc28b02020-06-01 17:59:05 +0200391 fake_cpu_resource_(FakeResource::Create("FakeResource[CPU]")),
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200392 fake_quality_resource_(FakeResource::Create("FakeResource[QP]")),
Evan Shrubsoledc4d4222020-07-09 11:47:10 +0200393 fake_adaptation_constraint_("FakeAdaptationConstraint") {
Henrik Boströmc55516d2020-05-11 16:29:22 +0200394 InjectAdaptationResource(fake_quality_resource_,
Evan Shrubsolece0a11d2020-04-16 11:36:55 +0200395 VideoAdaptationReason::kQuality);
Henrik Boströmc55516d2020-05-11 16:29:22 +0200396 InjectAdaptationResource(fake_cpu_resource_, VideoAdaptationReason::kCpu);
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200397 InjectAdaptationConstraint(&fake_adaptation_constraint_);
Evan Shrubsoleaa6fbc12020-02-25 16:26:01 +0100398 }
perkj803d97f2016-11-01 11:45:46 -0700399
Henrik Boström381d1092020-05-12 18:49:07 +0200400 void SetSourceAndWaitForRestrictionsUpdated(
401 rtc::VideoSourceInterface<VideoFrame>* source,
402 const DegradationPreference& degradation_preference) {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200403 FakeVideoSourceRestrictionsListener listener;
404 AddRestrictionsListenerForTesting(&listener);
Henrik Boström381d1092020-05-12 18:49:07 +0200405 SetSource(source, degradation_preference);
406 listener.restrictions_updated_event()->Wait(5000);
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200407 RemoveRestrictionsListenerForTesting(&listener);
Henrik Boström381d1092020-05-12 18:49:07 +0200408 }
409
410 void SetSourceAndWaitForFramerateUpdated(
411 rtc::VideoSourceInterface<VideoFrame>* source,
412 const DegradationPreference& degradation_preference) {
413 overuse_detector_proxy_->framerate_updated_event()->Reset();
414 SetSource(source, degradation_preference);
415 overuse_detector_proxy_->framerate_updated_event()->Wait(5000);
416 }
417
418 void OnBitrateUpdatedAndWaitForManagedResources(
419 DataRate target_bitrate,
420 DataRate stable_target_bitrate,
421 DataRate link_allocation,
422 uint8_t fraction_lost,
423 int64_t round_trip_time_ms,
424 double cwnd_reduce_ratio) {
425 OnBitrateUpdated(target_bitrate, stable_target_bitrate, link_allocation,
426 fraction_lost, round_trip_time_ms, cwnd_reduce_ratio);
427 // Bitrate is updated on the encoder queue.
428 WaitUntilTaskQueueIsIdle();
Henrik Boström381d1092020-05-12 18:49:07 +0200429 }
430
kthelgason2fc52542017-03-03 00:24:41 -0800431 // This is used as a synchronisation mechanism, to make sure that the
432 // encoder queue is not blocked before we start sending it frames.
433 void WaitUntilTaskQueueIsIdle() {
Markus Handell28c71802021-11-08 10:11:55 +0100434 time_controller_->AdvanceTime(TimeDelta::Zero());
kthelgason2fc52542017-03-03 00:24:41 -0800435 }
436
Henrik Boström91aa7322020-04-28 12:24:33 +0200437 // Triggers resource usage measurements on the fake CPU resource.
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200438 void TriggerCpuOveruse() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200439 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200440 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200441 fake_cpu_resource_->SetUsageState(ResourceUsageState::kOveruse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200442 event.Set();
443 });
444 ASSERT_TRUE(event.Wait(5000));
Markus Handell28c71802021-11-08 10:11:55 +0100445 time_controller_->AdvanceTime(TimeDelta::Zero());
Henrik Boström91aa7322020-04-28 12:24:33 +0200446 }
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200447
Henrik Boström91aa7322020-04-28 12:24:33 +0200448 void TriggerCpuUnderuse() {
449 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200450 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200451 fake_cpu_resource_->SetUsageState(ResourceUsageState::kUnderuse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200452 event.Set();
453 });
454 ASSERT_TRUE(event.Wait(5000));
Markus Handell28c71802021-11-08 10:11:55 +0100455 time_controller_->AdvanceTime(TimeDelta::Zero());
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200456 }
kthelgason876222f2016-11-29 01:44:11 -0800457
Henrik Boström91aa7322020-04-28 12:24:33 +0200458 // Triggers resource usage measurements on the fake quality resource.
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200459 void TriggerQualityLow() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200460 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200461 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200462 fake_quality_resource_->SetUsageState(ResourceUsageState::kOveruse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200463 event.Set();
464 });
465 ASSERT_TRUE(event.Wait(5000));
Markus Handell28c71802021-11-08 10:11:55 +0100466 time_controller_->AdvanceTime(TimeDelta::Zero());
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200467 }
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200468 void TriggerQualityHigh() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200469 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200470 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200471 fake_quality_resource_->SetUsageState(ResourceUsageState::kUnderuse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200472 event.Set();
473 });
474 ASSERT_TRUE(event.Wait(5000));
Markus Handell28c71802021-11-08 10:11:55 +0100475 time_controller_->AdvanceTime(TimeDelta::Zero());
Henrik Boström91aa7322020-04-28 12:24:33 +0200476 }
477
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200478 TimeController* const time_controller_;
Niels Möller7dc26b72017-12-06 10:27:48 +0100479 CpuOveruseDetectorProxy* overuse_detector_proxy_;
Henrik Boströmc55516d2020-05-11 16:29:22 +0200480 rtc::scoped_refptr<FakeResource> fake_cpu_resource_;
481 rtc::scoped_refptr<FakeResource> fake_quality_resource_;
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200482 FakeAdaptationConstraint fake_adaptation_constraint_;
perkj803d97f2016-11-01 11:45:46 -0700483};
484
Noah Richards51db4212019-06-12 06:59:12 -0700485// Simulates simulcast behavior and makes highest stream resolutions divisible
486// by 4.
487class CroppingVideoStreamFactory
488 : public VideoEncoderConfig::VideoStreamFactoryInterface {
489 public:
Åsa Persson17b29b92020-10-17 12:57:58 +0200490 CroppingVideoStreamFactory() {}
Noah Richards51db4212019-06-12 06:59:12 -0700491
492 private:
493 std::vector<VideoStream> CreateEncoderStreams(
494 int width,
495 int height,
496 const VideoEncoderConfig& encoder_config) override {
497 std::vector<VideoStream> streams = test::CreateVideoStreams(
498 width - width % 4, height - height % 4, encoder_config);
Noah Richards51db4212019-06-12 06:59:12 -0700499 return streams;
500 }
Noah Richards51db4212019-06-12 06:59:12 -0700501};
502
sprangb1ca0732017-02-01 08:38:12 -0800503class AdaptingFrameForwarder : public test::FrameForwarder {
504 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200505 explicit AdaptingFrameForwarder(TimeController* time_controller)
506 : time_controller_(time_controller), adaptation_enabled_(false) {}
asaperssonfab67072017-04-04 05:51:49 -0700507 ~AdaptingFrameForwarder() override {}
sprangb1ca0732017-02-01 08:38:12 -0800508
509 void set_adaptation_enabled(bool enabled) {
Markus Handella3765182020-07-08 13:13:32 +0200510 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -0800511 adaptation_enabled_ = enabled;
512 }
513
asaperssonfab67072017-04-04 05:51:49 -0700514 bool adaption_enabled() const {
Markus Handella3765182020-07-08 13:13:32 +0200515 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -0800516 return adaptation_enabled_;
517 }
518
Henrik Boström1124ed12021-02-25 10:30:39 +0100519 // The "last wants" is a snapshot of the previous rtc::VideoSinkWants where
520 // the resolution or frame rate was different than it is currently. If
521 // something else is modified, such as encoder resolutions, but the resolution
522 // and frame rate stays the same, last wants is not updated.
asapersson09f05612017-05-15 23:40:18 -0700523 rtc::VideoSinkWants last_wants() const {
Markus Handella3765182020-07-08 13:13:32 +0200524 MutexLock lock(&mutex_);
asapersson09f05612017-05-15 23:40:18 -0700525 return last_wants_;
526 }
527
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200528 absl::optional<int> last_sent_width() const { return last_width_; }
529 absl::optional<int> last_sent_height() const { return last_height_; }
Jonathan Yubc771b72017-12-08 17:04:29 -0800530
sprangb1ca0732017-02-01 08:38:12 -0800531 void IncomingCapturedFrame(const VideoFrame& video_frame) override {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200532 RTC_DCHECK(time_controller_->GetMainThread()->IsCurrent());
Markus Handell28c71802021-11-08 10:11:55 +0100533 time_controller_->AdvanceTime(TimeDelta::Zero());
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200534
sprangb1ca0732017-02-01 08:38:12 -0800535 int cropped_width = 0;
536 int cropped_height = 0;
537 int out_width = 0;
538 int out_height = 0;
sprangc5d62e22017-04-02 23:53:04 -0700539 if (adaption_enabled()) {
Harald Alvestrand97597c02021-11-04 12:01:23 +0000540 RTC_DLOG(LS_INFO) << "IncomingCapturedFrame: AdaptFrameResolution()"
541 << "w=" << video_frame.width()
542 << "h=" << video_frame.height();
sprangc5d62e22017-04-02 23:53:04 -0700543 if (adapter_.AdaptFrameResolution(
544 video_frame.width(), video_frame.height(),
545 video_frame.timestamp_us() * 1000, &cropped_width,
546 &cropped_height, &out_width, &out_height)) {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100547 VideoFrame adapted_frame =
548 VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200549 .set_video_frame_buffer(rtc::make_ref_counted<TestBuffer>(
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100550 nullptr, out_width, out_height))
Åsa Persson90719572021-04-08 19:05:30 +0200551 .set_ntp_time_ms(video_frame.ntp_time_ms())
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100552 .set_timestamp_ms(99)
553 .set_rotation(kVideoRotation_0)
554 .build();
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +0100555 if (video_frame.has_update_rect()) {
556 adapted_frame.set_update_rect(
557 video_frame.update_rect().ScaleWithFrame(
558 video_frame.width(), video_frame.height(), 0, 0,
559 video_frame.width(), video_frame.height(), out_width,
560 out_height));
561 }
sprangc5d62e22017-04-02 23:53:04 -0700562 test::FrameForwarder::IncomingCapturedFrame(adapted_frame);
Jonathan Yubc771b72017-12-08 17:04:29 -0800563 last_width_.emplace(adapted_frame.width());
564 last_height_.emplace(adapted_frame.height());
565 } else {
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200566 last_width_ = absl::nullopt;
567 last_height_ = absl::nullopt;
sprangc5d62e22017-04-02 23:53:04 -0700568 }
sprangb1ca0732017-02-01 08:38:12 -0800569 } else {
Harald Alvestrand97597c02021-11-04 12:01:23 +0000570 RTC_DLOG(LS_INFO) << "IncomingCapturedFrame: adaptation not enabled";
sprangb1ca0732017-02-01 08:38:12 -0800571 test::FrameForwarder::IncomingCapturedFrame(video_frame);
Jonathan Yubc771b72017-12-08 17:04:29 -0800572 last_width_.emplace(video_frame.width());
573 last_height_.emplace(video_frame.height());
sprangb1ca0732017-02-01 08:38:12 -0800574 }
575 }
576
Åsa Perssonf5f7e8e2021-06-09 22:55:38 +0200577 void OnOutputFormatRequest(int width, int height) {
578 absl::optional<std::pair<int, int>> target_aspect_ratio =
579 std::make_pair(width, height);
580 absl::optional<int> max_pixel_count = width * height;
581 absl::optional<int> max_fps;
582 adapter_.OnOutputFormatRequest(target_aspect_ratio, max_pixel_count,
583 max_fps);
584 }
585
sprangb1ca0732017-02-01 08:38:12 -0800586 void AddOrUpdateSink(rtc::VideoSinkInterface<VideoFrame>* sink,
587 const rtc::VideoSinkWants& wants) override {
Markus Handella3765182020-07-08 13:13:32 +0200588 MutexLock lock(&mutex_);
Henrik Boström1124ed12021-02-25 10:30:39 +0100589 rtc::VideoSinkWants prev_wants = sink_wants_locked();
590 bool did_adapt =
591 prev_wants.max_pixel_count != wants.max_pixel_count ||
592 prev_wants.target_pixel_count != wants.target_pixel_count ||
593 prev_wants.max_framerate_fps != wants.max_framerate_fps;
594 if (did_adapt) {
595 last_wants_ = prev_wants;
596 }
Rasmus Brandt287e4642019-11-15 16:56:01 +0100597 adapter_.OnSinkWants(wants);
Markus Handell16038ab2020-05-28 08:37:30 +0200598 test::FrameForwarder::AddOrUpdateSinkLocked(sink, wants);
sprangb1ca0732017-02-01 08:38:12 -0800599 }
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200600
601 TimeController* const time_controller_;
sprangb1ca0732017-02-01 08:38:12 -0800602 cricket::VideoAdapter adapter_;
Markus Handella3765182020-07-08 13:13:32 +0200603 bool adaptation_enabled_ RTC_GUARDED_BY(mutex_);
604 rtc::VideoSinkWants last_wants_ RTC_GUARDED_BY(mutex_);
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200605 absl::optional<int> last_width_;
606 absl::optional<int> last_height_;
sprangb1ca0732017-02-01 08:38:12 -0800607};
sprangc5d62e22017-04-02 23:53:04 -0700608
Niels Möller213618e2018-07-24 09:29:58 +0200609// TODO(nisse): Mock only VideoStreamEncoderObserver.
sprangc5d62e22017-04-02 23:53:04 -0700610class MockableSendStatisticsProxy : public SendStatisticsProxy {
611 public:
612 MockableSendStatisticsProxy(Clock* clock,
613 const VideoSendStream::Config& config,
614 VideoEncoderConfig::ContentType content_type)
615 : SendStatisticsProxy(clock, config, content_type) {}
616
617 VideoSendStream::Stats GetStats() override {
Markus Handella3765182020-07-08 13:13:32 +0200618 MutexLock lock(&lock_);
sprangc5d62e22017-04-02 23:53:04 -0700619 if (mock_stats_)
620 return *mock_stats_;
621 return SendStatisticsProxy::GetStats();
622 }
623
Niels Möller213618e2018-07-24 09:29:58 +0200624 int GetInputFrameRate() const override {
Markus Handella3765182020-07-08 13:13:32 +0200625 MutexLock lock(&lock_);
Niels Möller213618e2018-07-24 09:29:58 +0200626 if (mock_stats_)
627 return mock_stats_->input_frame_rate;
628 return SendStatisticsProxy::GetInputFrameRate();
629 }
sprangc5d62e22017-04-02 23:53:04 -0700630 void SetMockStats(const VideoSendStream::Stats& stats) {
Markus Handella3765182020-07-08 13:13:32 +0200631 MutexLock lock(&lock_);
sprangc5d62e22017-04-02 23:53:04 -0700632 mock_stats_.emplace(stats);
633 }
634
635 void ResetMockStats() {
Markus Handella3765182020-07-08 13:13:32 +0200636 MutexLock lock(&lock_);
sprangc5d62e22017-04-02 23:53:04 -0700637 mock_stats_.reset();
638 }
639
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200640 void SetDroppedFrameCallback(std::function<void(DropReason)> callback) {
641 on_frame_dropped_ = std::move(callback);
642 }
643
sprangc5d62e22017-04-02 23:53:04 -0700644 private:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200645 void OnFrameDropped(DropReason reason) override {
646 SendStatisticsProxy::OnFrameDropped(reason);
647 if (on_frame_dropped_)
648 on_frame_dropped_(reason);
649 }
650
Markus Handella3765182020-07-08 13:13:32 +0200651 mutable Mutex lock_;
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200652 absl::optional<VideoSendStream::Stats> mock_stats_ RTC_GUARDED_BY(lock_);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200653 std::function<void(DropReason)> on_frame_dropped_;
sprangc5d62e22017-04-02 23:53:04 -0700654};
655
Markus Handellb4e96d42021-11-05 12:00:55 +0100656class SimpleVideoStreamEncoderFactory {
657 public:
658 class AdaptedVideoStreamEncoder : public VideoStreamEncoder {
659 public:
660 using VideoStreamEncoder::VideoStreamEncoder;
661 ~AdaptedVideoStreamEncoder() { Stop(); }
662 };
663
Markus Handell8d87c462021-12-16 11:37:16 +0100664 class MockFakeEncoder : public test::FakeEncoder {
665 public:
666 using FakeEncoder::FakeEncoder;
667 MOCK_METHOD(CodecSpecificInfo,
668 EncodeHook,
669 (EncodedImage & encoded_image,
670 rtc::scoped_refptr<EncodedImageBuffer> buffer),
671 (override));
672 };
673
Markus Handellee225432021-11-29 12:35:12 +0100674 SimpleVideoStreamEncoderFactory() {
Markus Handellb4e96d42021-11-05 12:00:55 +0100675 encoder_settings_.encoder_factory = &encoder_factory_;
Markus Handellee225432021-11-29 12:35:12 +0100676 encoder_settings_.bitrate_allocator_factory =
677 bitrate_allocator_factory_.get();
Markus Handellb4e96d42021-11-05 12:00:55 +0100678 }
679
Markus Handell818e7fb2021-12-30 13:01:33 +0100680 std::unique_ptr<AdaptedVideoStreamEncoder> CreateWithEncoderQueue(
Markus Handellee225432021-11-29 12:35:12 +0100681 std::unique_ptr<FrameCadenceAdapterInterface> zero_hertz_adapter,
Jonas Orelandc7f691a2022-03-09 15:12:07 +0100682 std::unique_ptr<TaskQueueBase, TaskQueueDeleter> encoder_queue,
683 const WebRtcKeyValueConfig* field_trials = nullptr) {
Markus Handellb4e96d42021-11-05 12:00:55 +0100684 auto result = std::make_unique<AdaptedVideoStreamEncoder>(
685 time_controller_.GetClock(),
686 /*number_of_cores=*/1,
687 /*stats_proxy=*/stats_proxy_.get(), encoder_settings_,
688 std::make_unique<CpuOveruseDetectorProxy>(/*stats_proxy=*/nullptr),
Markus Handellee225432021-11-29 12:35:12 +0100689 std::move(zero_hertz_adapter), std::move(encoder_queue),
Markus Handellb4e96d42021-11-05 12:00:55 +0100690 VideoStreamEncoder::BitrateAllocationCallbackType::
Jonas Orelandc7f691a2022-03-09 15:12:07 +0100691 kVideoBitrateAllocation,
692 field_trials ? *field_trials : field_trials_);
Markus Handellb4e96d42021-11-05 12:00:55 +0100693 result->SetSink(&sink_, /*rotation_applied=*/false);
694 return result;
695 }
696
Markus Handell818e7fb2021-12-30 13:01:33 +0100697 std::unique_ptr<AdaptedVideoStreamEncoder> Create(
698 std::unique_ptr<FrameCadenceAdapterInterface> zero_hertz_adapter,
699 TaskQueueBase** encoder_queue_ptr = nullptr) {
700 auto encoder_queue =
701 time_controller_.GetTaskQueueFactory()->CreateTaskQueue(
702 "EncoderQueue", TaskQueueFactory::Priority::NORMAL);
703 if (encoder_queue_ptr)
704 *encoder_queue_ptr = encoder_queue.get();
705 return CreateWithEncoderQueue(std::move(zero_hertz_adapter),
706 std::move(encoder_queue));
707 }
708
Markus Handell9a478b52021-11-18 16:07:01 +0100709 void DepleteTaskQueues() { time_controller_.AdvanceTime(TimeDelta::Zero()); }
Markus Handell8d87c462021-12-16 11:37:16 +0100710 MockFakeEncoder& GetMockFakeEncoder() { return mock_fake_encoder_; }
Markus Handell9a478b52021-11-18 16:07:01 +0100711
Markus Handell818e7fb2021-12-30 13:01:33 +0100712 GlobalSimulatedTimeController* GetTimeController() {
713 return &time_controller_;
714 }
715
Markus Handellb4e96d42021-11-05 12:00:55 +0100716 private:
717 class NullEncoderSink : public VideoStreamEncoderInterface::EncoderSink {
718 public:
719 ~NullEncoderSink() override = default;
720 void OnEncoderConfigurationChanged(
721 std::vector<VideoStream> streams,
722 bool is_svc,
723 VideoEncoderConfig::ContentType content_type,
724 int min_transmit_bitrate_bps) override {}
725 void OnBitrateAllocationUpdated(
726 const VideoBitrateAllocation& allocation) override {}
727 void OnVideoLayersAllocationUpdated(
728 VideoLayersAllocation allocation) override {}
729 Result OnEncodedImage(
730 const EncodedImage& encoded_image,
731 const CodecSpecificInfo* codec_specific_info) override {
732 return Result(EncodedImageCallback::Result::OK);
733 }
734 };
735
Jonas Orelandc7f691a2022-03-09 15:12:07 +0100736 test::ScopedKeyValueConfig field_trials_;
Markus Handellee225432021-11-29 12:35:12 +0100737 GlobalSimulatedTimeController time_controller_{Timestamp::Millis(0)};
738 std::unique_ptr<TaskQueueFactory> task_queue_factory_{
739 time_controller_.CreateTaskQueueFactory()};
740 std::unique_ptr<MockableSendStatisticsProxy> stats_proxy_ =
741 std::make_unique<MockableSendStatisticsProxy>(
742 time_controller_.GetClock(),
743 VideoSendStream::Config(nullptr),
744 webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo);
745 std::unique_ptr<VideoBitrateAllocatorFactory> bitrate_allocator_factory_ =
746 CreateBuiltinVideoBitrateAllocatorFactory();
747 VideoStreamEncoderSettings encoder_settings_{
748 VideoEncoder::Capabilities(/*loss_notification=*/false)};
Markus Handell8d87c462021-12-16 11:37:16 +0100749 MockFakeEncoder mock_fake_encoder_{time_controller_.GetClock()};
750 test::VideoEncoderProxyFactory encoder_factory_{&mock_fake_encoder_};
Markus Handellb4e96d42021-11-05 12:00:55 +0100751 NullEncoderSink sink_;
752};
753
754class MockFrameCadenceAdapter : public FrameCadenceAdapterInterface {
755 public:
756 MOCK_METHOD(void, Initialize, (Callback * callback), (override));
Markus Handell8d87c462021-12-16 11:37:16 +0100757 MOCK_METHOD(void,
758 SetZeroHertzModeEnabled,
759 (absl::optional<ZeroHertzModeParams>),
760 (override));
Markus Handellb4e96d42021-11-05 12:00:55 +0100761 MOCK_METHOD(void, OnFrame, (const VideoFrame&), (override));
Markus Handellee225432021-11-29 12:35:12 +0100762 MOCK_METHOD(absl::optional<uint32_t>, GetInputFrameRateFps, (), (override));
763 MOCK_METHOD(void, UpdateFrameRate, (), (override));
Markus Handell8d87c462021-12-16 11:37:16 +0100764 MOCK_METHOD(void,
765 UpdateLayerQualityConvergence,
766 (int spatial_index, bool converged),
767 (override));
768 MOCK_METHOD(void,
769 UpdateLayerStatus,
770 (int spatial_index, bool enabled),
771 (override));
Markus Handell818e7fb2021-12-30 13:01:33 +0100772 MOCK_METHOD(void, ProcessKeyFrameRequest, (), (override));
Markus Handellb4e96d42021-11-05 12:00:55 +0100773};
774
philipel9b058032020-02-10 11:30:00 +0100775class MockEncoderSelector
776 : public VideoEncoderFactory::EncoderSelectorInterface {
777 public:
Danil Chapovalov91fdc602020-05-14 19:17:51 +0200778 MOCK_METHOD(void,
779 OnCurrentEncoder,
780 (const SdpVideoFormat& format),
781 (override));
782 MOCK_METHOD(absl::optional<SdpVideoFormat>,
783 OnAvailableBitrate,
784 (const DataRate& rate),
785 (override));
786 MOCK_METHOD(absl::optional<SdpVideoFormat>, OnEncoderBroken, (), (override));
philipel9b058032020-02-10 11:30:00 +0100787};
788
Markus Handell2e0f4f02021-12-21 19:14:58 +0100789class MockVideoSourceInterface : public rtc::VideoSourceInterface<VideoFrame> {
790 public:
791 MOCK_METHOD(void,
792 AddOrUpdateSink,
793 (rtc::VideoSinkInterface<VideoFrame>*,
794 const rtc::VideoSinkWants&),
795 (override));
796 MOCK_METHOD(void,
797 RemoveSink,
798 (rtc::VideoSinkInterface<VideoFrame>*),
799 (override));
800 MOCK_METHOD(void, RequestRefreshFrame, (), (override));
801};
802
perkj803d97f2016-11-01 11:45:46 -0700803} // namespace
804
mflodmancc3d4422017-08-03 08:27:51 -0700805class VideoStreamEncoderTest : public ::testing::Test {
perkj26091b12016-09-01 01:17:40 -0700806 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200807 static const int kDefaultTimeoutMs = 1000;
perkj26091b12016-09-01 01:17:40 -0700808
mflodmancc3d4422017-08-03 08:27:51 -0700809 VideoStreamEncoderTest()
perkj26091b12016-09-01 01:17:40 -0700810 : video_send_config_(VideoSendStream::Config(nullptr)),
perkjfa10b552016-10-02 23:45:26 -0700811 codec_width_(320),
812 codec_height_(240),
Åsa Persson8c1bf952018-09-13 10:42:19 +0200813 max_framerate_(kDefaultFramerate),
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200814 fake_encoder_(&time_controller_),
Niels Möller4db138e2018-04-19 09:04:13 +0200815 encoder_factory_(&fake_encoder_),
sprangc5d62e22017-04-02 23:53:04 -0700816 stats_proxy_(new MockableSendStatisticsProxy(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200817 time_controller_.GetClock(),
perkj803d97f2016-11-01 11:45:46 -0700818 video_send_config_,
819 webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo)),
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200820 sink_(&time_controller_, &fake_encoder_) {}
perkj26091b12016-09-01 01:17:40 -0700821
822 void SetUp() override {
perkj803d97f2016-11-01 11:45:46 -0700823 metrics::Reset();
perkj26091b12016-09-01 01:17:40 -0700824 video_send_config_ = VideoSendStream::Config(nullptr);
Niels Möller4db138e2018-04-19 09:04:13 +0200825 video_send_config_.encoder_settings.encoder_factory = &encoder_factory_;
Jiawei Ouc2ebe212018-11-08 10:02:56 -0800826 video_send_config_.encoder_settings.bitrate_allocator_factory =
Sergey Silkin5ee69672019-07-02 14:18:34 +0200827 &bitrate_allocator_factory_;
Niels Möller259a4972018-04-05 15:36:51 +0200828 video_send_config_.rtp.payload_name = "FAKE";
829 video_send_config_.rtp.payload_type = 125;
perkj26091b12016-09-01 01:17:40 -0700830
Per512ecb32016-09-23 15:52:06 +0200831 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +0200832 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Åsa Persson17107062020-10-08 08:57:51 +0200833 EXPECT_EQ(1u, video_encoder_config.simulcast_layers.size());
834 video_encoder_config.simulcast_layers[0].num_temporal_layers = 1;
835 video_encoder_config.simulcast_layers[0].max_framerate = max_framerate_;
Erik Språng08127a92016-11-16 16:41:30 +0100836 video_encoder_config_ = video_encoder_config.Copy();
sprang4847ae62017-06-27 07:06:52 -0700837
Niels Möllerf1338562018-04-26 09:51:47 +0200838 ConfigureEncoder(std::move(video_encoder_config));
asapersson5f7226f2016-11-25 04:37:00 -0800839 }
840
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100841 void ConfigureEncoder(
842 VideoEncoderConfig video_encoder_config,
843 VideoStreamEncoder::BitrateAllocationCallbackType
844 allocation_callback_type =
845 VideoStreamEncoder::BitrateAllocationCallbackType::
846 kVideoBitrateAllocationWhenScreenSharing) {
mflodmancc3d4422017-08-03 08:27:51 -0700847 if (video_stream_encoder_)
848 video_stream_encoder_->Stop();
Markus Handell9a478b52021-11-18 16:07:01 +0100849
850 auto encoder_queue = GetTaskQueueFactory()->CreateTaskQueue(
851 "EncoderQueue", TaskQueueFactory::Priority::NORMAL);
852 TaskQueueBase* encoder_queue_ptr = encoder_queue.get();
853 std::unique_ptr<FrameCadenceAdapterInterface> cadence_adapter =
854 FrameCadenceAdapterInterface::Create(time_controller_.GetClock(),
855 encoder_queue_ptr);
856 video_stream_encoder_ = std::make_unique<VideoStreamEncoderUnderTest>(
857 &time_controller_, std::move(cadence_adapter), std::move(encoder_queue),
858 stats_proxy_.get(), video_send_config_.encoder_settings,
Jonas Orelandc7f691a2022-03-09 15:12:07 +0100859 allocation_callback_type, field_trials_);
Asa Persson606d3cb2021-10-04 10:07:11 +0200860 video_stream_encoder_->SetSink(&sink_, /*rotation_applied=*/false);
mflodmancc3d4422017-08-03 08:27:51 -0700861 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -0700862 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Asa Persson606d3cb2021-10-04 10:07:11 +0200863 video_stream_encoder_->SetStartBitrate(kTargetBitrate.bps());
mflodmancc3d4422017-08-03 08:27:51 -0700864 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +0200865 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -0700866 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
asapersson5f7226f2016-11-25 04:37:00 -0800867 }
868
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100869 void ResetEncoder(const std::string& payload_name,
870 size_t num_streams,
871 size_t num_temporal_layers,
872 unsigned char num_spatial_layers,
873 bool screenshare,
874 VideoStreamEncoder::BitrateAllocationCallbackType
875 allocation_callback_type =
876 VideoStreamEncoder::BitrateAllocationCallbackType::
877 kVideoBitrateAllocationWhenScreenSharing) {
Niels Möller259a4972018-04-05 15:36:51 +0200878 video_send_config_.rtp.payload_name = payload_name;
asapersson5f7226f2016-11-25 04:37:00 -0800879
880 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +0200881 test::FillEncoderConfiguration(PayloadStringToCodecType(payload_name),
882 num_streams, &video_encoder_config);
883 for (auto& layer : video_encoder_config.simulcast_layers) {
884 layer.num_temporal_layers = num_temporal_layers;
885 layer.max_framerate = kDefaultFramerate;
886 }
Erik Språngd7329ca2019-02-21 21:19:53 +0100887 video_encoder_config.max_bitrate_bps =
Asa Persson606d3cb2021-10-04 10:07:11 +0200888 num_streams == 1 ? kTargetBitrate.bps() : kSimulcastTargetBitrate.bps();
sprang4847ae62017-06-27 07:06:52 -0700889 video_encoder_config.content_type =
890 screenshare ? VideoEncoderConfig::ContentType::kScreen
891 : VideoEncoderConfig::ContentType::kRealtimeVideo;
emircanbbcc3562017-08-18 00:28:40 -0700892 if (payload_name == "VP9") {
893 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
894 vp9_settings.numberOfSpatialLayers = num_spatial_layers;
Mirta Dvornicic97910da2020-07-14 15:29:23 +0200895 vp9_settings.automaticResizeOn = num_spatial_layers <= 1;
emircanbbcc3562017-08-18 00:28:40 -0700896 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200897 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
898 vp9_settings);
emircanbbcc3562017-08-18 00:28:40 -0700899 }
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100900 ConfigureEncoder(std::move(video_encoder_config), allocation_callback_type);
perkj26091b12016-09-01 01:17:40 -0700901 }
902
sprang57c2fff2017-01-16 06:24:02 -0800903 VideoFrame CreateFrame(int64_t ntp_time_ms,
904 rtc::Event* destruction_event) const {
Åsa Persson90719572021-04-08 19:05:30 +0200905 return VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200906 .set_video_frame_buffer(rtc::make_ref_counted<TestBuffer>(
Åsa Persson90719572021-04-08 19:05:30 +0200907 destruction_event, codec_width_, codec_height_))
908 .set_ntp_time_ms(ntp_time_ms)
909 .set_timestamp_ms(99)
910 .set_rotation(kVideoRotation_0)
911 .build();
perkj26091b12016-09-01 01:17:40 -0700912 }
913
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100914 VideoFrame CreateFrameWithUpdatedPixel(int64_t ntp_time_ms,
915 rtc::Event* destruction_event,
916 int offset_x) const {
Åsa Persson90719572021-04-08 19:05:30 +0200917 return VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200918 .set_video_frame_buffer(rtc::make_ref_counted<TestBuffer>(
Åsa Persson90719572021-04-08 19:05:30 +0200919 destruction_event, codec_width_, codec_height_))
920 .set_ntp_time_ms(ntp_time_ms)
921 .set_timestamp_ms(99)
922 .set_rotation(kVideoRotation_0)
923 .set_update_rect(VideoFrame::UpdateRect{offset_x, 0, 1, 1})
924 .build();
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100925 }
926
sprang57c2fff2017-01-16 06:24:02 -0800927 VideoFrame CreateFrame(int64_t ntp_time_ms, int width, int height) const {
Erik Språng7444b192021-06-02 14:02:13 +0200928 auto buffer = rtc::make_ref_counted<TestBuffer>(nullptr, width, height);
929 I420Buffer::SetBlack(buffer.get());
Åsa Persson90719572021-04-08 19:05:30 +0200930 return VideoFrame::Builder()
Erik Språng7444b192021-06-02 14:02:13 +0200931 .set_video_frame_buffer(std::move(buffer))
Åsa Persson90719572021-04-08 19:05:30 +0200932 .set_ntp_time_ms(ntp_time_ms)
933 .set_timestamp_ms(ntp_time_ms)
934 .set_rotation(kVideoRotation_0)
935 .build();
perkj803d97f2016-11-01 11:45:46 -0700936 }
937
Evan Shrubsole895556e2020-10-05 09:15:13 +0200938 VideoFrame CreateNV12Frame(int64_t ntp_time_ms, int width, int height) const {
Åsa Persson90719572021-04-08 19:05:30 +0200939 return VideoFrame::Builder()
940 .set_video_frame_buffer(NV12Buffer::Create(width, height))
941 .set_ntp_time_ms(ntp_time_ms)
942 .set_timestamp_ms(ntp_time_ms)
943 .set_rotation(kVideoRotation_0)
944 .build();
Evan Shrubsole895556e2020-10-05 09:15:13 +0200945 }
946
Noah Richards51db4212019-06-12 06:59:12 -0700947 VideoFrame CreateFakeNativeFrame(int64_t ntp_time_ms,
948 rtc::Event* destruction_event,
949 int width,
950 int height) const {
Åsa Persson90719572021-04-08 19:05:30 +0200951 return VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200952 .set_video_frame_buffer(rtc::make_ref_counted<FakeNativeBuffer>(
Åsa Persson90719572021-04-08 19:05:30 +0200953 destruction_event, width, height))
954 .set_ntp_time_ms(ntp_time_ms)
955 .set_timestamp_ms(99)
956 .set_rotation(kVideoRotation_0)
957 .build();
Noah Richards51db4212019-06-12 06:59:12 -0700958 }
959
Evan Shrubsole895556e2020-10-05 09:15:13 +0200960 VideoFrame CreateFakeNV12NativeFrame(int64_t ntp_time_ms,
961 rtc::Event* destruction_event,
962 int width,
963 int height) const {
Åsa Persson90719572021-04-08 19:05:30 +0200964 return VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200965 .set_video_frame_buffer(rtc::make_ref_counted<FakeNV12NativeBuffer>(
Åsa Persson90719572021-04-08 19:05:30 +0200966 destruction_event, width, height))
967 .set_ntp_time_ms(ntp_time_ms)
968 .set_timestamp_ms(99)
969 .set_rotation(kVideoRotation_0)
970 .build();
Evan Shrubsole895556e2020-10-05 09:15:13 +0200971 }
972
Noah Richards51db4212019-06-12 06:59:12 -0700973 VideoFrame CreateFakeNativeFrame(int64_t ntp_time_ms,
974 rtc::Event* destruction_event) const {
975 return CreateFakeNativeFrame(ntp_time_ms, destruction_event, codec_width_,
976 codec_height_);
977 }
978
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100979 void VerifyAllocatedBitrate(const VideoBitrateAllocation& expected_bitrate) {
Henrik Boström381d1092020-05-12 18:49:07 +0200980 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +0200981 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100982
983 video_source_.IncomingCapturedFrame(
984 CreateFrame(1, codec_width_, codec_height_));
985 WaitForEncodedFrame(1);
Per Kjellanderdcef6412020-10-07 15:09:05 +0200986 EXPECT_EQ(expected_bitrate, sink_.GetLastVideoBitrateAllocation());
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100987 }
988
sprang4847ae62017-06-27 07:06:52 -0700989 void WaitForEncodedFrame(int64_t expected_ntp_time) {
990 sink_.WaitForEncodedFrame(expected_ntp_time);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200991 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700992 }
993
994 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time, int64_t timeout_ms) {
995 bool ok = sink_.TimedWaitForEncodedFrame(expected_ntp_time, timeout_ms);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200996 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700997 return ok;
998 }
999
1000 void WaitForEncodedFrame(uint32_t expected_width, uint32_t expected_height) {
1001 sink_.WaitForEncodedFrame(expected_width, expected_height);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001002 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -07001003 }
1004
1005 void ExpectDroppedFrame() {
1006 sink_.ExpectDroppedFrame();
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001007 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -07001008 }
1009
1010 bool WaitForFrame(int64_t timeout_ms) {
1011 bool ok = sink_.WaitForFrame(timeout_ms);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001012 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -07001013 return ok;
1014 }
1015
perkj26091b12016-09-01 01:17:40 -07001016 class TestEncoder : public test::FakeEncoder {
1017 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001018 explicit TestEncoder(TimeController* time_controller)
1019 : FakeEncoder(time_controller->GetClock()),
1020 time_controller_(time_controller) {
1021 RTC_DCHECK(time_controller_);
1022 }
perkj26091b12016-09-01 01:17:40 -07001023
Erik Språngaed30702018-11-05 12:57:17 +01001024 VideoEncoder::EncoderInfo GetEncoderInfo() const override {
Markus Handella3765182020-07-08 13:13:32 +02001025 MutexLock lock(&local_mutex_);
Erik Språng9d69cbe2020-10-22 17:44:42 +02001026 EncoderInfo info = FakeEncoder::GetEncoderInfo();
Erik Språngb7cb7b52019-02-26 15:52:33 +01001027 if (initialized_ == EncoderState::kInitialized) {
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01001028 if (quality_scaling_) {
Åsa Perssone644a032019-11-08 15:56:00 +01001029 info.scaling_settings = VideoEncoder::ScalingSettings(
1030 kQpLow, kQpHigh, kMinPixelsPerFrame);
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01001031 }
1032 info.is_hardware_accelerated = is_hardware_accelerated_;
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001033 for (int i = 0; i < kMaxSpatialLayers; ++i) {
1034 if (temporal_layers_supported_[i]) {
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01001035 info.fps_allocation[i].clear();
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001036 int num_layers = temporal_layers_supported_[i].value() ? 2 : 1;
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01001037 for (int tid = 0; tid < num_layers; ++tid)
1038 info.fps_allocation[i].push_back(255 / (num_layers - tid));
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001039 }
1040 }
Erik Språngaed30702018-11-05 12:57:17 +01001041 }
Sergey Silkin6456e352019-07-08 17:56:40 +02001042
1043 info.resolution_bitrate_limits = resolution_bitrate_limits_;
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01001044 info.requested_resolution_alignment = requested_resolution_alignment_;
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02001045 info.apply_alignment_to_all_simulcast_layers =
1046 apply_alignment_to_all_simulcast_layers_;
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001047 info.preferred_pixel_formats = preferred_pixel_formats_;
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08001048 if (is_qp_trusted_.has_value()) {
1049 info.is_qp_trusted = is_qp_trusted_;
1050 }
Erik Språngaed30702018-11-05 12:57:17 +01001051 return info;
kthelgason876222f2016-11-29 01:44:11 -08001052 }
1053
Erik Språngb7cb7b52019-02-26 15:52:33 +01001054 int32_t RegisterEncodeCompleteCallback(
1055 EncodedImageCallback* callback) override {
Markus Handella3765182020-07-08 13:13:32 +02001056 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001057 encoded_image_callback_ = callback;
1058 return FakeEncoder::RegisterEncodeCompleteCallback(callback);
1059 }
1060
perkjfa10b552016-10-02 23:45:26 -07001061 void ContinueEncode() { continue_encode_event_.Set(); }
1062
1063 void CheckLastTimeStampsMatch(int64_t ntp_time_ms,
1064 uint32_t timestamp) const {
Markus Handella3765182020-07-08 13:13:32 +02001065 MutexLock lock(&local_mutex_);
perkjfa10b552016-10-02 23:45:26 -07001066 EXPECT_EQ(timestamp_, timestamp);
1067 EXPECT_EQ(ntp_time_ms_, ntp_time_ms);
1068 }
1069
kthelgason2fc52542017-03-03 00:24:41 -08001070 void SetQualityScaling(bool b) {
Markus Handella3765182020-07-08 13:13:32 +02001071 MutexLock lock(&local_mutex_);
kthelgason2fc52542017-03-03 00:24:41 -08001072 quality_scaling_ = b;
1073 }
kthelgasonad9010c2017-02-14 00:46:51 -08001074
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01001075 void SetRequestedResolutionAlignment(int requested_resolution_alignment) {
Markus Handella3765182020-07-08 13:13:32 +02001076 MutexLock lock(&local_mutex_);
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01001077 requested_resolution_alignment_ = requested_resolution_alignment;
1078 }
1079
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02001080 void SetApplyAlignmentToAllSimulcastLayers(bool b) {
1081 MutexLock lock(&local_mutex_);
1082 apply_alignment_to_all_simulcast_layers_ = b;
1083 }
1084
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01001085 void SetIsHardwareAccelerated(bool is_hardware_accelerated) {
Markus Handella3765182020-07-08 13:13:32 +02001086 MutexLock lock(&local_mutex_);
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01001087 is_hardware_accelerated_ = is_hardware_accelerated;
1088 }
1089
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001090 void SetTemporalLayersSupported(size_t spatial_idx, bool supported) {
1091 RTC_DCHECK_LT(spatial_idx, kMaxSpatialLayers);
Markus Handella3765182020-07-08 13:13:32 +02001092 MutexLock lock(&local_mutex_);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001093 temporal_layers_supported_[spatial_idx] = supported;
1094 }
1095
Sergey Silkin6456e352019-07-08 17:56:40 +02001096 void SetResolutionBitrateLimits(
1097 std::vector<ResolutionBitrateLimits> thresholds) {
Markus Handella3765182020-07-08 13:13:32 +02001098 MutexLock lock(&local_mutex_);
Sergey Silkin6456e352019-07-08 17:56:40 +02001099 resolution_bitrate_limits_ = thresholds;
1100 }
1101
sprangfe627f32017-03-29 08:24:59 -07001102 void ForceInitEncodeFailure(bool force_failure) {
Markus Handella3765182020-07-08 13:13:32 +02001103 MutexLock lock(&local_mutex_);
sprangfe627f32017-03-29 08:24:59 -07001104 force_init_encode_failed_ = force_failure;
1105 }
1106
Niels Möller6bb5ab92019-01-11 11:11:10 +01001107 void SimulateOvershoot(double rate_factor) {
Markus Handella3765182020-07-08 13:13:32 +02001108 MutexLock lock(&local_mutex_);
Niels Möller6bb5ab92019-01-11 11:11:10 +01001109 rate_factor_ = rate_factor;
1110 }
1111
Erik Språngd7329ca2019-02-21 21:19:53 +01001112 uint32_t GetLastFramerate() const {
Markus Handella3765182020-07-08 13:13:32 +02001113 MutexLock lock(&local_mutex_);
Niels Möller6bb5ab92019-01-11 11:11:10 +01001114 return last_framerate_;
1115 }
1116
Erik Språngd7329ca2019-02-21 21:19:53 +01001117 VideoFrame::UpdateRect GetLastUpdateRect() const {
Markus Handella3765182020-07-08 13:13:32 +02001118 MutexLock lock(&local_mutex_);
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01001119 return last_update_rect_;
1120 }
1121
Niels Möller87e2d782019-03-07 10:18:23 +01001122 const std::vector<VideoFrameType>& LastFrameTypes() const {
Markus Handella3765182020-07-08 13:13:32 +02001123 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001124 return last_frame_types_;
1125 }
1126
1127 void InjectFrame(const VideoFrame& input_image, bool keyframe) {
Niels Möller87e2d782019-03-07 10:18:23 +01001128 const std::vector<VideoFrameType> frame_type = {
Niels Möller8f7ce222019-03-21 15:43:58 +01001129 keyframe ? VideoFrameType::kVideoFrameKey
1130 : VideoFrameType::kVideoFrameDelta};
Erik Språngd7329ca2019-02-21 21:19:53 +01001131 {
Markus Handella3765182020-07-08 13:13:32 +02001132 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001133 last_frame_types_ = frame_type;
1134 }
Niels Möllerb859b322019-03-07 12:40:01 +01001135 FakeEncoder::Encode(input_image, &frame_type);
Erik Språngd7329ca2019-02-21 21:19:53 +01001136 }
1137
Sergey Silkin0e42cf72021-03-15 10:12:57 +01001138 void InjectEncodedImage(const EncodedImage& image,
1139 const CodecSpecificInfo* codec_specific_info) {
Markus Handella3765182020-07-08 13:13:32 +02001140 MutexLock lock(&local_mutex_);
Sergey Silkin0e42cf72021-03-15 10:12:57 +01001141 encoded_image_callback_->OnEncodedImage(image, codec_specific_info);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001142 }
1143
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001144 void SetEncodedImageData(
1145 rtc::scoped_refptr<EncodedImageBufferInterface> encoded_image_data) {
Markus Handella3765182020-07-08 13:13:32 +02001146 MutexLock lock(&local_mutex_);
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001147 encoded_image_data_ = encoded_image_data;
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001148 }
1149
Erik Språngd7329ca2019-02-21 21:19:53 +01001150 void ExpectNullFrame() {
Markus Handella3765182020-07-08 13:13:32 +02001151 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001152 expect_null_frame_ = true;
1153 }
1154
Erik Språng5056af02019-09-02 15:53:11 +02001155 absl::optional<VideoEncoder::RateControlParameters>
1156 GetAndResetLastRateControlSettings() {
1157 auto settings = last_rate_control_settings_;
1158 last_rate_control_settings_.reset();
1159 return settings;
Erik Språngd7329ca2019-02-21 21:19:53 +01001160 }
1161
Henrik Boström56db9ff2021-03-24 09:06:45 +01001162 int GetLastInputWidth() const {
1163 MutexLock lock(&local_mutex_);
1164 return last_input_width_;
1165 }
1166
1167 int GetLastInputHeight() const {
1168 MutexLock lock(&local_mutex_);
1169 return last_input_height_;
1170 }
1171
Evan Shrubsole895556e2020-10-05 09:15:13 +02001172 absl::optional<VideoFrameBuffer::Type> GetLastInputPixelFormat() {
1173 MutexLock lock(&local_mutex_);
1174 return last_input_pixel_format_;
1175 }
1176
Evan Shrubsole7c079f62019-09-26 09:55:03 +02001177 int GetNumSetRates() const {
Markus Handella3765182020-07-08 13:13:32 +02001178 MutexLock lock(&local_mutex_);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02001179 return num_set_rates_;
1180 }
1181
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001182 void SetPreferredPixelFormats(
1183 absl::InlinedVector<VideoFrameBuffer::Type, kMaxPreferredPixelFormats>
1184 pixel_formats) {
1185 MutexLock lock(&local_mutex_);
1186 preferred_pixel_formats_ = std::move(pixel_formats);
1187 }
1188
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08001189 void SetIsQpTrusted(absl::optional<bool> trusted) {
1190 MutexLock lock(&local_mutex_);
1191 is_qp_trusted_ = trusted;
1192 }
1193
perkjfa10b552016-10-02 23:45:26 -07001194 private:
perkj26091b12016-09-01 01:17:40 -07001195 int32_t Encode(const VideoFrame& input_image,
Niels Möller87e2d782019-03-07 10:18:23 +01001196 const std::vector<VideoFrameType>* frame_types) override {
perkj26091b12016-09-01 01:17:40 -07001197 {
Markus Handella3765182020-07-08 13:13:32 +02001198 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001199 if (expect_null_frame_) {
1200 EXPECT_EQ(input_image.timestamp(), 0u);
1201 EXPECT_EQ(input_image.width(), 1);
1202 last_frame_types_ = *frame_types;
1203 expect_null_frame_ = false;
1204 } else {
1205 EXPECT_GT(input_image.timestamp(), timestamp_);
1206 EXPECT_GT(input_image.ntp_time_ms(), ntp_time_ms_);
1207 EXPECT_EQ(input_image.timestamp(), input_image.ntp_time_ms() * 90);
1208 }
perkj26091b12016-09-01 01:17:40 -07001209
1210 timestamp_ = input_image.timestamp();
1211 ntp_time_ms_ = input_image.ntp_time_ms();
perkj803d97f2016-11-01 11:45:46 -07001212 last_input_width_ = input_image.width();
1213 last_input_height_ = input_image.height();
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01001214 last_update_rect_ = input_image.update_rect();
Erik Språngd7329ca2019-02-21 21:19:53 +01001215 last_frame_types_ = *frame_types;
Evan Shrubsole895556e2020-10-05 09:15:13 +02001216 last_input_pixel_format_ = input_image.video_frame_buffer()->type();
perkj26091b12016-09-01 01:17:40 -07001217 }
Niels Möllerb859b322019-03-07 12:40:01 +01001218 int32_t result = FakeEncoder::Encode(input_image, frame_types);
perkj26091b12016-09-01 01:17:40 -07001219 return result;
1220 }
1221
Niels Möller08ae7ce2020-09-23 15:58:12 +02001222 CodecSpecificInfo EncodeHook(
1223 EncodedImage& encoded_image,
1224 rtc::scoped_refptr<EncodedImageBuffer> buffer) override {
Danil Chapovalov2549f172020-08-12 17:30:36 +02001225 CodecSpecificInfo codec_specific;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001226 {
1227 MutexLock lock(&mutex_);
Danil Chapovalov2549f172020-08-12 17:30:36 +02001228 codec_specific.codecType = config_.codecType;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001229 }
1230 MutexLock lock(&local_mutex_);
1231 if (encoded_image_data_) {
Danil Chapovalov2549f172020-08-12 17:30:36 +02001232 encoded_image.SetEncodedData(encoded_image_data_);
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001233 }
Danil Chapovalov2549f172020-08-12 17:30:36 +02001234 return codec_specific;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001235 }
1236
sprangfe627f32017-03-29 08:24:59 -07001237 int32_t InitEncode(const VideoCodec* config,
Elad Alon370f93a2019-06-11 14:57:57 +02001238 const Settings& settings) override {
1239 int res = FakeEncoder::InitEncode(config, settings);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001240
Markus Handella3765182020-07-08 13:13:32 +02001241 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001242 EXPECT_EQ(initialized_, EncoderState::kUninitialized);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001243
Erik Språng82fad3d2018-03-21 09:57:23 +01001244 if (config->codecType == kVideoCodecVP8) {
sprangfe627f32017-03-29 08:24:59 -07001245 // Simulate setting up temporal layers, in order to validate the life
1246 // cycle of these objects.
Elad Aloncde8ab22019-03-20 11:56:20 +01001247 Vp8TemporalLayersFactory factory;
Elad Alon45befc52019-07-02 11:20:09 +02001248 frame_buffer_controller_ =
1249 factory.Create(*config, settings, &fec_controller_override_);
sprangfe627f32017-03-29 08:24:59 -07001250 }
Erik Språngb7cb7b52019-02-26 15:52:33 +01001251 if (force_init_encode_failed_) {
1252 initialized_ = EncoderState::kInitializationFailed;
sprangfe627f32017-03-29 08:24:59 -07001253 return -1;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001254 }
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01001255
Erik Språngb7cb7b52019-02-26 15:52:33 +01001256 initialized_ = EncoderState::kInitialized;
sprangfe627f32017-03-29 08:24:59 -07001257 return res;
1258 }
1259
Erik Språngb7cb7b52019-02-26 15:52:33 +01001260 int32_t Release() override {
Markus Handella3765182020-07-08 13:13:32 +02001261 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001262 EXPECT_NE(initialized_, EncoderState::kUninitialized);
1263 initialized_ = EncoderState::kUninitialized;
1264 return FakeEncoder::Release();
1265 }
1266
Erik Språng16cb8f52019-04-12 13:59:09 +02001267 void SetRates(const RateControlParameters& parameters) {
Markus Handella3765182020-07-08 13:13:32 +02001268 MutexLock lock(&local_mutex_);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02001269 num_set_rates_++;
Niels Möller6bb5ab92019-01-11 11:11:10 +01001270 VideoBitrateAllocation adjusted_rate_allocation;
1271 for (size_t si = 0; si < kMaxSpatialLayers; ++si) {
1272 for (size_t ti = 0; ti < kMaxTemporalStreams; ++ti) {
Erik Språng16cb8f52019-04-12 13:59:09 +02001273 if (parameters.bitrate.HasBitrate(si, ti)) {
Niels Möller6bb5ab92019-01-11 11:11:10 +01001274 adjusted_rate_allocation.SetBitrate(
1275 si, ti,
Erik Språng16cb8f52019-04-12 13:59:09 +02001276 static_cast<uint32_t>(parameters.bitrate.GetBitrate(si, ti) *
Niels Möller6bb5ab92019-01-11 11:11:10 +01001277 rate_factor_));
1278 }
1279 }
1280 }
Erik Språng16cb8f52019-04-12 13:59:09 +02001281 last_framerate_ = static_cast<uint32_t>(parameters.framerate_fps + 0.5);
Erik Språng5056af02019-09-02 15:53:11 +02001282 last_rate_control_settings_ = parameters;
Erik Språng16cb8f52019-04-12 13:59:09 +02001283 RateControlParameters adjusted_paramters = parameters;
1284 adjusted_paramters.bitrate = adjusted_rate_allocation;
1285 FakeEncoder::SetRates(adjusted_paramters);
Niels Möller6bb5ab92019-01-11 11:11:10 +01001286 }
1287
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001288 TimeController* const time_controller_;
Markus Handella3765182020-07-08 13:13:32 +02001289 mutable Mutex local_mutex_;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001290 enum class EncoderState {
1291 kUninitialized,
1292 kInitializationFailed,
1293 kInitialized
Markus Handella3765182020-07-08 13:13:32 +02001294 } initialized_ RTC_GUARDED_BY(local_mutex_) = EncoderState::kUninitialized;
perkj26091b12016-09-01 01:17:40 -07001295 rtc::Event continue_encode_event_;
Markus Handella3765182020-07-08 13:13:32 +02001296 uint32_t timestamp_ RTC_GUARDED_BY(local_mutex_) = 0;
1297 int64_t ntp_time_ms_ RTC_GUARDED_BY(local_mutex_) = 0;
1298 int last_input_width_ RTC_GUARDED_BY(local_mutex_) = 0;
1299 int last_input_height_ RTC_GUARDED_BY(local_mutex_) = 0;
1300 bool quality_scaling_ RTC_GUARDED_BY(local_mutex_) = true;
1301 int requested_resolution_alignment_ RTC_GUARDED_BY(local_mutex_) = 1;
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02001302 bool apply_alignment_to_all_simulcast_layers_ RTC_GUARDED_BY(local_mutex_) =
1303 false;
Markus Handella3765182020-07-08 13:13:32 +02001304 bool is_hardware_accelerated_ RTC_GUARDED_BY(local_mutex_) = false;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001305 rtc::scoped_refptr<EncodedImageBufferInterface> encoded_image_data_
1306 RTC_GUARDED_BY(local_mutex_);
Elad Aloncde8ab22019-03-20 11:56:20 +01001307 std::unique_ptr<Vp8FrameBufferController> frame_buffer_controller_
Markus Handella3765182020-07-08 13:13:32 +02001308 RTC_GUARDED_BY(local_mutex_);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001309 absl::optional<bool>
1310 temporal_layers_supported_[kMaxSpatialLayers] RTC_GUARDED_BY(
Markus Handella3765182020-07-08 13:13:32 +02001311 local_mutex_);
1312 bool force_init_encode_failed_ RTC_GUARDED_BY(local_mutex_) = false;
1313 double rate_factor_ RTC_GUARDED_BY(local_mutex_) = 1.0;
1314 uint32_t last_framerate_ RTC_GUARDED_BY(local_mutex_) = 0;
Erik Språng5056af02019-09-02 15:53:11 +02001315 absl::optional<VideoEncoder::RateControlParameters>
1316 last_rate_control_settings_;
Markus Handella3765182020-07-08 13:13:32 +02001317 VideoFrame::UpdateRect last_update_rect_ RTC_GUARDED_BY(local_mutex_) = {
1318 0, 0, 0, 0};
Niels Möller87e2d782019-03-07 10:18:23 +01001319 std::vector<VideoFrameType> last_frame_types_;
Erik Språngd7329ca2019-02-21 21:19:53 +01001320 bool expect_null_frame_ = false;
Markus Handella3765182020-07-08 13:13:32 +02001321 EncodedImageCallback* encoded_image_callback_ RTC_GUARDED_BY(local_mutex_) =
1322 nullptr;
Evan Shrubsolefb862742020-03-16 16:18:36 +01001323 NiceMock<MockFecControllerOverride> fec_controller_override_;
Sergey Silkin6456e352019-07-08 17:56:40 +02001324 std::vector<ResolutionBitrateLimits> resolution_bitrate_limits_
Markus Handella3765182020-07-08 13:13:32 +02001325 RTC_GUARDED_BY(local_mutex_);
1326 int num_set_rates_ RTC_GUARDED_BY(local_mutex_) = 0;
Evan Shrubsole895556e2020-10-05 09:15:13 +02001327 absl::optional<VideoFrameBuffer::Type> last_input_pixel_format_
1328 RTC_GUARDED_BY(local_mutex_);
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001329 absl::InlinedVector<VideoFrameBuffer::Type, kMaxPreferredPixelFormats>
1330 preferred_pixel_formats_ RTC_GUARDED_BY(local_mutex_);
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08001331 absl::optional<bool> is_qp_trusted_ RTC_GUARDED_BY(local_mutex_);
perkj26091b12016-09-01 01:17:40 -07001332 };
1333
mflodmancc3d4422017-08-03 08:27:51 -07001334 class TestSink : public VideoStreamEncoder::EncoderSink {
perkj26091b12016-09-01 01:17:40 -07001335 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001336 TestSink(TimeController* time_controller, TestEncoder* test_encoder)
1337 : time_controller_(time_controller), test_encoder_(test_encoder) {
1338 RTC_DCHECK(time_controller_);
1339 }
perkj26091b12016-09-01 01:17:40 -07001340
perkj26091b12016-09-01 01:17:40 -07001341 void WaitForEncodedFrame(int64_t expected_ntp_time) {
sprang4847ae62017-06-27 07:06:52 -07001342 EXPECT_TRUE(
1343 TimedWaitForEncodedFrame(expected_ntp_time, kDefaultTimeoutMs));
1344 }
1345
1346 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time,
1347 int64_t timeout_ms) {
perkj26091b12016-09-01 01:17:40 -07001348 uint32_t timestamp = 0;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001349 if (!WaitForFrame(timeout_ms))
sprang4847ae62017-06-27 07:06:52 -07001350 return false;
perkj26091b12016-09-01 01:17:40 -07001351 {
Markus Handella3765182020-07-08 13:13:32 +02001352 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -08001353 timestamp = last_timestamp_;
perkj26091b12016-09-01 01:17:40 -07001354 }
1355 test_encoder_->CheckLastTimeStampsMatch(expected_ntp_time, timestamp);
sprang4847ae62017-06-27 07:06:52 -07001356 return true;
perkj26091b12016-09-01 01:17:40 -07001357 }
1358
sprangb1ca0732017-02-01 08:38:12 -08001359 void WaitForEncodedFrame(uint32_t expected_width,
1360 uint32_t expected_height) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001361 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Åsa Perssonc74d8da2017-12-04 14:13:56 +01001362 CheckLastFrameSizeMatches(expected_width, expected_height);
sprangc5d62e22017-04-02 23:53:04 -07001363 }
1364
Åsa Perssonc74d8da2017-12-04 14:13:56 +01001365 void CheckLastFrameSizeMatches(uint32_t expected_width,
sprangc5d62e22017-04-02 23:53:04 -07001366 uint32_t expected_height) {
sprangb1ca0732017-02-01 08:38:12 -08001367 uint32_t width = 0;
1368 uint32_t height = 0;
sprangb1ca0732017-02-01 08:38:12 -08001369 {
Markus Handella3765182020-07-08 13:13:32 +02001370 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -08001371 width = last_width_;
1372 height = last_height_;
1373 }
1374 EXPECT_EQ(expected_height, height);
1375 EXPECT_EQ(expected_width, width);
1376 }
1377
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001378 void CheckLastFrameRotationMatches(VideoRotation expected_rotation) {
1379 VideoRotation rotation;
1380 {
Markus Handella3765182020-07-08 13:13:32 +02001381 MutexLock lock(&mutex_);
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001382 rotation = last_rotation_;
1383 }
1384 EXPECT_EQ(expected_rotation, rotation);
1385 }
1386
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001387 void ExpectDroppedFrame() { EXPECT_FALSE(WaitForFrame(100)); }
kthelgason2bc68642017-02-07 07:02:22 -08001388
sprangc5d62e22017-04-02 23:53:04 -07001389 bool WaitForFrame(int64_t timeout_ms) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001390 RTC_DCHECK(time_controller_->GetMainThread()->IsCurrent());
Markus Handell28c71802021-11-08 10:11:55 +01001391 time_controller_->AdvanceTime(TimeDelta::Zero());
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001392 bool ret = encoded_frame_event_.Wait(timeout_ms);
Markus Handell28c71802021-11-08 10:11:55 +01001393 time_controller_->AdvanceTime(TimeDelta::Zero());
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001394 return ret;
sprangc5d62e22017-04-02 23:53:04 -07001395 }
1396
perkj26091b12016-09-01 01:17:40 -07001397 void SetExpectNoFrames() {
Markus Handella3765182020-07-08 13:13:32 +02001398 MutexLock lock(&mutex_);
perkj26091b12016-09-01 01:17:40 -07001399 expect_frames_ = false;
1400 }
1401
asaperssonfab67072017-04-04 05:51:49 -07001402 int number_of_reconfigurations() const {
Markus Handella3765182020-07-08 13:13:32 +02001403 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001404 return number_of_reconfigurations_;
1405 }
1406
asaperssonfab67072017-04-04 05:51:49 -07001407 int last_min_transmit_bitrate() const {
Markus Handella3765182020-07-08 13:13:32 +02001408 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001409 return min_transmit_bitrate_bps_;
1410 }
1411
Erik Språngd7329ca2019-02-21 21:19:53 +01001412 void SetNumExpectedLayers(size_t num_layers) {
Markus Handella3765182020-07-08 13:13:32 +02001413 MutexLock lock(&mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001414 num_expected_layers_ = num_layers;
1415 }
1416
Erik Språngb7cb7b52019-02-26 15:52:33 +01001417 int64_t GetLastCaptureTimeMs() const {
Markus Handella3765182020-07-08 13:13:32 +02001418 MutexLock lock(&mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001419 return last_capture_time_ms_;
1420 }
1421
Sergey Silkin0e42cf72021-03-15 10:12:57 +01001422 const EncodedImage& GetLastEncodedImage() {
1423 MutexLock lock(&mutex_);
1424 return last_encoded_image_;
1425 }
1426
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001427 std::vector<uint8_t> GetLastEncodedImageData() {
Markus Handella3765182020-07-08 13:13:32 +02001428 MutexLock lock(&mutex_);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001429 return std::move(last_encoded_image_data_);
1430 }
1431
Per Kjellanderdcef6412020-10-07 15:09:05 +02001432 VideoBitrateAllocation GetLastVideoBitrateAllocation() {
1433 MutexLock lock(&mutex_);
1434 return last_bitrate_allocation_;
1435 }
1436
1437 int number_of_bitrate_allocations() const {
1438 MutexLock lock(&mutex_);
1439 return number_of_bitrate_allocations_;
1440 }
1441
Per Kjellandera9434842020-10-15 17:53:22 +02001442 VideoLayersAllocation GetLastVideoLayersAllocation() {
1443 MutexLock lock(&mutex_);
1444 return last_layers_allocation_;
1445 }
1446
1447 int number_of_layers_allocations() const {
1448 MutexLock lock(&mutex_);
1449 return number_of_layers_allocations_;
1450 }
1451
perkj26091b12016-09-01 01:17:40 -07001452 private:
sergeyu2cb155a2016-11-04 11:39:29 -07001453 Result OnEncodedImage(
1454 const EncodedImage& encoded_image,
Danil Chapovalov2549f172020-08-12 17:30:36 +02001455 const CodecSpecificInfo* codec_specific_info) override {
Markus Handella3765182020-07-08 13:13:32 +02001456 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001457 EXPECT_TRUE(expect_frames_);
Sergey Silkin0e42cf72021-03-15 10:12:57 +01001458 last_encoded_image_ = EncodedImage(encoded_image);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001459 last_encoded_image_data_ = std::vector<uint8_t>(
1460 encoded_image.data(), encoded_image.data() + encoded_image.size());
Erik Språngd7329ca2019-02-21 21:19:53 +01001461 uint32_t timestamp = encoded_image.Timestamp();
1462 if (last_timestamp_ != timestamp) {
1463 num_received_layers_ = 1;
Erik Språng7444b192021-06-02 14:02:13 +02001464 last_width_ = encoded_image._encodedWidth;
1465 last_height_ = encoded_image._encodedHeight;
Erik Språngd7329ca2019-02-21 21:19:53 +01001466 } else {
1467 ++num_received_layers_;
Erik Språng7444b192021-06-02 14:02:13 +02001468 last_width_ = std::max(encoded_image._encodedWidth, last_width_);
1469 last_height_ = std::max(encoded_image._encodedHeight, last_height_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001470 }
1471 last_timestamp_ = timestamp;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001472 last_capture_time_ms_ = encoded_image.capture_time_ms_;
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001473 last_rotation_ = encoded_image.rotation_;
Erik Språngd7329ca2019-02-21 21:19:53 +01001474 if (num_received_layers_ == num_expected_layers_) {
1475 encoded_frame_event_.Set();
1476 }
sprangb1ca0732017-02-01 08:38:12 -08001477 return Result(Result::OK, last_timestamp_);
Per512ecb32016-09-23 15:52:06 +02001478 }
1479
Rasmus Brandtc402dbe2019-02-04 11:09:46 +01001480 void OnEncoderConfigurationChanged(
1481 std::vector<VideoStream> streams,
Ilya Nikolaevskiy93be66c2020-04-02 14:10:27 +02001482 bool is_svc,
Rasmus Brandtc402dbe2019-02-04 11:09:46 +01001483 VideoEncoderConfig::ContentType content_type,
1484 int min_transmit_bitrate_bps) override {
Markus Handella3765182020-07-08 13:13:32 +02001485 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001486 ++number_of_reconfigurations_;
1487 min_transmit_bitrate_bps_ = min_transmit_bitrate_bps;
1488 }
1489
Per Kjellanderdcef6412020-10-07 15:09:05 +02001490 void OnBitrateAllocationUpdated(
1491 const VideoBitrateAllocation& allocation) override {
1492 MutexLock lock(&mutex_);
1493 ++number_of_bitrate_allocations_;
1494 last_bitrate_allocation_ = allocation;
1495 }
1496
Per Kjellandera9434842020-10-15 17:53:22 +02001497 void OnVideoLayersAllocationUpdated(
1498 VideoLayersAllocation allocation) override {
1499 MutexLock lock(&mutex_);
1500 ++number_of_layers_allocations_;
1501 last_layers_allocation_ = allocation;
1502 rtc::StringBuilder log;
1503 for (const auto& layer : allocation.active_spatial_layers) {
1504 log << layer.width << "x" << layer.height << "@" << layer.frame_rate_fps
1505 << "[";
1506 for (const auto target_bitrate :
1507 layer.target_bitrate_per_temporal_layer) {
1508 log << target_bitrate.kbps() << ",";
1509 }
1510 log << "]";
1511 }
Harald Alvestrand97597c02021-11-04 12:01:23 +00001512 RTC_DLOG(LS_INFO) << "OnVideoLayersAllocationUpdated " << log.str();
Per Kjellandera9434842020-10-15 17:53:22 +02001513 }
1514
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001515 TimeController* const time_controller_;
Markus Handella3765182020-07-08 13:13:32 +02001516 mutable Mutex mutex_;
perkj26091b12016-09-01 01:17:40 -07001517 TestEncoder* test_encoder_;
1518 rtc::Event encoded_frame_event_;
Sergey Silkin0e42cf72021-03-15 10:12:57 +01001519 EncodedImage last_encoded_image_;
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001520 std::vector<uint8_t> last_encoded_image_data_;
sprangb1ca0732017-02-01 08:38:12 -08001521 uint32_t last_timestamp_ = 0;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001522 int64_t last_capture_time_ms_ = 0;
sprangb1ca0732017-02-01 08:38:12 -08001523 uint32_t last_height_ = 0;
1524 uint32_t last_width_ = 0;
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001525 VideoRotation last_rotation_ = kVideoRotation_0;
Erik Språngd7329ca2019-02-21 21:19:53 +01001526 size_t num_expected_layers_ = 1;
1527 size_t num_received_layers_ = 0;
perkj26091b12016-09-01 01:17:40 -07001528 bool expect_frames_ = true;
Per512ecb32016-09-23 15:52:06 +02001529 int number_of_reconfigurations_ = 0;
1530 int min_transmit_bitrate_bps_ = 0;
Per Kjellanderdcef6412020-10-07 15:09:05 +02001531 VideoBitrateAllocation last_bitrate_allocation_ RTC_GUARDED_BY(&mutex_);
1532 int number_of_bitrate_allocations_ RTC_GUARDED_BY(&mutex_) = 0;
Per Kjellandera9434842020-10-15 17:53:22 +02001533 VideoLayersAllocation last_layers_allocation_ RTC_GUARDED_BY(&mutex_);
1534 int number_of_layers_allocations_ RTC_GUARDED_BY(&mutex_) = 0;
perkj26091b12016-09-01 01:17:40 -07001535 };
1536
Sergey Silkin5ee69672019-07-02 14:18:34 +02001537 class VideoBitrateAllocatorProxyFactory
1538 : public VideoBitrateAllocatorFactory {
1539 public:
1540 VideoBitrateAllocatorProxyFactory()
1541 : bitrate_allocator_factory_(
1542 CreateBuiltinVideoBitrateAllocatorFactory()) {}
1543
1544 std::unique_ptr<VideoBitrateAllocator> CreateVideoBitrateAllocator(
1545 const VideoCodec& codec) override {
Markus Handella3765182020-07-08 13:13:32 +02001546 MutexLock lock(&mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001547 codec_config_ = codec;
1548 return bitrate_allocator_factory_->CreateVideoBitrateAllocator(codec);
1549 }
1550
1551 VideoCodec codec_config() const {
Markus Handella3765182020-07-08 13:13:32 +02001552 MutexLock lock(&mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001553 return codec_config_;
1554 }
1555
1556 private:
1557 std::unique_ptr<VideoBitrateAllocatorFactory> bitrate_allocator_factory_;
1558
Markus Handella3765182020-07-08 13:13:32 +02001559 mutable Mutex mutex_;
1560 VideoCodec codec_config_ RTC_GUARDED_BY(mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001561 };
1562
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001563 Clock* clock() { return time_controller_.GetClock(); }
1564 void AdvanceTime(TimeDelta duration) {
1565 time_controller_.AdvanceTime(duration);
1566 }
1567
1568 int64_t CurrentTimeMs() { return clock()->CurrentTime().ms(); }
1569
1570 protected:
1571 virtual TaskQueueFactory* GetTaskQueueFactory() {
1572 return time_controller_.GetTaskQueueFactory();
1573 }
1574
Jonas Orelandc7f691a2022-03-09 15:12:07 +01001575 test::ScopedKeyValueConfig field_trials_;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001576 GlobalSimulatedTimeController time_controller_{Timestamp::Micros(1234)};
perkj26091b12016-09-01 01:17:40 -07001577 VideoSendStream::Config video_send_config_;
Erik Språng08127a92016-11-16 16:41:30 +01001578 VideoEncoderConfig video_encoder_config_;
Per512ecb32016-09-23 15:52:06 +02001579 int codec_width_;
1580 int codec_height_;
sprang4847ae62017-06-27 07:06:52 -07001581 int max_framerate_;
perkj26091b12016-09-01 01:17:40 -07001582 TestEncoder fake_encoder_;
Niels Möllercbcbc222018-09-28 09:07:24 +02001583 test::VideoEncoderProxyFactory encoder_factory_;
Sergey Silkin5ee69672019-07-02 14:18:34 +02001584 VideoBitrateAllocatorProxyFactory bitrate_allocator_factory_;
sprangc5d62e22017-04-02 23:53:04 -07001585 std::unique_ptr<MockableSendStatisticsProxy> stats_proxy_;
perkj26091b12016-09-01 01:17:40 -07001586 TestSink sink_;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001587 AdaptingFrameForwarder video_source_{&time_controller_};
mflodmancc3d4422017-08-03 08:27:51 -07001588 std::unique_ptr<VideoStreamEncoderUnderTest> video_stream_encoder_;
perkj26091b12016-09-01 01:17:40 -07001589};
1590
mflodmancc3d4422017-08-03 08:27:51 -07001591TEST_F(VideoStreamEncoderTest, EncodeOneFrame) {
Henrik Boström381d1092020-05-12 18:49:07 +02001592 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001593 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möllerc572ff32018-11-07 08:43:50 +01001594 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 07:53:41 -07001595 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
sprang4847ae62017-06-27 07:06:52 -07001596 WaitForEncodedFrame(1);
perkja49cbd32016-09-16 07:53:41 -07001597 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
mflodmancc3d4422017-08-03 08:27:51 -07001598 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001599}
1600
mflodmancc3d4422017-08-03 08:27:51 -07001601TEST_F(VideoStreamEncoderTest, DropsFramesBeforeFirstOnBitrateUpdated) {
perkj26091b12016-09-01 01:17:40 -07001602 // Dropped since no target bitrate has been set.
Niels Möllerc572ff32018-11-07 08:43:50 +01001603 rtc::Event frame_destroyed_event;
Sebastian Janssona3177052018-04-10 13:05:49 +02001604 // The encoder will cache up to one frame for a short duration. Adding two
1605 // frames means that the first frame will be dropped and the second frame will
1606 // be sent when the encoder is enabled.
perkja49cbd32016-09-16 07:53:41 -07001607 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001608 AdvanceTime(TimeDelta::Millis(10));
Sebastian Janssona3177052018-04-10 13:05:49 +02001609 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Markus Handell28c71802021-11-08 10:11:55 +01001610 AdvanceTime(TimeDelta::Zero());
perkja49cbd32016-09-16 07:53:41 -07001611 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -07001612
Henrik Boström381d1092020-05-12 18:49:07 +02001613 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001614 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001615
Sebastian Janssona3177052018-04-10 13:05:49 +02001616 // The pending frame should be received.
sprang4847ae62017-06-27 07:06:52 -07001617 WaitForEncodedFrame(2);
Sebastian Janssona3177052018-04-10 13:05:49 +02001618 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
1619
1620 WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 08:27:51 -07001621 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001622}
1623
mflodmancc3d4422017-08-03 08:27:51 -07001624TEST_F(VideoStreamEncoderTest, DropsFramesWhenRateSetToZero) {
Henrik Boström381d1092020-05-12 18:49:07 +02001625 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001626 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkja49cbd32016-09-16 07:53:41 -07001627 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001628 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001629
Henrik Boström381d1092020-05-12 18:49:07 +02001630 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001631 DataRate::Zero(), DataRate::Zero(), DataRate::Zero(), 0, 0, 0);
1632
Sebastian Janssona3177052018-04-10 13:05:49 +02001633 // The encoder will cache up to one frame for a short duration. Adding two
1634 // frames means that the first frame will be dropped and the second frame will
1635 // be sent when the encoder is resumed.
perkja49cbd32016-09-16 07:53:41 -07001636 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Sebastian Janssona3177052018-04-10 13:05:49 +02001637 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
perkj26091b12016-09-01 01:17:40 -07001638
Henrik Boström381d1092020-05-12 18:49:07 +02001639 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001640 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07001641 WaitForEncodedFrame(3);
Sebastian Janssona3177052018-04-10 13:05:49 +02001642 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
1643 WaitForEncodedFrame(4);
mflodmancc3d4422017-08-03 08:27:51 -07001644 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001645}
1646
mflodmancc3d4422017-08-03 08:27:51 -07001647TEST_F(VideoStreamEncoderTest, DropsFramesWithSameOrOldNtpTimestamp) {
Henrik Boström381d1092020-05-12 18:49:07 +02001648 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001649 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkja49cbd32016-09-16 07:53:41 -07001650 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001651 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001652
1653 // This frame will be dropped since it has the same ntp timestamp.
perkja49cbd32016-09-16 07:53:41 -07001654 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
perkj26091b12016-09-01 01:17:40 -07001655
perkja49cbd32016-09-16 07:53:41 -07001656 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001657 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07001658 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001659}
1660
mflodmancc3d4422017-08-03 08:27:51 -07001661TEST_F(VideoStreamEncoderTest, DropsFrameAfterStop) {
Henrik Boström381d1092020-05-12 18:49:07 +02001662 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001663 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001664
perkja49cbd32016-09-16 07:53:41 -07001665 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001666 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001667
mflodmancc3d4422017-08-03 08:27:51 -07001668 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001669 sink_.SetExpectNoFrames();
Niels Möllerc572ff32018-11-07 08:43:50 +01001670 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 07:53:41 -07001671 video_source_.IncomingCapturedFrame(CreateFrame(2, &frame_destroyed_event));
1672 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -07001673}
1674
Markus Handell9a478b52021-11-18 16:07:01 +01001675TEST_F(VideoStreamEncoderTest, DropsPendingFramesOnSlowEncode) {
1676 test::FrameForwarder source;
1677 video_stream_encoder_->SetSource(&source,
1678 DegradationPreference::MAINTAIN_FRAMERATE);
Henrik Boström381d1092020-05-12 18:49:07 +02001679 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001680 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001681
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001682 int dropped_count = 0;
1683 stats_proxy_->SetDroppedFrameCallback(
1684 [&dropped_count](VideoStreamEncoderObserver::DropReason) {
1685 ++dropped_count;
1686 });
1687
Markus Handell9a478b52021-11-18 16:07:01 +01001688 source.IncomingCapturedFrame(CreateFrame(1, nullptr));
1689 source.IncomingCapturedFrame(CreateFrame(2, nullptr));
1690 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07001691 video_stream_encoder_->Stop();
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001692 EXPECT_EQ(1, dropped_count);
perkj26091b12016-09-01 01:17:40 -07001693}
1694
Henrik Boström56db9ff2021-03-24 09:06:45 +01001695TEST_F(VideoStreamEncoderTest, NativeFrameWithoutI420SupportGetsDelivered) {
Henrik Boström381d1092020-05-12 18:49:07 +02001696 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001697 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Noah Richards51db4212019-06-12 06:59:12 -07001698
1699 rtc::Event frame_destroyed_event;
1700 video_source_.IncomingCapturedFrame(
1701 CreateFakeNativeFrame(1, &frame_destroyed_event));
Henrik Boström56db9ff2021-03-24 09:06:45 +01001702 WaitForEncodedFrame(1);
1703 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
1704 fake_encoder_.GetLastInputPixelFormat());
Asa Persson606d3cb2021-10-04 10:07:11 +02001705 EXPECT_EQ(fake_encoder_.config().width, fake_encoder_.GetLastInputWidth());
1706 EXPECT_EQ(fake_encoder_.config().height, fake_encoder_.GetLastInputHeight());
Noah Richards51db4212019-06-12 06:59:12 -07001707 video_stream_encoder_->Stop();
1708}
1709
Henrik Boström56db9ff2021-03-24 09:06:45 +01001710TEST_F(VideoStreamEncoderTest,
1711 NativeFrameWithoutI420SupportGetsCroppedIfNecessary) {
Noah Richards51db4212019-06-12 06:59:12 -07001712 // Use the cropping factory.
1713 video_encoder_config_.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02001714 rtc::make_ref_counted<CroppingVideoStreamFactory>();
Noah Richards51db4212019-06-12 06:59:12 -07001715 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config_),
1716 kMaxPayloadLength);
1717 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
1718
1719 // Capture a frame at codec_width_/codec_height_.
Henrik Boström381d1092020-05-12 18:49:07 +02001720 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001721 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Noah Richards51db4212019-06-12 06:59:12 -07001722 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1723 WaitForEncodedFrame(1);
1724 // The encoder will have been configured once.
1725 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Asa Persson606d3cb2021-10-04 10:07:11 +02001726 EXPECT_EQ(codec_width_, fake_encoder_.config().width);
1727 EXPECT_EQ(codec_height_, fake_encoder_.config().height);
Noah Richards51db4212019-06-12 06:59:12 -07001728
1729 // Now send in a fake frame that needs to be cropped as the width/height
1730 // aren't divisible by 4 (see CreateEncoderStreams above).
1731 rtc::Event frame_destroyed_event;
1732 video_source_.IncomingCapturedFrame(CreateFakeNativeFrame(
1733 2, &frame_destroyed_event, codec_width_ + 1, codec_height_ + 1));
Henrik Boström56db9ff2021-03-24 09:06:45 +01001734 WaitForEncodedFrame(2);
1735 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
1736 fake_encoder_.GetLastInputPixelFormat());
Asa Persson606d3cb2021-10-04 10:07:11 +02001737 EXPECT_EQ(fake_encoder_.config().width, fake_encoder_.GetLastInputWidth());
1738 EXPECT_EQ(fake_encoder_.config().height, fake_encoder_.GetLastInputHeight());
Noah Richards51db4212019-06-12 06:59:12 -07001739 video_stream_encoder_->Stop();
1740}
1741
Evan Shrubsole895556e2020-10-05 09:15:13 +02001742TEST_F(VideoStreamEncoderTest, NonI420FramesShouldNotBeConvertedToI420) {
1743 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001744 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsole895556e2020-10-05 09:15:13 +02001745
1746 video_source_.IncomingCapturedFrame(
1747 CreateNV12Frame(1, codec_width_, codec_height_));
1748 WaitForEncodedFrame(1);
1749 EXPECT_EQ(VideoFrameBuffer::Type::kNV12,
1750 fake_encoder_.GetLastInputPixelFormat());
1751 video_stream_encoder_->Stop();
1752}
1753
Henrik Boström56db9ff2021-03-24 09:06:45 +01001754TEST_F(VideoStreamEncoderTest, NativeFrameGetsDelivered_NoFrameTypePreference) {
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001755 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001756 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001757
1758 fake_encoder_.SetPreferredPixelFormats({});
1759
1760 rtc::Event frame_destroyed_event;
1761 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1762 1, &frame_destroyed_event, codec_width_, codec_height_));
1763 WaitForEncodedFrame(1);
Henrik Boström56db9ff2021-03-24 09:06:45 +01001764 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001765 fake_encoder_.GetLastInputPixelFormat());
1766 video_stream_encoder_->Stop();
1767}
1768
Henrik Boström56db9ff2021-03-24 09:06:45 +01001769TEST_F(VideoStreamEncoderTest,
1770 NativeFrameGetsDelivered_PixelFormatPreferenceMatches) {
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001771 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001772 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001773
1774 fake_encoder_.SetPreferredPixelFormats({VideoFrameBuffer::Type::kNV12});
1775
1776 rtc::Event frame_destroyed_event;
1777 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1778 1, &frame_destroyed_event, codec_width_, codec_height_));
1779 WaitForEncodedFrame(1);
Henrik Boström56db9ff2021-03-24 09:06:45 +01001780 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001781 fake_encoder_.GetLastInputPixelFormat());
1782 video_stream_encoder_->Stop();
1783}
1784
Henrik Boström56db9ff2021-03-24 09:06:45 +01001785TEST_F(VideoStreamEncoderTest, NativeFrameGetsDelivered_MappingIsNotFeasible) {
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001786 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001787 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001788
1789 // Fake NV12 native frame does not allow mapping to I444.
1790 fake_encoder_.SetPreferredPixelFormats({VideoFrameBuffer::Type::kI444});
1791
1792 rtc::Event frame_destroyed_event;
1793 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1794 1, &frame_destroyed_event, codec_width_, codec_height_));
1795 WaitForEncodedFrame(1);
Henrik Boström56db9ff2021-03-24 09:06:45 +01001796 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001797 fake_encoder_.GetLastInputPixelFormat());
1798 video_stream_encoder_->Stop();
1799}
1800
Henrik Boström56db9ff2021-03-24 09:06:45 +01001801TEST_F(VideoStreamEncoderTest, NativeFrameGetsDelivered_BackedByNV12) {
Evan Shrubsole895556e2020-10-05 09:15:13 +02001802 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001803 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsole895556e2020-10-05 09:15:13 +02001804
1805 rtc::Event frame_destroyed_event;
1806 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1807 1, &frame_destroyed_event, codec_width_, codec_height_));
1808 WaitForEncodedFrame(1);
Henrik Boström56db9ff2021-03-24 09:06:45 +01001809 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
Evan Shrubsole895556e2020-10-05 09:15:13 +02001810 fake_encoder_.GetLastInputPixelFormat());
1811 video_stream_encoder_->Stop();
1812}
1813
Ying Wang9b881ab2020-02-07 14:29:32 +01001814TEST_F(VideoStreamEncoderTest, DropsFramesWhenCongestionWindowPushbackSet) {
Henrik Boström381d1092020-05-12 18:49:07 +02001815 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001816 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ying Wang9b881ab2020-02-07 14:29:32 +01001817 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1818 WaitForEncodedFrame(1);
1819
Henrik Boström381d1092020-05-12 18:49:07 +02001820 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001821 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0.5);
Ying Wang9b881ab2020-02-07 14:29:32 +01001822 // The congestion window pushback is set to 0.5, which will drop 1/2 of
1823 // frames. Adding two frames means that the first frame will be dropped and
1824 // the second frame will be sent to the encoder.
1825 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1826 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
1827 WaitForEncodedFrame(3);
1828 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
1829 video_source_.IncomingCapturedFrame(CreateFrame(5, nullptr));
1830 WaitForEncodedFrame(5);
1831 EXPECT_EQ(2u, stats_proxy_->GetStats().frames_dropped_by_congestion_window);
1832 video_stream_encoder_->Stop();
1833}
1834
mflodmancc3d4422017-08-03 08:27:51 -07001835TEST_F(VideoStreamEncoderTest,
1836 ConfigureEncoderTriggersOnEncoderConfigurationChanged) {
Henrik Boström381d1092020-05-12 18:49:07 +02001837 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001838 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per21d45d22016-10-30 21:37:57 +01001839 EXPECT_EQ(0, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 15:52:06 +02001840
1841 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 16:24:55 +02001842 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001843 WaitForEncodedFrame(1);
Per21d45d22016-10-30 21:37:57 +01001844 // The encoder will have been configured once when the first frame is
1845 // received.
1846 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 15:52:06 +02001847
1848 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02001849 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Per512ecb32016-09-23 15:52:06 +02001850 video_encoder_config.min_transmit_bitrate_bps = 9999;
mflodmancc3d4422017-08-03 08:27:51 -07001851 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02001852 kMaxPayloadLength);
Per512ecb32016-09-23 15:52:06 +02001853
1854 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 16:24:55 +02001855 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001856 WaitForEncodedFrame(2);
Per21d45d22016-10-30 21:37:57 +01001857 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkj3b703ed2016-09-29 23:25:40 -07001858 EXPECT_EQ(9999, sink_.last_min_transmit_bitrate());
perkj26105b42016-09-29 22:39:10 -07001859
mflodmancc3d4422017-08-03 08:27:51 -07001860 video_stream_encoder_->Stop();
perkj26105b42016-09-29 22:39:10 -07001861}
1862
mflodmancc3d4422017-08-03 08:27:51 -07001863TEST_F(VideoStreamEncoderTest, FrameResolutionChangeReconfigureEncoder) {
Henrik Boström381d1092020-05-12 18:49:07 +02001864 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001865 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkjfa10b552016-10-02 23:45:26 -07001866
1867 // Capture a frame and wait for it to synchronize with the encoder thread.
1868 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001869 WaitForEncodedFrame(1);
Per21d45d22016-10-30 21:37:57 +01001870 // The encoder will have been configured once.
1871 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Asa Persson606d3cb2021-10-04 10:07:11 +02001872 EXPECT_EQ(codec_width_, fake_encoder_.config().width);
1873 EXPECT_EQ(codec_height_, fake_encoder_.config().height);
perkjfa10b552016-10-02 23:45:26 -07001874
1875 codec_width_ *= 2;
1876 codec_height_ *= 2;
1877 // Capture a frame with a higher resolution and wait for it to synchronize
1878 // with the encoder thread.
1879 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001880 WaitForEncodedFrame(2);
Asa Persson606d3cb2021-10-04 10:07:11 +02001881 EXPECT_EQ(codec_width_, fake_encoder_.config().width);
1882 EXPECT_EQ(codec_height_, fake_encoder_.config().height);
Per21d45d22016-10-30 21:37:57 +01001883 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkjfa10b552016-10-02 23:45:26 -07001884
mflodmancc3d4422017-08-03 08:27:51 -07001885 video_stream_encoder_->Stop();
perkjfa10b552016-10-02 23:45:26 -07001886}
1887
Sergey Silkin443b7ee2019-06-28 12:53:07 +02001888TEST_F(VideoStreamEncoderTest,
1889 EncoderInstanceDestroyedBeforeAnotherInstanceCreated) {
Henrik Boström381d1092020-05-12 18:49:07 +02001890 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001891 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkin443b7ee2019-06-28 12:53:07 +02001892
1893 // Capture a frame and wait for it to synchronize with the encoder thread.
1894 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1895 WaitForEncodedFrame(1);
1896
1897 VideoEncoderConfig video_encoder_config;
1898 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1899 // Changing the max payload data length recreates encoder.
1900 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1901 kMaxPayloadLength / 2);
1902
1903 // Capture a frame and wait for it to synchronize with the encoder thread.
1904 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1905 WaitForEncodedFrame(2);
1906 EXPECT_EQ(1, encoder_factory_.GetMaxNumberOfSimultaneousEncoderInstances());
1907
1908 video_stream_encoder_->Stop();
1909}
1910
Sergey Silkin5ee69672019-07-02 14:18:34 +02001911TEST_F(VideoStreamEncoderTest, BitrateLimitsChangeReconfigureRateAllocator) {
Henrik Boström381d1092020-05-12 18:49:07 +02001912 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001913 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001914
1915 VideoEncoderConfig video_encoder_config;
1916 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02001917 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
1918 video_stream_encoder_->SetStartBitrate(kStartBitrate.bps());
Sergey Silkin5ee69672019-07-02 14:18:34 +02001919 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1920 kMaxPayloadLength);
1921
1922 // Capture a frame and wait for it to synchronize with the encoder thread.
1923 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1924 WaitForEncodedFrame(1);
1925 // The encoder will have been configured once when the first frame is
1926 // received.
1927 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Asa Persson606d3cb2021-10-04 10:07:11 +02001928 EXPECT_EQ(kTargetBitrate.bps(),
Sergey Silkin5ee69672019-07-02 14:18:34 +02001929 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
Asa Persson606d3cb2021-10-04 10:07:11 +02001930 EXPECT_EQ(kStartBitrate.bps(),
Sergey Silkin5ee69672019-07-02 14:18:34 +02001931 bitrate_allocator_factory_.codec_config().startBitrate * 1000);
1932
Sergey Silkin6456e352019-07-08 17:56:40 +02001933 test::FillEncoderConfiguration(kVideoCodecVP8, 1,
1934 &video_encoder_config); //???
Asa Persson606d3cb2021-10-04 10:07:11 +02001935 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps() * 2;
1936 video_stream_encoder_->SetStartBitrate(kStartBitrate.bps() * 2);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001937 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1938 kMaxPayloadLength);
1939
1940 // Capture a frame and wait for it to synchronize with the encoder thread.
1941 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1942 WaitForEncodedFrame(2);
1943 EXPECT_EQ(2, sink_.number_of_reconfigurations());
1944 // Bitrate limits have changed - rate allocator should be reconfigured,
1945 // encoder should not be reconfigured.
Asa Persson606d3cb2021-10-04 10:07:11 +02001946 EXPECT_EQ(kTargetBitrate.bps() * 2,
Sergey Silkin5ee69672019-07-02 14:18:34 +02001947 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
Asa Persson606d3cb2021-10-04 10:07:11 +02001948 EXPECT_EQ(kStartBitrate.bps() * 2,
Sergey Silkin5ee69672019-07-02 14:18:34 +02001949 bitrate_allocator_factory_.codec_config().startBitrate * 1000);
Asa Persson606d3cb2021-10-04 10:07:11 +02001950 EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
Sergey Silkin5ee69672019-07-02 14:18:34 +02001951
1952 video_stream_encoder_->Stop();
1953}
1954
Sergey Silkin6456e352019-07-08 17:56:40 +02001955TEST_F(VideoStreamEncoderTest,
Sergey Silkincd02eba2020-01-20 14:48:40 +01001956 IntersectionOfEncoderAndAppBitrateLimitsUsedWhenBothProvided) {
Henrik Boström381d1092020-05-12 18:49:07 +02001957 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001958 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkin6456e352019-07-08 17:56:40 +02001959
Sergey Silkincd02eba2020-01-20 14:48:40 +01001960 const uint32_t kMinEncBitrateKbps = 100;
1961 const uint32_t kMaxEncBitrateKbps = 1000;
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001962 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
Sergey Silkincd02eba2020-01-20 14:48:40 +01001963 /*frame_size_pixels=*/codec_width_ * codec_height_,
1964 /*min_start_bitrate_bps=*/0,
1965 /*min_bitrate_bps=*/kMinEncBitrateKbps * 1000,
1966 /*max_bitrate_bps=*/kMaxEncBitrateKbps * 1000);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001967 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
1968
Sergey Silkincd02eba2020-01-20 14:48:40 +01001969 VideoEncoderConfig video_encoder_config;
1970 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1971 video_encoder_config.max_bitrate_bps = (kMaxEncBitrateKbps + 1) * 1000;
1972 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1973 (kMinEncBitrateKbps + 1) * 1000;
1974 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1975 kMaxPayloadLength);
1976
1977 // When both encoder and app provide bitrate limits, the intersection of
1978 // provided sets should be used.
1979 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1980 WaitForEncodedFrame(1);
1981 EXPECT_EQ(kMaxEncBitrateKbps,
1982 bitrate_allocator_factory_.codec_config().maxBitrate);
1983 EXPECT_EQ(kMinEncBitrateKbps + 1,
1984 bitrate_allocator_factory_.codec_config().minBitrate);
1985
1986 video_encoder_config.max_bitrate_bps = (kMaxEncBitrateKbps - 1) * 1000;
1987 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1988 (kMinEncBitrateKbps - 1) * 1000;
1989 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1990 kMaxPayloadLength);
1991 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001992 WaitForEncodedFrame(2);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001993 EXPECT_EQ(kMaxEncBitrateKbps - 1,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001994 bitrate_allocator_factory_.codec_config().maxBitrate);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001995 EXPECT_EQ(kMinEncBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001996 bitrate_allocator_factory_.codec_config().minBitrate);
1997
Sergey Silkincd02eba2020-01-20 14:48:40 +01001998 video_stream_encoder_->Stop();
1999}
2000
2001TEST_F(VideoStreamEncoderTest,
2002 EncoderAndAppLimitsDontIntersectEncoderLimitsIgnored) {
Henrik Boström381d1092020-05-12 18:49:07 +02002003 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002004 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkincd02eba2020-01-20 14:48:40 +01002005
2006 const uint32_t kMinAppBitrateKbps = 100;
2007 const uint32_t kMaxAppBitrateKbps = 200;
2008 const uint32_t kMinEncBitrateKbps = kMaxAppBitrateKbps + 1;
2009 const uint32_t kMaxEncBitrateKbps = kMaxAppBitrateKbps * 2;
2010 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
2011 /*frame_size_pixels=*/codec_width_ * codec_height_,
2012 /*min_start_bitrate_bps=*/0,
2013 /*min_bitrate_bps=*/kMinEncBitrateKbps * 1000,
2014 /*max_bitrate_bps=*/kMaxEncBitrateKbps * 1000);
2015 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
2016
2017 VideoEncoderConfig video_encoder_config;
2018 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
2019 video_encoder_config.max_bitrate_bps = kMaxAppBitrateKbps * 1000;
2020 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
2021 kMinAppBitrateKbps * 1000;
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002022 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
2023 kMaxPayloadLength);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002024
Sergey Silkincd02eba2020-01-20 14:48:40 +01002025 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
2026 WaitForEncodedFrame(1);
2027 EXPECT_EQ(kMaxAppBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002028 bitrate_allocator_factory_.codec_config().maxBitrate);
Sergey Silkincd02eba2020-01-20 14:48:40 +01002029 EXPECT_EQ(kMinAppBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002030 bitrate_allocator_factory_.codec_config().minBitrate);
Sergey Silkin6456e352019-07-08 17:56:40 +02002031
2032 video_stream_encoder_->Stop();
2033}
2034
2035TEST_F(VideoStreamEncoderTest,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002036 EncoderRecommendedMaxAndMinBitratesUsedForGivenResolution) {
Henrik Boström381d1092020-05-12 18:49:07 +02002037 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002038 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkin6456e352019-07-08 17:56:40 +02002039
2040 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits_270p(
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002041 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002042 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits_360p(
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002043 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002044 fake_encoder_.SetResolutionBitrateLimits(
2045 {encoder_bitrate_limits_270p, encoder_bitrate_limits_360p});
2046
2047 VideoEncoderConfig video_encoder_config;
2048 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
2049 video_encoder_config.max_bitrate_bps = 0;
2050 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
2051 kMaxPayloadLength);
2052
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002053 // 270p. The bitrate limits recommended by encoder for 270p should be used.
Sergey Silkin6456e352019-07-08 17:56:40 +02002054 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
2055 WaitForEncodedFrame(1);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002056 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
2057 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002058 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
2059 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
2060
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002061 // 360p. The bitrate limits recommended by encoder for 360p should be used.
Sergey Silkin6456e352019-07-08 17:56:40 +02002062 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2063 WaitForEncodedFrame(2);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002064 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
2065 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002066 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
2067 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
2068
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002069 // Resolution between 270p and 360p. The bitrate limits recommended by
Sergey Silkin6456e352019-07-08 17:56:40 +02002070 // encoder for 360p should be used.
2071 video_source_.IncomingCapturedFrame(
2072 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
2073 WaitForEncodedFrame(3);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002074 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
2075 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002076 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
2077 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
2078
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002079 // Resolution higher than 360p. The caps recommended by encoder should be
Sergey Silkin6456e352019-07-08 17:56:40 +02002080 // ignored.
2081 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
2082 WaitForEncodedFrame(4);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002083 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
2084 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002085 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
2086 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002087 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
2088 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002089 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
2090 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
2091
2092 // Resolution lower than 270p. The max bitrate limit recommended by encoder
2093 // for 270p should be used.
2094 video_source_.IncomingCapturedFrame(CreateFrame(5, 320, 180));
2095 WaitForEncodedFrame(5);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002096 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
2097 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002098 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
2099 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
2100
2101 video_stream_encoder_->Stop();
2102}
2103
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002104TEST_F(VideoStreamEncoderTest, EncoderRecommendedMaxBitrateCapsTargetBitrate) {
Henrik Boström381d1092020-05-12 18:49:07 +02002105 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002106 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002107
2108 VideoEncoderConfig video_encoder_config;
2109 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
2110 video_encoder_config.max_bitrate_bps = 0;
2111 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
2112 kMaxPayloadLength);
2113
2114 // Encode 720p frame to get the default encoder target bitrate.
2115 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
2116 WaitForEncodedFrame(1);
2117 const uint32_t kDefaultTargetBitrateFor720pKbps =
2118 bitrate_allocator_factory_.codec_config()
2119 .simulcastStream[0]
2120 .targetBitrate;
2121
2122 // Set the max recommended encoder bitrate to something lower than the default
2123 // target bitrate.
2124 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
2125 1280 * 720, 10 * 1000, 10 * 1000,
2126 kDefaultTargetBitrateFor720pKbps / 2 * 1000);
2127 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
2128
2129 // Change resolution to trigger encoder reinitialization.
2130 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2131 WaitForEncodedFrame(2);
2132 video_source_.IncomingCapturedFrame(CreateFrame(3, 1280, 720));
2133 WaitForEncodedFrame(3);
2134
2135 // Ensure the target bitrate is capped by the max bitrate.
2136 EXPECT_EQ(bitrate_allocator_factory_.codec_config().maxBitrate * 1000,
2137 static_cast<uint32_t>(encoder_bitrate_limits.max_bitrate_bps));
2138 EXPECT_EQ(bitrate_allocator_factory_.codec_config()
2139 .simulcastStream[0]
2140 .targetBitrate *
2141 1000,
2142 static_cast<uint32_t>(encoder_bitrate_limits.max_bitrate_bps));
2143
2144 video_stream_encoder_->Stop();
2145}
2146
Åsa Perssona7e34d32021-01-20 15:36:13 +01002147TEST_F(VideoStreamEncoderTest,
2148 EncoderMaxAndMinBitratesUsedForTwoStreamsHighestActive) {
2149 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2150 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2151 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2152 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2153 fake_encoder_.SetResolutionBitrateLimits(
2154 {kEncoderLimits270p, kEncoderLimits360p});
2155
2156 // Two streams, highest stream active.
2157 VideoEncoderConfig config;
2158 const int kNumStreams = 2;
2159 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2160 config.max_bitrate_bps = 0;
2161 config.simulcast_layers[0].active = false;
2162 config.simulcast_layers[1].active = true;
2163 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002164 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssona7e34d32021-01-20 15:36:13 +01002165 "VP8", /*max qp*/ 56, /*screencast*/ false,
2166 /*screenshare enabled*/ false);
2167 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2168
2169 // The encoder bitrate limits for 270p should be used.
2170 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
2171 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02002172 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002173 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002174 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002175 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002176 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002177
2178 // The encoder bitrate limits for 360p should be used.
2179 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2180 EXPECT_FALSE(WaitForFrame(1000));
2181 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002182 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002183 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002184 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002185
2186 // Resolution b/w 270p and 360p. The encoder limits for 360p should be used.
2187 video_source_.IncomingCapturedFrame(
2188 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
2189 EXPECT_FALSE(WaitForFrame(1000));
2190 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002191 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002192 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002193 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002194
2195 // Resolution higher than 360p. Encoder limits should be ignored.
2196 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
2197 EXPECT_FALSE(WaitForFrame(1000));
2198 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002199 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002200 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002201 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002202 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002203 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002204 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002205 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002206
2207 // Resolution lower than 270p. The encoder limits for 270p should be used.
2208 video_source_.IncomingCapturedFrame(CreateFrame(5, 320, 180));
2209 EXPECT_FALSE(WaitForFrame(1000));
2210 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002211 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002212 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002213 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002214
2215 video_stream_encoder_->Stop();
2216}
2217
2218TEST_F(VideoStreamEncoderTest,
Åsa Persson258e9892021-02-25 10:39:51 +01002219 DefaultEncoderMaxAndMinBitratesUsedForTwoStreamsHighestActive) {
2220 // Two streams, highest stream active.
2221 VideoEncoderConfig config;
2222 const int kNumStreams = 2;
2223 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2224 config.max_bitrate_bps = 0;
2225 config.simulcast_layers[0].active = false;
2226 config.simulcast_layers[1].active = true;
2227 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002228 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Persson258e9892021-02-25 10:39:51 +01002229 "VP8", /*max qp*/ 56, /*screencast*/ false,
2230 /*screenshare enabled*/ false);
2231 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2232
2233 // Default bitrate limits for 270p should be used.
2234 const absl::optional<VideoEncoder::ResolutionBitrateLimits>
2235 kDefaultLimits270p =
2236 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
Sergey Silkina86b29b2021-03-05 13:29:19 +01002237 kVideoCodecVP8, 480 * 270);
Åsa Persson258e9892021-02-25 10:39:51 +01002238 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
2239 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02002240 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
Åsa Persson258e9892021-02-25 10:39:51 +01002241 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits270p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002242 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002243 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits270p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002244 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002245
2246 // Default bitrate limits for 360p should be used.
2247 const absl::optional<VideoEncoder::ResolutionBitrateLimits>
2248 kDefaultLimits360p =
2249 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
Sergey Silkina86b29b2021-03-05 13:29:19 +01002250 kVideoCodecVP8, 640 * 360);
Åsa Persson258e9892021-02-25 10:39:51 +01002251 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2252 EXPECT_FALSE(WaitForFrame(1000));
2253 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002254 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002255 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002256 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002257
2258 // Resolution b/w 270p and 360p. The default limits for 360p should be used.
2259 video_source_.IncomingCapturedFrame(
2260 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
2261 EXPECT_FALSE(WaitForFrame(1000));
2262 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002263 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002264 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002265 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002266
2267 // Default bitrate limits for 540p should be used.
2268 const absl::optional<VideoEncoder::ResolutionBitrateLimits>
2269 kDefaultLimits540p =
2270 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
Sergey Silkina86b29b2021-03-05 13:29:19 +01002271 kVideoCodecVP8, 960 * 540);
Åsa Persson258e9892021-02-25 10:39:51 +01002272 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
2273 EXPECT_FALSE(WaitForFrame(1000));
2274 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits540p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002275 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002276 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits540p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002277 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002278
2279 video_stream_encoder_->Stop();
2280}
2281
2282TEST_F(VideoStreamEncoderTest,
Åsa Perssona7e34d32021-01-20 15:36:13 +01002283 EncoderMaxAndMinBitratesUsedForThreeStreamsMiddleActive) {
2284 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2285 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2286 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2287 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2288 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
2289 1280 * 720, 54 * 1000, 31 * 1000, 3456 * 1000);
2290 fake_encoder_.SetResolutionBitrateLimits(
2291 {kEncoderLimits270p, kEncoderLimits360p, kEncoderLimits720p});
2292
2293 // Three streams, middle stream active.
2294 VideoEncoderConfig config;
2295 const int kNumStreams = 3;
2296 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2297 config.simulcast_layers[0].active = false;
2298 config.simulcast_layers[1].active = true;
2299 config.simulcast_layers[2].active = false;
2300 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002301 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssona7e34d32021-01-20 15:36:13 +01002302 "VP8", /*max qp*/ 56, /*screencast*/ false,
2303 /*screenshare enabled*/ false);
2304 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2305
2306 // The encoder bitrate limits for 360p should be used.
2307 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
2308 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02002309 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002310 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002311 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002312 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002313 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002314
2315 // The encoder bitrate limits for 270p should be used.
2316 video_source_.IncomingCapturedFrame(CreateFrame(2, 960, 540));
2317 EXPECT_FALSE(WaitForFrame(1000));
2318 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002319 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002320 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002321 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002322
2323 video_stream_encoder_->Stop();
2324}
2325
2326TEST_F(VideoStreamEncoderTest,
2327 EncoderMaxAndMinBitratesNotUsedForThreeStreamsLowestActive) {
2328 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2329 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2330 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2331 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2332 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
2333 1280 * 720, 54 * 1000, 31 * 1000, 3456 * 1000);
2334 fake_encoder_.SetResolutionBitrateLimits(
2335 {kEncoderLimits270p, kEncoderLimits360p, kEncoderLimits720p});
2336
2337 // Three streams, lowest stream active.
2338 VideoEncoderConfig config;
2339 const int kNumStreams = 3;
2340 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2341 config.simulcast_layers[0].active = true;
2342 config.simulcast_layers[1].active = false;
2343 config.simulcast_layers[2].active = false;
2344 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002345 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssona7e34d32021-01-20 15:36:13 +01002346 "VP8", /*max qp*/ 56, /*screencast*/ false,
2347 /*screenshare enabled*/ false);
2348 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2349
2350 // Resolution on lowest stream lower than 270p. The encoder limits not applied
2351 // on lowest stream, limits for 270p should not be used
2352 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
2353 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02002354 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002355 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002356 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002357 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002358 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002359
2360 video_stream_encoder_->Stop();
2361}
2362
2363TEST_F(VideoStreamEncoderTest,
2364 EncoderMaxBitrateCappedByConfigForTwoStreamsHighestActive) {
2365 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2366 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2367 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2368 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2369 fake_encoder_.SetResolutionBitrateLimits(
2370 {kEncoderLimits270p, kEncoderLimits360p});
2371 const int kMaxBitrateBps = kEncoderLimits360p.max_bitrate_bps - 100 * 1000;
2372
2373 // Two streams, highest stream active.
2374 VideoEncoderConfig config;
2375 const int kNumStreams = 2;
2376 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2377 config.simulcast_layers[0].active = false;
2378 config.simulcast_layers[1].active = true;
2379 config.simulcast_layers[1].max_bitrate_bps = kMaxBitrateBps;
2380 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002381 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssona7e34d32021-01-20 15:36:13 +01002382 "VP8", /*max qp*/ 56, /*screencast*/ false,
2383 /*screenshare enabled*/ false);
2384 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2385
2386 // The encoder bitrate limits for 270p should be used.
2387 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
2388 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02002389 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002390 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002391 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002392 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002393 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002394
2395 // The max configured bitrate is less than the encoder limit for 360p.
2396 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2397 EXPECT_FALSE(WaitForFrame(1000));
2398 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002399 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002400 EXPECT_EQ(static_cast<uint32_t>(kMaxBitrateBps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002401 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002402
2403 video_stream_encoder_->Stop();
2404}
2405
mflodmancc3d4422017-08-03 08:27:51 -07002406TEST_F(VideoStreamEncoderTest, SwitchSourceDeregisterEncoderAsSink) {
perkj803d97f2016-11-01 11:45:46 -07002407 EXPECT_TRUE(video_source_.has_sinks());
2408 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002409 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002410 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 11:45:46 -07002411 EXPECT_FALSE(video_source_.has_sinks());
2412 EXPECT_TRUE(new_video_source.has_sinks());
2413
mflodmancc3d4422017-08-03 08:27:51 -07002414 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002415}
2416
mflodmancc3d4422017-08-03 08:27:51 -07002417TEST_F(VideoStreamEncoderTest, SinkWantsRotationApplied) {
perkj803d97f2016-11-01 11:45:46 -07002418 EXPECT_FALSE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 08:27:51 -07002419 video_stream_encoder_->SetSink(&sink_, true /*rotation_applied*/);
perkj803d97f2016-11-01 11:45:46 -07002420 EXPECT_TRUE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 08:27:51 -07002421 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002422}
2423
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002424class ResolutionAlignmentTest
2425 : public VideoStreamEncoderTest,
2426 public ::testing::WithParamInterface<
2427 ::testing::tuple<int, std::vector<double>>> {
2428 public:
2429 ResolutionAlignmentTest()
2430 : requested_alignment_(::testing::get<0>(GetParam())),
2431 scale_factors_(::testing::get<1>(GetParam())) {}
2432
2433 protected:
2434 const int requested_alignment_;
2435 const std::vector<double> scale_factors_;
2436};
2437
2438INSTANTIATE_TEST_SUITE_P(
2439 AlignmentAndScaleFactors,
2440 ResolutionAlignmentTest,
2441 ::testing::Combine(
2442 ::testing::Values(1, 2, 3, 4, 5, 6, 16, 22), // requested_alignment_
2443 ::testing::Values(std::vector<double>{-1.0}, // scale_factors_
2444 std::vector<double>{-1.0, -1.0},
2445 std::vector<double>{-1.0, -1.0, -1.0},
2446 std::vector<double>{4.0, 2.0, 1.0},
2447 std::vector<double>{9999.0, -1.0, 1.0},
2448 std::vector<double>{3.99, 2.01, 1.0},
2449 std::vector<double>{4.9, 1.7, 1.25},
2450 std::vector<double>{10.0, 4.0, 3.0},
2451 std::vector<double>{1.75, 3.5},
2452 std::vector<double>{1.5, 2.5},
2453 std::vector<double>{1.3, 1.0})));
2454
2455TEST_P(ResolutionAlignmentTest, SinkWantsAlignmentApplied) {
2456 // Set requested resolution alignment.
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002457 video_source_.set_adaptation_enabled(true);
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002458 fake_encoder_.SetRequestedResolutionAlignment(requested_alignment_);
2459 fake_encoder_.SetApplyAlignmentToAllSimulcastLayers(true);
2460
2461 // Fill config with the scaling factor by which to reduce encoding size.
2462 const int num_streams = scale_factors_.size();
2463 VideoEncoderConfig config;
2464 test::FillEncoderConfiguration(kVideoCodecVP8, num_streams, &config);
2465 for (int i = 0; i < num_streams; ++i) {
2466 config.simulcast_layers[i].scale_resolution_down_by = scale_factors_[i];
2467 }
2468 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002469 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002470 "VP8", /*max qp*/ 56, /*screencast*/ false,
2471 /*screenshare enabled*/ false);
2472 video_stream_encoder_->ConfigureEncoder(std::move(config), kMaxPayloadLength);
2473
Henrik Boström381d1092020-05-12 18:49:07 +02002474 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002475 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
2476 0, 0, 0);
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002477 // Wait for all layers before triggering event.
2478 sink_.SetNumExpectedLayers(num_streams);
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002479
2480 // On the 1st frame, we should have initialized the encoder and
2481 // asked for its resolution requirements.
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002482 int64_t timestamp_ms = kFrameIntervalMs;
2483 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
2484 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02002485 EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002486
2487 // On the 2nd frame, we should be receiving a correctly aligned resolution.
2488 // (It's up the to the encoder to potentially drop the previous frame,
2489 // to avoid coding back-to-back keyframes.)
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002490 timestamp_ms += kFrameIntervalMs;
2491 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
2492 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02002493 EXPECT_GE(fake_encoder_.GetNumInitializations(), 1);
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002494
Asa Persson606d3cb2021-10-04 10:07:11 +02002495 VideoCodec codec = fake_encoder_.config();
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002496 EXPECT_EQ(codec.numberOfSimulcastStreams, num_streams);
2497 // Frame size should be a multiple of the requested alignment.
2498 for (int i = 0; i < codec.numberOfSimulcastStreams; ++i) {
2499 EXPECT_EQ(codec.simulcastStream[i].width % requested_alignment_, 0);
2500 EXPECT_EQ(codec.simulcastStream[i].height % requested_alignment_, 0);
2501 // Aspect ratio should match.
2502 EXPECT_EQ(codec.width * codec.simulcastStream[i].height,
2503 codec.height * codec.simulcastStream[i].width);
2504 }
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002505
2506 video_stream_encoder_->Stop();
2507}
2508
Jonathan Yubc771b72017-12-08 17:04:29 -08002509TEST_F(VideoStreamEncoderTest, TestCpuDowngrades_BalancedMode) {
2510 const int kFramerateFps = 30;
asaperssonf7e294d2017-06-13 23:25:22 -07002511 const int kWidth = 1280;
2512 const int kHeight = 720;
Jonathan Yubc771b72017-12-08 17:04:29 -08002513
2514 // We rely on the automatic resolution adaptation, but we handle framerate
2515 // adaptation manually by mocking the stats proxy.
2516 video_source_.set_adaptation_enabled(true);
asaperssonf7e294d2017-06-13 23:25:22 -07002517
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002518 // Enable BALANCED preference, no initial limitation.
Henrik Boström381d1092020-05-12 18:49:07 +02002519 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002520 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002521 video_stream_encoder_->SetSource(&video_source_,
2522 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002523 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
asaperssonf7e294d2017-06-13 23:25:22 -07002524 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002525 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asaperssonf7e294d2017-06-13 23:25:22 -07002526 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2527
Jonathan Yubc771b72017-12-08 17:04:29 -08002528 // Adapt down as far as possible.
2529 rtc::VideoSinkWants last_wants;
2530 int64_t t = 1;
2531 int loop_count = 0;
2532 do {
2533 ++loop_count;
2534 last_wants = video_source_.sink_wants();
2535
2536 // Simulate the framerate we've been asked to adapt to.
2537 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
2538 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
2539 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2540 mock_stats.input_frame_rate = fps;
2541 stats_proxy_->SetMockStats(mock_stats);
2542
2543 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
2544 sink_.WaitForEncodedFrame(t);
2545 t += frame_interval_ms;
2546
mflodmancc3d4422017-08-03 08:27:51 -07002547 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002548 EXPECT_THAT(
Jonathan Yubc771b72017-12-08 17:04:29 -08002549 video_source_.sink_wants(),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002550 FpsInRangeForPixelsInBalanced(*video_source_.last_sent_width() *
2551 *video_source_.last_sent_height()));
Jonathan Yubc771b72017-12-08 17:04:29 -08002552 } while (video_source_.sink_wants().max_pixel_count <
2553 last_wants.max_pixel_count ||
2554 video_source_.sink_wants().max_framerate_fps <
2555 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07002556
Jonathan Yubc771b72017-12-08 17:04:29 -08002557 // Verify that we've adapted all the way down.
2558 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-13 23:25:22 -07002559 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002560 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
2561 EXPECT_EQ(loop_count - 1,
asaperssonf7e294d2017-06-13 23:25:22 -07002562 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Jonathan Yubc771b72017-12-08 17:04:29 -08002563 EXPECT_EQ(kMinPixelsPerFrame, *video_source_.last_sent_width() *
2564 *video_source_.last_sent_height());
2565 EXPECT_EQ(kMinBalancedFramerateFps,
2566 video_source_.sink_wants().max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07002567
Jonathan Yubc771b72017-12-08 17:04:29 -08002568 // Adapt back up the same number of times we adapted down.
2569 for (int i = 0; i < loop_count - 1; ++i) {
2570 last_wants = video_source_.sink_wants();
2571
2572 // Simulate the framerate we've been asked to adapt to.
2573 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
2574 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
2575 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2576 mock_stats.input_frame_rate = fps;
2577 stats_proxy_->SetMockStats(mock_stats);
2578
2579 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
2580 sink_.WaitForEncodedFrame(t);
2581 t += frame_interval_ms;
2582
Henrik Boström91aa7322020-04-28 12:24:33 +02002583 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002584 EXPECT_THAT(
Jonathan Yubc771b72017-12-08 17:04:29 -08002585 video_source_.sink_wants(),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002586 FpsInRangeForPixelsInBalanced(*video_source_.last_sent_width() *
2587 *video_source_.last_sent_height()));
Jonathan Yubc771b72017-12-08 17:04:29 -08002588 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count >
2589 last_wants.max_pixel_count ||
2590 video_source_.sink_wants().max_framerate_fps >
2591 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07002592 }
2593
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002594 EXPECT_THAT(video_source_.sink_wants(), FpsMaxResolutionMax());
Jonathan Yubc771b72017-12-08 17:04:29 -08002595 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-13 23:25:22 -07002596 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002597 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2598 EXPECT_EQ((loop_count - 1) * 2,
2599 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07002600
mflodmancc3d4422017-08-03 08:27:51 -07002601 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07002602}
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002603
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002604TEST_F(VideoStreamEncoderTest,
2605 SinkWantsNotChangedByResourceLimitedBeforeDegradationPreferenceChange) {
Asa Persson606d3cb2021-10-04 10:07:11 +02002606 video_stream_encoder_->OnBitrateUpdated(kTargetBitrate, kTargetBitrate,
2607 kTargetBitrate, 0, 0, 0);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002608 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002609
2610 const int kFrameWidth = 1280;
2611 const int kFrameHeight = 720;
2612
2613 int64_t ntp_time = kFrameIntervalMs;
2614
2615 // Force an input frame rate to be available, or the adaptation call won't
2616 // know what framerate to adapt form.
2617 const int kInputFps = 30;
2618 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2619 stats.input_frame_rate = kInputFps;
2620 stats_proxy_->SetMockStats(stats);
2621
2622 video_source_.set_adaptation_enabled(true);
2623 video_stream_encoder_->SetSource(
2624 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002625 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002626 video_source_.IncomingCapturedFrame(
2627 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2628 sink_.WaitForEncodedFrame(ntp_time);
2629 ntp_time += kFrameIntervalMs;
2630
2631 // Trigger CPU overuse.
2632 video_stream_encoder_->TriggerCpuOveruse();
2633 video_source_.IncomingCapturedFrame(
2634 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2635 sink_.WaitForEncodedFrame(ntp_time);
2636 ntp_time += kFrameIntervalMs;
2637
2638 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2639 EXPECT_EQ(std::numeric_limits<int>::max(),
2640 video_source_.sink_wants().max_pixel_count);
2641 // Some framerate constraint should be set.
2642 int restricted_fps = video_source_.sink_wants().max_framerate_fps;
2643 EXPECT_LT(restricted_fps, kInputFps);
2644 video_source_.IncomingCapturedFrame(
2645 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2646 sink_.WaitForEncodedFrame(ntp_time);
2647 ntp_time += 100;
2648
Henrik Boström2671dac2020-05-19 16:29:09 +02002649 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002650 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
2651 // Give the encoder queue time to process the change in degradation preference
2652 // by waiting for an encoded frame.
2653 video_source_.IncomingCapturedFrame(
2654 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2655 sink_.WaitForEncodedFrame(ntp_time);
2656 ntp_time += kFrameIntervalMs;
2657
2658 video_stream_encoder_->TriggerQualityLow();
2659 video_source_.IncomingCapturedFrame(
2660 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2661 sink_.WaitForEncodedFrame(ntp_time);
2662 ntp_time += kFrameIntervalMs;
2663
2664 // Some resolution constraint should be set.
2665 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2666 EXPECT_LT(video_source_.sink_wants().max_pixel_count,
2667 kFrameWidth * kFrameHeight);
2668 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, kInputFps);
2669
2670 int pixel_count = video_source_.sink_wants().max_pixel_count;
2671 // Triggering a CPU underuse should not change the sink wants since it has
2672 // not been overused for resolution since we changed degradation preference.
2673 video_stream_encoder_->TriggerCpuUnderuse();
2674 video_source_.IncomingCapturedFrame(
2675 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2676 sink_.WaitForEncodedFrame(ntp_time);
2677 ntp_time += kFrameIntervalMs;
2678 EXPECT_EQ(video_source_.sink_wants().max_pixel_count, pixel_count);
2679 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, kInputFps);
2680
Evan Shrubsole64469032020-06-11 10:45:29 +02002681 // Change the degradation preference back. CPU underuse should not adapt since
2682 // QP is most limited.
Henrik Boström2671dac2020-05-19 16:29:09 +02002683 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002684 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
2685 video_source_.IncomingCapturedFrame(
2686 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2687 sink_.WaitForEncodedFrame(ntp_time);
2688 ntp_time += 100;
2689 // Resolution adaptations is gone after changing degradation preference.
2690 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2691 EXPECT_EQ(std::numeric_limits<int>::max(),
2692 video_source_.sink_wants().max_pixel_count);
2693 // The fps adaptation from above is now back.
2694 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, restricted_fps);
2695
2696 // Trigger CPU underuse.
2697 video_stream_encoder_->TriggerCpuUnderuse();
2698 video_source_.IncomingCapturedFrame(
2699 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2700 sink_.WaitForEncodedFrame(ntp_time);
2701 ntp_time += kFrameIntervalMs;
Evan Shrubsole64469032020-06-11 10:45:29 +02002702 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, restricted_fps);
2703
2704 // Trigger QP underuse, fps should return to normal.
2705 video_stream_encoder_->TriggerQualityHigh();
2706 video_source_.IncomingCapturedFrame(
2707 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2708 sink_.WaitForEncodedFrame(ntp_time);
2709 ntp_time += kFrameIntervalMs;
2710 EXPECT_THAT(video_source_.sink_wants(), FpsMax());
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002711
2712 video_stream_encoder_->Stop();
2713}
2714
mflodmancc3d4422017-08-03 08:27:51 -07002715TEST_F(VideoStreamEncoderTest, SinkWantsStoredByDegradationPreference) {
Henrik Boström381d1092020-05-12 18:49:07 +02002716 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002717 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002718 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
perkj803d97f2016-11-01 11:45:46 -07002719
sprangc5d62e22017-04-02 23:53:04 -07002720 const int kFrameWidth = 1280;
2721 const int kFrameHeight = 720;
sprangc5d62e22017-04-02 23:53:04 -07002722
Åsa Persson8c1bf952018-09-13 10:42:19 +02002723 int64_t frame_timestamp = 1;
perkj803d97f2016-11-01 11:45:46 -07002724
kthelgason5e13d412016-12-01 03:59:51 -08002725 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002726 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002727 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002728 frame_timestamp += kFrameIntervalMs;
2729
perkj803d97f2016-11-01 11:45:46 -07002730 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002731 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07002732 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002733 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002734 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002735 frame_timestamp += kFrameIntervalMs;
sprang3ea3c772017-03-30 07:23:48 -07002736
asapersson0944a802017-04-07 00:57:58 -07002737 // Default degradation preference is maintain-framerate, so will lower max
sprangc5d62e22017-04-02 23:53:04 -07002738 // wanted resolution.
2739 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2740 EXPECT_LT(video_source_.sink_wants().max_pixel_count,
2741 kFrameWidth * kFrameHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002742 EXPECT_EQ(kDefaultFramerate, video_source_.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07002743
2744 // Set new source, switch to maintain-resolution.
perkj803d97f2016-11-01 11:45:46 -07002745 test::FrameForwarder new_video_source;
Henrik Boström381d1092020-05-12 18:49:07 +02002746 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002747 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01002748 // Give the encoder queue time to process the change in degradation preference
2749 // by waiting for an encoded frame.
2750 new_video_source.IncomingCapturedFrame(
2751 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2752 sink_.WaitForEncodedFrame(frame_timestamp);
2753 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002754 // Initially no degradation registered.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002755 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
perkj803d97f2016-11-01 11:45:46 -07002756
sprangc5d62e22017-04-02 23:53:04 -07002757 // Force an input frame rate to be available, or the adaptation call won't
2758 // know what framerate to adapt form.
asapersson02465b82017-04-10 01:12:52 -07002759 const int kInputFps = 30;
sprangc5d62e22017-04-02 23:53:04 -07002760 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson02465b82017-04-10 01:12:52 -07002761 stats.input_frame_rate = kInputFps;
sprangc5d62e22017-04-02 23:53:04 -07002762 stats_proxy_->SetMockStats(stats);
2763
mflodmancc3d4422017-08-03 08:27:51 -07002764 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07002765 new_video_source.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002766 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002767 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002768 frame_timestamp += kFrameIntervalMs;
2769
2770 // Some framerate constraint should be set.
sprang84a37592017-02-10 07:04:27 -08002771 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
sprangc5d62e22017-04-02 23:53:04 -07002772 EXPECT_EQ(std::numeric_limits<int>::max(),
2773 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 01:12:52 -07002774 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
sprangc5d62e22017-04-02 23:53:04 -07002775
asapersson02465b82017-04-10 01:12:52 -07002776 // Turn off degradation completely.
Henrik Boström381d1092020-05-12 18:49:07 +02002777 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
2778 &new_video_source, webrtc::DegradationPreference::DISABLED);
Henrik Boström07b17df2020-01-15 11:42:12 +01002779 // Give the encoder queue time to process the change in degradation preference
2780 // by waiting for an encoded frame.
2781 new_video_source.IncomingCapturedFrame(
2782 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2783 sink_.WaitForEncodedFrame(frame_timestamp);
2784 frame_timestamp += kFrameIntervalMs;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002785 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
sprangc5d62e22017-04-02 23:53:04 -07002786
mflodmancc3d4422017-08-03 08:27:51 -07002787 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07002788 new_video_source.IncomingCapturedFrame(
2789 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002790 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002791 frame_timestamp += kFrameIntervalMs;
2792
2793 // Still no degradation.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002794 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
perkj803d97f2016-11-01 11:45:46 -07002795
2796 // Calling SetSource with resolution scaling enabled apply the old SinkWants.
Henrik Boström381d1092020-05-12 18:49:07 +02002797 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002798 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Henrik Boström07b17df2020-01-15 11:42:12 +01002799 // Give the encoder queue time to process the change in degradation preference
2800 // by waiting for an encoded frame.
2801 new_video_source.IncomingCapturedFrame(
2802 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2803 sink_.WaitForEncodedFrame(frame_timestamp);
2804 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002805 EXPECT_LT(new_video_source.sink_wants().max_pixel_count,
2806 kFrameWidth * kFrameHeight);
sprang84a37592017-02-10 07:04:27 -08002807 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002808 EXPECT_EQ(kDefaultFramerate, new_video_source.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07002809
2810 // Calling SetSource with framerate scaling enabled apply the old SinkWants.
Henrik Boström381d1092020-05-12 18:49:07 +02002811 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002812 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01002813 // Give the encoder queue time to process the change in degradation preference
2814 // by waiting for an encoded frame.
2815 new_video_source.IncomingCapturedFrame(
2816 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2817 sink_.WaitForEncodedFrame(frame_timestamp);
2818 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002819 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
2820 EXPECT_EQ(std::numeric_limits<int>::max(),
2821 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 01:12:52 -07002822 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
perkj803d97f2016-11-01 11:45:46 -07002823
mflodmancc3d4422017-08-03 08:27:51 -07002824 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002825}
2826
mflodmancc3d4422017-08-03 08:27:51 -07002827TEST_F(VideoStreamEncoderTest, StatsTracksQualityAdaptationStats) {
Henrik Boström381d1092020-05-12 18:49:07 +02002828 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002829 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkj803d97f2016-11-01 11:45:46 -07002830
asaperssonfab67072017-04-04 05:51:49 -07002831 const int kWidth = 1280;
2832 const int kHeight = 720;
asaperssonfab67072017-04-04 05:51:49 -07002833 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002834 WaitForEncodedFrame(1);
asaperssonfab67072017-04-04 05:51:49 -07002835 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2836 EXPECT_FALSE(stats.bw_limited_resolution);
2837 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
2838
2839 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002840 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07002841 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002842 WaitForEncodedFrame(2);
asaperssonfab67072017-04-04 05:51:49 -07002843
2844 stats = stats_proxy_->GetStats();
2845 EXPECT_TRUE(stats.bw_limited_resolution);
2846 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
2847
2848 // Trigger adapt up.
mflodmancc3d4422017-08-03 08:27:51 -07002849 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 05:51:49 -07002850 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002851 WaitForEncodedFrame(3);
asaperssonfab67072017-04-04 05:51:49 -07002852
2853 stats = stats_proxy_->GetStats();
2854 EXPECT_FALSE(stats.bw_limited_resolution);
2855 EXPECT_EQ(2, stats.number_of_quality_adapt_changes);
2856 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2857
mflodmancc3d4422017-08-03 08:27:51 -07002858 video_stream_encoder_->Stop();
asaperssonfab67072017-04-04 05:51:49 -07002859}
2860
mflodmancc3d4422017-08-03 08:27:51 -07002861TEST_F(VideoStreamEncoderTest, StatsTracksCpuAdaptationStats) {
Henrik Boström381d1092020-05-12 18:49:07 +02002862 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002863 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07002864
2865 const int kWidth = 1280;
2866 const int kHeight = 720;
asaperssonfab67072017-04-04 05:51:49 -07002867 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002868 WaitForEncodedFrame(1);
perkj803d97f2016-11-01 11:45:46 -07002869 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2870 EXPECT_FALSE(stats.cpu_limited_resolution);
2871 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2872
2873 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002874 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07002875 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002876 WaitForEncodedFrame(2);
perkj803d97f2016-11-01 11:45:46 -07002877
2878 stats = stats_proxy_->GetStats();
2879 EXPECT_TRUE(stats.cpu_limited_resolution);
2880 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2881
2882 // Trigger CPU normal use.
Henrik Boström91aa7322020-04-28 12:24:33 +02002883 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07002884 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002885 WaitForEncodedFrame(3);
perkj803d97f2016-11-01 11:45:46 -07002886
2887 stats = stats_proxy_->GetStats();
2888 EXPECT_FALSE(stats.cpu_limited_resolution);
2889 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asaperssonfab67072017-04-04 05:51:49 -07002890 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07002891
mflodmancc3d4422017-08-03 08:27:51 -07002892 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002893}
2894
mflodmancc3d4422017-08-03 08:27:51 -07002895TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsCpuAdaptation) {
Henrik Boström381d1092020-05-12 18:49:07 +02002896 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002897 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08002898
asaperssonfab67072017-04-04 05:51:49 -07002899 const int kWidth = 1280;
2900 const int kHeight = 720;
2901 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002902 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08002903 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002904 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002905 EXPECT_FALSE(stats.cpu_limited_resolution);
2906 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2907
asaperssonfab67072017-04-04 05:51:49 -07002908 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002909 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07002910 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002911 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 01:44:11 -08002912 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002913 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002914 EXPECT_TRUE(stats.cpu_limited_resolution);
2915 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2916
2917 // Set new source with adaptation still enabled.
2918 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002919 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002920 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 01:44:11 -08002921
asaperssonfab67072017-04-04 05:51:49 -07002922 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002923 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08002924 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002925 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002926 EXPECT_TRUE(stats.cpu_limited_resolution);
2927 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2928
2929 // Set adaptation disabled.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002930 video_stream_encoder_->SetSource(&new_video_source,
2931 webrtc::DegradationPreference::DISABLED);
kthelgason876222f2016-11-29 01:44:11 -08002932
asaperssonfab67072017-04-04 05:51:49 -07002933 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002934 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08002935 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002936 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002937 EXPECT_FALSE(stats.cpu_limited_resolution);
2938 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2939
2940 // Set adaptation back to enabled.
mflodmancc3d4422017-08-03 08:27:51 -07002941 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002942 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 01:44:11 -08002943
asaperssonfab67072017-04-04 05:51:49 -07002944 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002945 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 01:44:11 -08002946 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002947 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002948 EXPECT_TRUE(stats.cpu_limited_resolution);
2949 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2950
asaperssonfab67072017-04-04 05:51:49 -07002951 // Trigger CPU normal use.
Henrik Boström91aa7322020-04-28 12:24:33 +02002952 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07002953 new_video_source.IncomingCapturedFrame(CreateFrame(6, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002954 WaitForEncodedFrame(6);
kthelgason876222f2016-11-29 01:44:11 -08002955 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002956 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002957 EXPECT_FALSE(stats.cpu_limited_resolution);
2958 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 01:12:52 -07002959 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002960
mflodmancc3d4422017-08-03 08:27:51 -07002961 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08002962}
2963
mflodmancc3d4422017-08-03 08:27:51 -07002964TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsQualityAdaptation) {
Henrik Boström381d1092020-05-12 18:49:07 +02002965 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002966 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08002967
asaperssonfab67072017-04-04 05:51:49 -07002968 const int kWidth = 1280;
2969 const int kHeight = 720;
2970 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002971 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08002972 VideoSendStream::Stats stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002973 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002974 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002975 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002976
2977 // Set new source with adaptation still enabled.
2978 test::FrameForwarder new_video_source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002979 video_stream_encoder_->SetSource(&new_video_source,
2980 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 01:44:11 -08002981
asaperssonfab67072017-04-04 05:51:49 -07002982 new_video_source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002983 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 01:44:11 -08002984 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002985 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002986 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002987 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002988
asaperssonfab67072017-04-04 05:51:49 -07002989 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002990 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07002991 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002992 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08002993 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002994 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002995 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002996 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002997
asaperssonfab67072017-04-04 05:51:49 -07002998 // Set new source with adaptation still enabled.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002999 video_stream_encoder_->SetSource(&new_video_source,
3000 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 01:44:11 -08003001
asaperssonfab67072017-04-04 05:51:49 -07003002 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003003 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08003004 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08003005 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003006 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07003007 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08003008
asapersson02465b82017-04-10 01:12:52 -07003009 // Disable resolution scaling.
mflodmancc3d4422017-08-03 08:27:51 -07003010 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003011 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 01:44:11 -08003012
asaperssonfab67072017-04-04 05:51:49 -07003013 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003014 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 01:44:11 -08003015 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08003016 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003017 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07003018 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
3019 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08003020
mflodmancc3d4422017-08-03 08:27:51 -07003021 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08003022}
3023
mflodmancc3d4422017-08-03 08:27:51 -07003024TEST_F(VideoStreamEncoderTest,
3025 QualityAdaptationStatsAreResetWhenScalerIsDisabled) {
Henrik Boström381d1092020-05-12 18:49:07 +02003026 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003027 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson36e9eb42017-03-31 05:29:12 -07003028
3029 const int kWidth = 1280;
3030 const int kHeight = 720;
Åsa Persson8c1bf952018-09-13 10:42:19 +02003031 int64_t timestamp_ms = kFrameIntervalMs;
asapersson36e9eb42017-03-31 05:29:12 -07003032 video_source_.set_adaptation_enabled(true);
Åsa Persson8c1bf952018-09-13 10:42:19 +02003033 video_source_.IncomingCapturedFrame(
3034 CreateFrame(timestamp_ms, kWidth, kHeight));
3035 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07003036 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3037 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3038 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3039
3040 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07003041 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003042 timestamp_ms += kFrameIntervalMs;
3043 video_source_.IncomingCapturedFrame(
3044 CreateFrame(timestamp_ms, kWidth, kHeight));
3045 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07003046 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3047 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3048 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3049
3050 // Trigger overuse.
mflodmancc3d4422017-08-03 08:27:51 -07003051 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003052 timestamp_ms += kFrameIntervalMs;
3053 video_source_.IncomingCapturedFrame(
3054 CreateFrame(timestamp_ms, kWidth, kHeight));
3055 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07003056 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3057 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3058 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3059
Niels Möller4db138e2018-04-19 09:04:13 +02003060 // Leave source unchanged, but disable quality scaler.
asapersson36e9eb42017-03-31 05:29:12 -07003061 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 09:04:13 +02003062
3063 VideoEncoderConfig video_encoder_config;
3064 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
3065 // Make format different, to force recreation of encoder.
3066 video_encoder_config.video_format.parameters["foo"] = "foo";
3067 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02003068 kMaxPayloadLength);
Åsa Persson8c1bf952018-09-13 10:42:19 +02003069 timestamp_ms += kFrameIntervalMs;
3070 video_source_.IncomingCapturedFrame(
3071 CreateFrame(timestamp_ms, kWidth, kHeight));
3072 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07003073 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3074 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3075 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3076
mflodmancc3d4422017-08-03 08:27:51 -07003077 video_stream_encoder_->Stop();
asapersson36e9eb42017-03-31 05:29:12 -07003078}
3079
mflodmancc3d4422017-08-03 08:27:51 -07003080TEST_F(VideoStreamEncoderTest,
Evan Shrubsole33be9df2020-03-05 18:39:32 +01003081 StatsTracksCpuAdaptationStatsWhenSwitchingSource_Balanced) {
Henrik Boström381d1092020-05-12 18:49:07 +02003082 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003083 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsole33be9df2020-03-05 18:39:32 +01003084
3085 const int kWidth = 1280;
3086 const int kHeight = 720;
3087 int sequence = 1;
3088
3089 // Enable BALANCED preference, no initial limitation.
3090 test::FrameForwarder source;
3091 video_stream_encoder_->SetSource(&source,
3092 webrtc::DegradationPreference::BALANCED);
3093 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3094 WaitForEncodedFrame(sequence++);
3095 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3096 EXPECT_FALSE(stats.cpu_limited_resolution);
3097 EXPECT_FALSE(stats.cpu_limited_framerate);
3098 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
3099
3100 // Trigger CPU overuse, should now adapt down.
3101 video_stream_encoder_->TriggerCpuOveruse();
3102 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3103 WaitForEncodedFrame(sequence++);
3104 stats = stats_proxy_->GetStats();
3105 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3106
3107 // Set new degradation preference should clear restrictions since we changed
3108 // from BALANCED.
Evan Shrubsolede2049e2020-05-25 16:54:21 +02003109 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole33be9df2020-03-05 18:39:32 +01003110 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
3111 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3112 WaitForEncodedFrame(sequence++);
3113 stats = stats_proxy_->GetStats();
3114 EXPECT_FALSE(stats.cpu_limited_resolution);
3115 EXPECT_FALSE(stats.cpu_limited_framerate);
3116 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3117
3118 // Force an input frame rate to be available, or the adaptation call won't
3119 // know what framerate to adapt from.
3120 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
3121 mock_stats.input_frame_rate = 30;
3122 stats_proxy_->SetMockStats(mock_stats);
3123 video_stream_encoder_->TriggerCpuOveruse();
3124 stats_proxy_->ResetMockStats();
3125 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3126 WaitForEncodedFrame(sequence++);
3127
3128 // We have now adapted once.
3129 stats = stats_proxy_->GetStats();
3130 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3131
3132 // Back to BALANCED, should clear the restrictions again.
Evan Shrubsolede2049e2020-05-25 16:54:21 +02003133 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
3134 &source, webrtc::DegradationPreference::BALANCED);
Evan Shrubsole33be9df2020-03-05 18:39:32 +01003135 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3136 WaitForEncodedFrame(sequence++);
3137 stats = stats_proxy_->GetStats();
3138 EXPECT_FALSE(stats.cpu_limited_resolution);
3139 EXPECT_FALSE(stats.cpu_limited_framerate);
3140 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3141
3142 video_stream_encoder_->Stop();
3143}
3144
3145TEST_F(VideoStreamEncoderTest,
mflodmancc3d4422017-08-03 08:27:51 -07003146 StatsTracksCpuAdaptationStatsWhenSwitchingSource) {
Henrik Boström381d1092020-05-12 18:49:07 +02003147 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003148 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkj803d97f2016-11-01 11:45:46 -07003149
asapersson0944a802017-04-07 00:57:58 -07003150 const int kWidth = 1280;
3151 const int kHeight = 720;
sprang84a37592017-02-10 07:04:27 -08003152 int sequence = 1;
perkj803d97f2016-11-01 11:45:46 -07003153
asaperssonfab67072017-04-04 05:51:49 -07003154 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003155 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003156 VideoSendStream::Stats stats = stats_proxy_->GetStats();
sprang84a37592017-02-10 07:04:27 -08003157 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003158 EXPECT_FALSE(stats.cpu_limited_framerate);
sprang84a37592017-02-10 07:04:27 -08003159 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
3160
asapersson02465b82017-04-10 01:12:52 -07003161 // Trigger CPU overuse, should now adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07003162 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07003163 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003164 WaitForEncodedFrame(sequence++);
sprang84a37592017-02-10 07:04:27 -08003165 stats = stats_proxy_->GetStats();
perkj803d97f2016-11-01 11:45:46 -07003166 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003167 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07003168 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3169
3170 // Set new source with adaptation still enabled.
3171 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07003172 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003173 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 11:45:46 -07003174
3175 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003176 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003177 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003178 stats = stats_proxy_->GetStats();
3179 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson09f05612017-05-15 23:40:18 -07003180 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07003181 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3182
sprangc5d62e22017-04-02 23:53:04 -07003183 // Set cpu adaptation by frame dropping.
mflodmancc3d4422017-08-03 08:27:51 -07003184 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003185 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
perkj803d97f2016-11-01 11:45:46 -07003186 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003187 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003188 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003189 stats = stats_proxy_->GetStats();
sprangc5d62e22017-04-02 23:53:04 -07003190 // Not adapted at first.
perkj803d97f2016-11-01 11:45:46 -07003191 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson09f05612017-05-15 23:40:18 -07003192 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07003193 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3194
sprangc5d62e22017-04-02 23:53:04 -07003195 // Force an input frame rate to be available, or the adaptation call won't
asapersson09f05612017-05-15 23:40:18 -07003196 // know what framerate to adapt from.
sprangc5d62e22017-04-02 23:53:04 -07003197 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
3198 mock_stats.input_frame_rate = 30;
3199 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 08:27:51 -07003200 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07003201 stats_proxy_->ResetMockStats();
3202
3203 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003204 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003205 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003206
3207 // Framerate now adapted.
3208 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07003209 EXPECT_FALSE(stats.cpu_limited_resolution);
3210 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003211 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3212
3213 // Disable CPU adaptation.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003214 video_stream_encoder_->SetSource(&new_video_source,
3215 webrtc::DegradationPreference::DISABLED);
sprangc5d62e22017-04-02 23:53:04 -07003216 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003217 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003218 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003219
3220 stats = stats_proxy_->GetStats();
3221 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003222 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003223 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3224
3225 // Try to trigger overuse. Should not succeed.
3226 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 08:27:51 -07003227 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07003228 stats_proxy_->ResetMockStats();
3229
3230 stats = stats_proxy_->GetStats();
3231 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003232 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003233 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3234
3235 // Switch back the source with resolution adaptation enabled.
mflodmancc3d4422017-08-03 08:27:51 -07003236 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003237 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssonfab67072017-04-04 05:51:49 -07003238 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003239 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003240 stats = stats_proxy_->GetStats();
3241 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003242 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003243 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07003244
3245 // Trigger CPU normal usage.
Henrik Boström91aa7322020-04-28 12:24:33 +02003246 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07003247 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003248 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003249 stats = stats_proxy_->GetStats();
3250 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003251 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003252 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
3253
3254 // Back to the source with adaptation off, set it back to maintain-resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003255 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003256 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07003257 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003258 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003259 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003260 stats = stats_proxy_->GetStats();
asapersson13874762017-06-07 00:01:02 -07003261 // Disabled, since we previously switched the source to disabled.
sprangc5d62e22017-04-02 23:53:04 -07003262 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003263 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003264 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
3265
3266 // Trigger CPU normal usage.
Henrik Boström91aa7322020-04-28 12:24:33 +02003267 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07003268 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003269 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003270 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003271 stats = stats_proxy_->GetStats();
3272 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003273 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003274 EXPECT_EQ(4, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 01:12:52 -07003275 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07003276
mflodmancc3d4422017-08-03 08:27:51 -07003277 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07003278}
3279
mflodmancc3d4422017-08-03 08:27:51 -07003280TEST_F(VideoStreamEncoderTest,
3281 ScalingUpAndDownDoesNothingWithMaintainResolution) {
asaperssonfab67072017-04-04 05:51:49 -07003282 const int kWidth = 1280;
3283 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003284 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003285 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08003286
asaperssonfab67072017-04-04 05:51:49 -07003287 // Expect no scaling to begin with.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003288 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
kthelgason876222f2016-11-29 01:44:11 -08003289
asaperssonfab67072017-04-04 05:51:49 -07003290 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003291 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08003292
asaperssonfab67072017-04-04 05:51:49 -07003293 // Trigger scale down.
mflodmancc3d4422017-08-03 08:27:51 -07003294 video_stream_encoder_->TriggerQualityLow();
kthelgason5e13d412016-12-01 03:59:51 -08003295
asaperssonfab67072017-04-04 05:51:49 -07003296 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003297 WaitForEncodedFrame(2);
kthelgason5e13d412016-12-01 03:59:51 -08003298
kthelgason876222f2016-11-29 01:44:11 -08003299 // Expect a scale down.
3300 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count);
asaperssonfab67072017-04-04 05:51:49 -07003301 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason876222f2016-11-29 01:44:11 -08003302
asapersson02465b82017-04-10 01:12:52 -07003303 // Set resolution scaling disabled.
kthelgason876222f2016-11-29 01:44:11 -08003304 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07003305 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003306 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 01:44:11 -08003307
asaperssonfab67072017-04-04 05:51:49 -07003308 // Trigger scale down.
mflodmancc3d4422017-08-03 08:27:51 -07003309 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07003310 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003311 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08003312
asaperssonfab67072017-04-04 05:51:49 -07003313 // Expect no scaling.
sprangc5d62e22017-04-02 23:53:04 -07003314 EXPECT_EQ(std::numeric_limits<int>::max(),
3315 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 01:44:11 -08003316
asaperssonfab67072017-04-04 05:51:49 -07003317 // Trigger scale up.
mflodmancc3d4422017-08-03 08:27:51 -07003318 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 05:51:49 -07003319 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003320 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08003321
asapersson02465b82017-04-10 01:12:52 -07003322 // Expect nothing to change, still no scaling.
sprangc5d62e22017-04-02 23:53:04 -07003323 EXPECT_EQ(std::numeric_limits<int>::max(),
3324 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 01:44:11 -08003325
mflodmancc3d4422017-08-03 08:27:51 -07003326 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08003327}
3328
mflodmancc3d4422017-08-03 08:27:51 -07003329TEST_F(VideoStreamEncoderTest,
3330 SkipsSameAdaptDownRequest_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07003331 const int kWidth = 1280;
3332 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003333 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003334 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003335
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003336 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07003337 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07003338 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003339 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07003340
3341 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003342 WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003343 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003344 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3345 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3346
3347 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003348 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003349 EXPECT_THAT(source.sink_wants(),
3350 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson02465b82017-04-10 01:12:52 -07003351 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
3352 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3353 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3354
3355 // Trigger adapt down for same input resolution, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003356 video_stream_encoder_->TriggerCpuOveruse();
asapersson02465b82017-04-10 01:12:52 -07003357 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
3358 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3359 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3360
mflodmancc3d4422017-08-03 08:27:51 -07003361 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003362}
3363
mflodmancc3d4422017-08-03 08:27:51 -07003364TEST_F(VideoStreamEncoderTest, SkipsSameOrLargerAdaptDownRequest_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07003365 const int kWidth = 1280;
3366 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003367 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003368 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003369
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003370 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07003371 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003372 video_stream_encoder_->SetSource(&source,
3373 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003374 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3375 sink_.WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003376 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003377
3378 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003379 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003380 EXPECT_THAT(source.sink_wants(),
3381 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07003382 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3383 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3384 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
3385
3386 // Trigger adapt down for same input resolution, expect no change.
3387 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
3388 sink_.WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07003389 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003390 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
3391 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3392 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3393
3394 // Trigger adapt down for larger input resolution, expect no change.
3395 source.IncomingCapturedFrame(CreateFrame(3, kWidth + 1, kHeight + 1));
3396 sink_.WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 08:27:51 -07003397 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003398 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
3399 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3400 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3401
mflodmancc3d4422017-08-03 08:27:51 -07003402 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003403}
3404
mflodmancc3d4422017-08-03 08:27:51 -07003405TEST_F(VideoStreamEncoderTest,
Åsa Perssonf5f7e8e2021-06-09 22:55:38 +02003406 FpsCountReturnsToZeroForFewerAdaptationsUpThanDown) {
3407 const int kWidth = 640;
3408 const int kHeight = 360;
3409 const int64_t kFrameIntervalMs = 150;
3410 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003411 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Åsa Perssonf5f7e8e2021-06-09 22:55:38 +02003412
3413 // Enable BALANCED preference, no initial limitation.
3414 AdaptingFrameForwarder source(&time_controller_);
3415 source.set_adaptation_enabled(true);
3416 video_stream_encoder_->SetSource(&source,
3417 webrtc::DegradationPreference::BALANCED);
3418
3419 int64_t timestamp_ms = kFrameIntervalMs;
3420 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3421 sink_.WaitForEncodedFrame(kWidth, kHeight);
3422 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
3423 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3424 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3425 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3426
3427 // Trigger adapt down, expect reduced fps (640x360@15fps).
3428 video_stream_encoder_->TriggerQualityLow();
3429 timestamp_ms += kFrameIntervalMs;
3430 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3431 sink_.WaitForEncodedFrame(timestamp_ms);
3432 EXPECT_THAT(source.sink_wants(),
3433 FpsMatchesResolutionMax(Lt(kDefaultFramerate)));
3434 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3435 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3436 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3437
3438 // Source requests 270p, expect reduced resolution (480x270@15fps).
3439 source.OnOutputFormatRequest(480, 270);
3440 timestamp_ms += kFrameIntervalMs;
3441 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3442 WaitForEncodedFrame(480, 270);
3443 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3444
3445 // Trigger adapt down, expect reduced fps (480x270@10fps).
3446 video_stream_encoder_->TriggerQualityLow();
3447 timestamp_ms += kFrameIntervalMs;
3448 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3449 sink_.WaitForEncodedFrame(timestamp_ms);
3450 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
3451 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3452 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3453 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3454
3455 // Source requests QVGA, expect reduced resolution (320x180@10fps).
3456 source.OnOutputFormatRequest(320, 180);
3457 timestamp_ms += kFrameIntervalMs;
3458 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3459 WaitForEncodedFrame(320, 180);
3460 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3461
3462 // Trigger adapt down, expect reduced fps (320x180@7fps).
3463 video_stream_encoder_->TriggerQualityLow();
3464 timestamp_ms += kFrameIntervalMs;
3465 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3466 sink_.WaitForEncodedFrame(timestamp_ms);
3467 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
3468 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3469 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3470 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3471
3472 // Source requests VGA, expect increased resolution (640x360@7fps).
3473 source.OnOutputFormatRequest(640, 360);
3474 timestamp_ms += kFrameIntervalMs;
3475 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3476 WaitForEncodedFrame(timestamp_ms);
3477 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3478
3479 // Trigger adapt up, expect increased fps (640x360@(max-2)fps).
3480 video_stream_encoder_->TriggerQualityHigh();
3481 timestamp_ms += kFrameIntervalMs;
3482 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3483 WaitForEncodedFrame(timestamp_ms);
3484 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3485 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3486 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3487 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3488
3489 // Trigger adapt up, expect increased fps (640x360@(max-1)fps).
3490 video_stream_encoder_->TriggerQualityHigh();
3491 timestamp_ms += kFrameIntervalMs;
3492 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3493 WaitForEncodedFrame(timestamp_ms);
3494 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3495 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3496 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3497 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3498
3499 // Trigger adapt up, expect increased fps (640x360@maxfps).
3500 video_stream_encoder_->TriggerQualityHigh();
3501 timestamp_ms += kFrameIntervalMs;
3502 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3503 WaitForEncodedFrame(timestamp_ms);
3504 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3505 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3506 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3507 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3508
3509 video_stream_encoder_->Stop();
3510}
3511
3512TEST_F(VideoStreamEncoderTest,
3513 FpsCountReturnsToZeroForFewerAdaptationsUpThanDownWithTwoResources) {
3514 const int kWidth = 1280;
3515 const int kHeight = 720;
3516 const int64_t kFrameIntervalMs = 150;
3517 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003518 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Åsa Perssonf5f7e8e2021-06-09 22:55:38 +02003519
3520 // Enable BALANCED preference, no initial limitation.
3521 AdaptingFrameForwarder source(&time_controller_);
3522 source.set_adaptation_enabled(true);
3523 video_stream_encoder_->SetSource(&source,
3524 webrtc::DegradationPreference::BALANCED);
3525
3526 int64_t timestamp_ms = kFrameIntervalMs;
3527 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3528 sink_.WaitForEncodedFrame(kWidth, kHeight);
3529 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
3530 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3531 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3532 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3533
3534 // Trigger adapt down, expect scaled down resolution (960x540@maxfps).
3535 video_stream_encoder_->TriggerQualityLow();
3536 timestamp_ms += kFrameIntervalMs;
3537 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3538 sink_.WaitForEncodedFrame(timestamp_ms);
3539 EXPECT_THAT(source.sink_wants(),
3540 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
3541 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3542 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3543 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3544
3545 // Trigger adapt down, expect scaled down resolution (640x360@maxfps).
3546 video_stream_encoder_->TriggerQualityLow();
3547 timestamp_ms += kFrameIntervalMs;
3548 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3549 sink_.WaitForEncodedFrame(timestamp_ms);
3550 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
3551 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3552 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3553 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3554
3555 // Trigger adapt down, expect reduced fps (640x360@15fps).
3556 video_stream_encoder_->TriggerQualityLow();
3557 timestamp_ms += kFrameIntervalMs;
3558 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3559 WaitForEncodedFrame(timestamp_ms);
3560 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
3561 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3562 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3563 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3564
3565 // Source requests QVGA, expect reduced resolution (320x180@15fps).
3566 source.OnOutputFormatRequest(320, 180);
3567 timestamp_ms += kFrameIntervalMs;
3568 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3569 WaitForEncodedFrame(320, 180);
3570 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3571 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3572
3573 // Trigger adapt down, expect reduced fps (320x180@7fps).
3574 video_stream_encoder_->TriggerCpuOveruse();
3575 timestamp_ms += kFrameIntervalMs;
3576 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3577 WaitForEncodedFrame(timestamp_ms);
3578 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
3579 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3580 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3581 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3582 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
3583 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3584 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3585
3586 // Source requests HD, expect increased resolution (640x360@7fps).
3587 source.OnOutputFormatRequest(1280, 720);
3588 timestamp_ms += kFrameIntervalMs;
3589 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3590 WaitForEncodedFrame(timestamp_ms);
3591 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3592 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3593
3594 // Trigger adapt up, expect increased fps (640x360@(max-1)fps).
3595 video_stream_encoder_->TriggerCpuUnderuse();
3596 timestamp_ms += kFrameIntervalMs;
3597 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3598 WaitForEncodedFrame(timestamp_ms);
3599 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3600 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3601 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3602 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3603 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
3604 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3605 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3606
3607 // Trigger adapt up, expect increased fps (640x360@maxfps).
3608 video_stream_encoder_->TriggerQualityHigh();
3609 video_stream_encoder_->TriggerCpuUnderuse();
3610 timestamp_ms += kFrameIntervalMs;
3611 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3612 WaitForEncodedFrame(timestamp_ms);
3613 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3614 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3615 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3616 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3617 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3618 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3619 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3620
3621 // Trigger adapt up, expect increased resolution (960x570@maxfps).
3622 video_stream_encoder_->TriggerQualityHigh();
3623 video_stream_encoder_->TriggerCpuUnderuse();
3624 timestamp_ms += kFrameIntervalMs;
3625 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3626 WaitForEncodedFrame(timestamp_ms);
3627 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
3628 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3629 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3630 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3631 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3632 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3633 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3634
3635 // Trigger adapt up, expect increased resolution (1280x720@maxfps).
3636 video_stream_encoder_->TriggerQualityHigh();
3637 video_stream_encoder_->TriggerCpuUnderuse();
3638 timestamp_ms += kFrameIntervalMs;
3639 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3640 WaitForEncodedFrame(timestamp_ms);
3641 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
3642 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3643 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3644 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3645 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3646 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3647 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3648
3649 video_stream_encoder_->Stop();
3650}
3651
3652TEST_F(VideoStreamEncoderTest,
mflodmancc3d4422017-08-03 08:27:51 -07003653 NoChangeForInitialNormalUsage_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07003654 const int kWidth = 1280;
3655 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003656 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003657 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003658
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003659 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07003660 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07003661 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003662 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07003663
3664 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003665 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003666 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003667 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3668 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3669
3670 // Trigger adapt up, expect no change.
Henrik Boström91aa7322020-04-28 12:24:33 +02003671 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003672 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003673 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3674 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3675
mflodmancc3d4422017-08-03 08:27:51 -07003676 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003677}
3678
mflodmancc3d4422017-08-03 08:27:51 -07003679TEST_F(VideoStreamEncoderTest,
3680 NoChangeForInitialNormalUsage_MaintainResolutionMode) {
asapersson02465b82017-04-10 01:12:52 -07003681 const int kWidth = 1280;
3682 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003683 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003684 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003685
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003686 // Enable MAINTAIN_RESOLUTION preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07003687 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07003688 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003689 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
asapersson02465b82017-04-10 01:12:52 -07003690
3691 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003692 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003693 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003694 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 01:12:52 -07003695 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3696
3697 // Trigger adapt up, expect no change.
Henrik Boström91aa7322020-04-28 12:24:33 +02003698 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003699 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003700 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 01:12:52 -07003701 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3702
mflodmancc3d4422017-08-03 08:27:51 -07003703 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003704}
3705
mflodmancc3d4422017-08-03 08:27:51 -07003706TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07003707 const int kWidth = 1280;
3708 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003709 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003710 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003711
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003712 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07003713 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003714 video_stream_encoder_->SetSource(&source,
3715 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003716
3717 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3718 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003719 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003720 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3721 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3722 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3723
3724 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003725 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003726 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003727 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3728 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3729 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3730
mflodmancc3d4422017-08-03 08:27:51 -07003731 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003732}
3733
mflodmancc3d4422017-08-03 08:27:51 -07003734TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_DisabledMode) {
asapersson09f05612017-05-15 23:40:18 -07003735 const int kWidth = 1280;
3736 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003737 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003738 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07003739
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003740 // Enable DISABLED preference, no initial limitation.
asapersson09f05612017-05-15 23:40:18 -07003741 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003742 video_stream_encoder_->SetSource(&source,
3743 webrtc::DegradationPreference::DISABLED);
asapersson09f05612017-05-15 23:40:18 -07003744
3745 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3746 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003747 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003748 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3749 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3750 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3751
3752 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003753 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003754 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003755 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3756 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3757 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3758
mflodmancc3d4422017-08-03 08:27:51 -07003759 video_stream_encoder_->Stop();
asapersson09f05612017-05-15 23:40:18 -07003760}
3761
mflodmancc3d4422017-08-03 08:27:51 -07003762TEST_F(VideoStreamEncoderTest,
3763 AdaptsResolutionForLowQuality_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07003764 const int kWidth = 1280;
3765 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003766 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003767 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003768
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003769 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003770 AdaptingFrameForwarder source(&time_controller_);
asapersson02465b82017-04-10 01:12:52 -07003771 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07003772 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003773 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07003774
3775 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003776 WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003777 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003778 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3779 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3780
3781 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003782 video_stream_encoder_->TriggerQualityLow();
asapersson02465b82017-04-10 01:12:52 -07003783 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003784 WaitForEncodedFrame(2);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003785 EXPECT_THAT(source.sink_wants(),
3786 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson02465b82017-04-10 01:12:52 -07003787 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3788 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3789
3790 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003791 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003792 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003793 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3794 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3795 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3796
mflodmancc3d4422017-08-03 08:27:51 -07003797 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003798}
3799
mflodmancc3d4422017-08-03 08:27:51 -07003800TEST_F(VideoStreamEncoderTest,
3801 AdaptsFramerateForLowQuality_MaintainResolutionMode) {
asapersson09f05612017-05-15 23:40:18 -07003802 const int kWidth = 1280;
3803 const int kHeight = 720;
3804 const int kInputFps = 30;
Henrik Boström381d1092020-05-12 18:49:07 +02003805 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003806 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07003807
3808 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3809 stats.input_frame_rate = kInputFps;
3810 stats_proxy_->SetMockStats(stats);
3811
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003812 // Expect no scaling to begin with (preference: MAINTAIN_FRAMERATE).
asapersson09f05612017-05-15 23:40:18 -07003813 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3814 sink_.WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003815 EXPECT_THAT(video_source_.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003816
3817 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003818 video_stream_encoder_->TriggerQualityLow();
asapersson09f05612017-05-15 23:40:18 -07003819 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
3820 sink_.WaitForEncodedFrame(2);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003821 EXPECT_THAT(video_source_.sink_wants(),
3822 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson09f05612017-05-15 23:40:18 -07003823
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003824 // Enable MAINTAIN_RESOLUTION preference.
asapersson09f05612017-05-15 23:40:18 -07003825 test::FrameForwarder new_video_source;
Henrik Boström2671dac2020-05-19 16:29:09 +02003826 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003827 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01003828 // Give the encoder queue time to process the change in degradation preference
3829 // by waiting for an encoded frame.
3830 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
3831 sink_.WaitForEncodedFrame(3);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003832 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003833
3834 // Trigger adapt down, expect reduced framerate.
mflodmancc3d4422017-08-03 08:27:51 -07003835 video_stream_encoder_->TriggerQualityLow();
Henrik Boström07b17df2020-01-15 11:42:12 +01003836 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
3837 sink_.WaitForEncodedFrame(4);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003838 EXPECT_THAT(new_video_source.sink_wants(),
3839 FpsMatchesResolutionMax(Lt(kInputFps)));
asapersson09f05612017-05-15 23:40:18 -07003840
3841 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003842 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003843 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003844
mflodmancc3d4422017-08-03 08:27:51 -07003845 video_stream_encoder_->Stop();
asapersson09f05612017-05-15 23:40:18 -07003846}
3847
mflodmancc3d4422017-08-03 08:27:51 -07003848TEST_F(VideoStreamEncoderTest, DoesNotScaleBelowSetResolutionLimit) {
asaperssond0de2952017-04-21 01:47:31 -07003849 const int kWidth = 1280;
3850 const int kHeight = 720;
3851 const size_t kNumFrames = 10;
3852
Henrik Boström381d1092020-05-12 18:49:07 +02003853 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003854 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
kthelgason5e13d412016-12-01 03:59:51 -08003855
asaperssond0de2952017-04-21 01:47:31 -07003856 // Enable adapter, expected input resolutions when downscaling:
asapersson142fcc92017-08-17 08:58:54 -07003857 // 1280x720 -> 960x540 -> 640x360 -> 480x270 -> 320x180 (kMinPixelsPerFrame)
asaperssond0de2952017-04-21 01:47:31 -07003858 video_source_.set_adaptation_enabled(true);
3859
3860 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3861 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3862
3863 int downscales = 0;
3864 for (size_t i = 1; i <= kNumFrames; i++) {
Åsa Persson8c1bf952018-09-13 10:42:19 +02003865 video_source_.IncomingCapturedFrame(
3866 CreateFrame(i * kFrameIntervalMs, kWidth, kHeight));
3867 WaitForEncodedFrame(i * kFrameIntervalMs);
asaperssond0de2952017-04-21 01:47:31 -07003868
asaperssonfab67072017-04-04 05:51:49 -07003869 // Trigger scale down.
asaperssond0de2952017-04-21 01:47:31 -07003870 rtc::VideoSinkWants last_wants = video_source_.sink_wants();
mflodmancc3d4422017-08-03 08:27:51 -07003871 video_stream_encoder_->TriggerQualityLow();
sprangc5d62e22017-04-02 23:53:04 -07003872 EXPECT_GE(video_source_.sink_wants().max_pixel_count, kMinPixelsPerFrame);
asaperssond0de2952017-04-21 01:47:31 -07003873
3874 if (video_source_.sink_wants().max_pixel_count < last_wants.max_pixel_count)
3875 ++downscales;
3876
3877 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3878 EXPECT_EQ(downscales,
3879 stats_proxy_->GetStats().number_of_quality_adapt_changes);
3880 EXPECT_GT(downscales, 0);
kthelgason5e13d412016-12-01 03:59:51 -08003881 }
mflodmancc3d4422017-08-03 08:27:51 -07003882 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07003883}
3884
mflodmancc3d4422017-08-03 08:27:51 -07003885TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07003886 AdaptsResolutionUpAndDownTwiceOnOveruse_MaintainFramerateMode) {
3887 const int kWidth = 1280;
3888 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003889 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003890 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07003891
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003892 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003893 AdaptingFrameForwarder source(&time_controller_);
asaperssond0de2952017-04-21 01:47:31 -07003894 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07003895 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003896 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07003897
Åsa Persson8c1bf952018-09-13 10:42:19 +02003898 int64_t timestamp_ms = kFrameIntervalMs;
3899 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003900 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003901 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003902 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3903 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3904
3905 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003906 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003907 timestamp_ms += kFrameIntervalMs;
3908 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3909 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003910 EXPECT_THAT(source.sink_wants(),
3911 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07003912 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3913 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3914
3915 // Trigger adapt up, expect no restriction.
Henrik Boström91aa7322020-04-28 12:24:33 +02003916 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003917 timestamp_ms += kFrameIntervalMs;
3918 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003919 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003920 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003921 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3922 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3923
3924 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003925 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003926 timestamp_ms += kFrameIntervalMs;
3927 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3928 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003929 EXPECT_THAT(source.sink_wants(),
3930 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07003931 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3932 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3933
3934 // Trigger adapt up, expect no restriction.
Henrik Boström91aa7322020-04-28 12:24:33 +02003935 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003936 timestamp_ms += kFrameIntervalMs;
3937 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asapersson09f05612017-05-15 23:40:18 -07003938 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003939 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003940 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3941 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3942
mflodmancc3d4422017-08-03 08:27:51 -07003943 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07003944}
3945
mflodmancc3d4422017-08-03 08:27:51 -07003946TEST_F(VideoStreamEncoderTest,
asaperssonf7e294d2017-06-13 23:25:22 -07003947 AdaptsResolutionUpAndDownTwiceForLowQuality_BalancedMode_NoFpsLimit) {
3948 const int kWidth = 1280;
3949 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003950 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003951 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003952
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003953 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003954 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07003955 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003956 video_stream_encoder_->SetSource(&source,
3957 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003958
Åsa Persson8c1bf952018-09-13 10:42:19 +02003959 int64_t timestamp_ms = kFrameIntervalMs;
3960 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07003961 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003962 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003963 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3964 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3965
3966 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003967 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003968 timestamp_ms += kFrameIntervalMs;
3969 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3970 sink_.WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003971 EXPECT_THAT(source.sink_wants(),
3972 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07003973 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3974 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3975
3976 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003977 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003978 timestamp_ms += kFrameIntervalMs;
3979 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07003980 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003981 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003982 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3983 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3984
3985 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003986 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003987 timestamp_ms += kFrameIntervalMs;
3988 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3989 sink_.WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003990 EXPECT_THAT(source.sink_wants(),
3991 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07003992 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3993 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3994
3995 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003996 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003997 timestamp_ms += kFrameIntervalMs;
3998 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07003999 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004000 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07004001 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4002 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4003
mflodmancc3d4422017-08-03 08:27:51 -07004004 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07004005}
4006
Sergey Silkin41c650b2019-10-14 13:12:19 +02004007TEST_F(VideoStreamEncoderTest, AdaptUpIfBwEstimateIsHigherThanMinBitrate) {
4008 fake_encoder_.SetResolutionBitrateLimits(
4009 {kEncoderBitrateLimits540p, kEncoderBitrateLimits720p});
4010
Henrik Boström381d1092020-05-12 18:49:07 +02004011 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004012 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
4013 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
4014 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps), 0,
4015 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004016
4017 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004018 AdaptingFrameForwarder source(&time_controller_);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004019 source.set_adaptation_enabled(true);
4020 video_stream_encoder_->SetSource(
4021 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
4022
4023 // Insert 720p frame.
4024 int64_t timestamp_ms = kFrameIntervalMs;
4025 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
4026 WaitForEncodedFrame(1280, 720);
4027
4028 // Reduce bitrate and trigger adapt down.
Henrik Boström381d1092020-05-12 18:49:07 +02004029 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004030 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
4031 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
4032 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps), 0,
4033 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004034 video_stream_encoder_->TriggerQualityLow();
4035
4036 // Insert 720p frame. It should be downscaled and encoded.
4037 timestamp_ms += kFrameIntervalMs;
4038 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
4039 WaitForEncodedFrame(960, 540);
4040
4041 // Trigger adapt up. Higher resolution should not be requested duo to lack
4042 // of bitrate.
4043 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004044 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMatches(Lt(1280 * 720)));
Sergey Silkin41c650b2019-10-14 13:12:19 +02004045
4046 // Increase bitrate.
Henrik Boström381d1092020-05-12 18:49:07 +02004047 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004048 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
4049 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
4050 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps), 0,
4051 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004052
4053 // Trigger adapt up. Higher resolution should be requested.
4054 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004055 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
Sergey Silkin41c650b2019-10-14 13:12:19 +02004056
4057 video_stream_encoder_->Stop();
4058}
4059
4060TEST_F(VideoStreamEncoderTest, DropFirstFramesIfBwEstimateIsTooLow) {
4061 fake_encoder_.SetResolutionBitrateLimits(
4062 {kEncoderBitrateLimits540p, kEncoderBitrateLimits720p});
4063
4064 // Set bitrate equal to min bitrate of 540p.
Henrik Boström381d1092020-05-12 18:49:07 +02004065 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004066 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
4067 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
4068 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps), 0,
4069 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004070
4071 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004072 AdaptingFrameForwarder source(&time_controller_);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004073 source.set_adaptation_enabled(true);
4074 video_stream_encoder_->SetSource(
4075 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
4076
4077 // Insert 720p frame. It should be dropped and lower resolution should be
4078 // requested.
4079 int64_t timestamp_ms = kFrameIntervalMs;
4080 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
4081 ExpectDroppedFrame();
Henrik Boström2671dac2020-05-19 16:29:09 +02004082 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < 1280 * 720, 5000);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004083
4084 // Insert 720p frame. It should be downscaled and encoded.
4085 timestamp_ms += kFrameIntervalMs;
4086 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
4087 WaitForEncodedFrame(960, 540);
4088
4089 video_stream_encoder_->Stop();
4090}
4091
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004092class BalancedDegradationTest : public VideoStreamEncoderTest {
4093 protected:
4094 void SetupTest() {
4095 // Reset encoder for field trials to take effect.
4096 ConfigureEncoder(video_encoder_config_.Copy());
Asa Persson606d3cb2021-10-04 10:07:11 +02004097 OnBitrateUpdated(kTargetBitrate);
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004098
4099 // Enable BALANCED preference.
4100 source_.set_adaptation_enabled(true);
Åsa Perssonccfb3402019-09-25 15:13:04 +02004101 video_stream_encoder_->SetSource(&source_, DegradationPreference::BALANCED);
4102 }
4103
Asa Persson606d3cb2021-10-04 10:07:11 +02004104 void OnBitrateUpdated(DataRate bitrate) {
Henrik Boström381d1092020-05-12 18:49:07 +02004105 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004106 bitrate, bitrate, bitrate, 0, 0, 0);
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004107 }
4108
Åsa Persson45b176f2019-09-30 11:19:05 +02004109 void InsertFrame() {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004110 timestamp_ms_ += kFrameIntervalMs;
4111 source_.IncomingCapturedFrame(CreateFrame(timestamp_ms_, kWidth, kHeight));
Åsa Persson45b176f2019-09-30 11:19:05 +02004112 }
4113
4114 void InsertFrameAndWaitForEncoded() {
4115 InsertFrame();
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004116 sink_.WaitForEncodedFrame(timestamp_ms_);
4117 }
4118
4119 const int kWidth = 640; // pixels:640x360=230400
4120 const int kHeight = 360;
4121 const int64_t kFrameIntervalMs = 150; // Use low fps to not drop any frame.
4122 int64_t timestamp_ms_ = 0;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004123 AdaptingFrameForwarder source_{&time_controller_};
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004124};
4125
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02004126TEST_F(BalancedDegradationTest, AdaptDownTwiceIfMinFpsDiffLtThreshold) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01004127 test::ScopedKeyValueConfig field_trials(
4128 field_trials_,
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004129 "WebRTC-Video-BalancedDegradationSettings/"
4130 "pixels:57600|129600|230400,fps:7|10|24,fps_diff:1|1|1/");
4131 SetupTest();
4132
4133 // Force input frame rate.
4134 const int kInputFps = 24;
4135 VideoSendStream::Stats stats = stats_proxy_->GetStats();
4136 stats.input_frame_rate = kInputFps;
4137 stats_proxy_->SetMockStats(stats);
4138
Åsa Persson45b176f2019-09-30 11:19:05 +02004139 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004140 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004141
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02004142 // Trigger adapt down, expect scaled down framerate and resolution,
4143 // since Fps diff (input-requested:0) < threshold.
4144 video_stream_encoder_->TriggerQualityLow();
4145 EXPECT_THAT(source_.sink_wants(),
4146 AllOf(WantsFps(Eq(24)), WantsMaxPixels(Le(230400))));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004147
4148 video_stream_encoder_->Stop();
4149}
4150
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02004151TEST_F(BalancedDegradationTest, AdaptDownOnceIfFpsDiffGeThreshold) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01004152 test::ScopedKeyValueConfig field_trials(
4153 field_trials_,
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004154 "WebRTC-Video-BalancedDegradationSettings/"
4155 "pixels:57600|129600|230400,fps:7|10|24,fps_diff:1|1|1/");
4156 SetupTest();
4157
4158 // Force input frame rate.
4159 const int kInputFps = 25;
4160 VideoSendStream::Stats stats = stats_proxy_->GetStats();
4161 stats.input_frame_rate = kInputFps;
4162 stats_proxy_->SetMockStats(stats);
4163
Åsa Persson45b176f2019-09-30 11:19:05 +02004164 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004165 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004166
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02004167 // Trigger adapt down, expect scaled down framerate only (640x360@24fps).
4168 // Fps diff (input-requested:1) == threshold.
4169 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004170 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(24)));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004171
4172 video_stream_encoder_->Stop();
4173}
4174
4175TEST_F(BalancedDegradationTest, AdaptDownUsesCodecSpecificFps) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01004176 test::ScopedKeyValueConfig field_trials(
4177 field_trials_,
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004178 "WebRTC-Video-BalancedDegradationSettings/"
4179 "pixels:57600|129600|230400,fps:7|10|24,vp8_fps:8|11|22/");
4180 SetupTest();
4181
4182 EXPECT_EQ(kVideoCodecVP8, video_encoder_config_.codec_type);
4183
Åsa Persson45b176f2019-09-30 11:19:05 +02004184 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004185 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004186
4187 // Trigger adapt down, expect scaled down framerate (640x360@22fps).
4188 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004189 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(22)));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004190
4191 video_stream_encoder_->Stop();
4192}
4193
Åsa Perssonccfb3402019-09-25 15:13:04 +02004194TEST_F(BalancedDegradationTest, NoAdaptUpIfBwEstimateIsLessThanMinBitrate) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01004195 test::ScopedKeyValueConfig field_trials(
4196 field_trials_,
Åsa Persson1b247f12019-08-14 17:26:39 +02004197 "WebRTC-Video-BalancedDegradationSettings/"
4198 "pixels:57600|129600|230400,fps:7|10|14,kbps:0|0|425/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02004199 SetupTest();
Åsa Persson1b247f12019-08-14 17:26:39 +02004200
Asa Persson606d3cb2021-10-04 10:07:11 +02004201 const DataRate kMinBitrate = DataRate::KilobitsPerSec(425);
4202 const DataRate kTooLowMinBitrate = DataRate::KilobitsPerSec(424);
4203 OnBitrateUpdated(kTooLowMinBitrate);
Åsa Persson1b247f12019-08-14 17:26:39 +02004204
Åsa Persson45b176f2019-09-30 11:19:05 +02004205 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004206 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson1b247f12019-08-14 17:26:39 +02004207 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4208
4209 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
4210 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004211 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004212 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson1b247f12019-08-14 17:26:39 +02004213 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4214
4215 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
4216 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004217 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004218 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson1b247f12019-08-14 17:26:39 +02004219 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4220
Åsa Persson30ab0152019-08-27 12:22:33 +02004221 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
4222 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004223 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004224 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Perssonccfb3402019-09-25 15:13:04 +02004225 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 10);
Åsa Persson30ab0152019-08-27 12:22:33 +02004226 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4227
4228 // Trigger adapt up, expect no upscale in fps (target bitrate < min bitrate).
Åsa Persson1b247f12019-08-14 17:26:39 +02004229 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004230 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02004231 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
Åsa Persson1b247f12019-08-14 17:26:39 +02004232
Åsa Persson30ab0152019-08-27 12:22:33 +02004233 // Trigger adapt up, expect upscaled fps (target bitrate == min bitrate).
Asa Persson606d3cb2021-10-04 10:07:11 +02004234 OnBitrateUpdated(kMinBitrate);
Åsa Persson1b247f12019-08-14 17:26:39 +02004235 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004236 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02004237 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 14);
Åsa Persson30ab0152019-08-27 12:22:33 +02004238 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4239
4240 video_stream_encoder_->Stop();
4241}
4242
Åsa Perssonccfb3402019-09-25 15:13:04 +02004243TEST_F(BalancedDegradationTest,
Åsa Persson45b176f2019-09-30 11:19:05 +02004244 InitialFrameDropAdaptsFpsAndResolutionInOneStep) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01004245 test::ScopedKeyValueConfig field_trials(
4246 field_trials_,
Åsa Persson45b176f2019-09-30 11:19:05 +02004247 "WebRTC-Video-BalancedDegradationSettings/"
4248 "pixels:57600|129600|230400,fps:7|24|24/");
4249 SetupTest();
Asa Persson606d3cb2021-10-04 10:07:11 +02004250 OnBitrateUpdated(kLowTargetBitrate);
Åsa Persson45b176f2019-09-30 11:19:05 +02004251
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004252 EXPECT_THAT(source_.sink_wants(), UnlimitedSinkWants());
Åsa Persson45b176f2019-09-30 11:19:05 +02004253
4254 // Insert frame, expect scaled down:
4255 // framerate (640x360@24fps) -> resolution (480x270@24fps).
4256 InsertFrame();
4257 EXPECT_FALSE(WaitForFrame(1000));
4258 EXPECT_LT(source_.sink_wants().max_pixel_count, kWidth * kHeight);
4259 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 24);
4260
4261 // Insert frame, expect scaled down:
4262 // resolution (320x180@24fps).
4263 InsertFrame();
4264 EXPECT_FALSE(WaitForFrame(1000));
4265 EXPECT_LT(source_.sink_wants().max_pixel_count,
4266 source_.last_wants().max_pixel_count);
4267 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 24);
4268
4269 // Frame should not be dropped (min pixels per frame reached).
4270 InsertFrameAndWaitForEncoded();
4271
4272 video_stream_encoder_->Stop();
4273}
4274
4275TEST_F(BalancedDegradationTest,
Åsa Persson30ab0152019-08-27 12:22:33 +02004276 NoAdaptUpInResolutionIfBwEstimateIsLessThanMinBitrate) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01004277 test::ScopedKeyValueConfig field_trials(
4278 field_trials_,
Åsa Persson30ab0152019-08-27 12:22:33 +02004279 "WebRTC-Video-BalancedDegradationSettings/"
4280 "pixels:57600|129600|230400,fps:7|10|14,kbps_res:0|0|435/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02004281 SetupTest();
Åsa Persson30ab0152019-08-27 12:22:33 +02004282
Asa Persson606d3cb2021-10-04 10:07:11 +02004283 const DataRate kResolutionMinBitrate = DataRate::KilobitsPerSec(435);
4284 const DataRate kTooLowMinResolutionBitrate = DataRate::KilobitsPerSec(434);
4285 OnBitrateUpdated(kTooLowMinResolutionBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004286
Åsa Persson45b176f2019-09-30 11:19:05 +02004287 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004288 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson30ab0152019-08-27 12:22:33 +02004289 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4290
4291 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
4292 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004293 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004294 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson30ab0152019-08-27 12:22:33 +02004295 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4296
4297 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
4298 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004299 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004300 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004301 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4302
4303 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
4304 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004305 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004306 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Persson1b247f12019-08-14 17:26:39 +02004307 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4308
Åsa Persson30ab0152019-08-27 12:22:33 +02004309 // Trigger adapt up, expect upscaled fps (no bitrate limit) (480x270@14fps).
4310 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004311 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004312 EXPECT_THAT(source_.sink_wants(), FpsGtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004313 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4314
4315 // Trigger adapt up, expect no upscale in res (target bitrate < min bitrate).
4316 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004317 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02004318 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4319
4320 // Trigger adapt up, expect upscaled res (target bitrate == min bitrate).
Asa Persson606d3cb2021-10-04 10:07:11 +02004321 OnBitrateUpdated(kResolutionMinBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004322 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004323 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004324 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionGt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004325 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4326
4327 video_stream_encoder_->Stop();
4328}
4329
Åsa Perssonccfb3402019-09-25 15:13:04 +02004330TEST_F(BalancedDegradationTest,
Åsa Persson30ab0152019-08-27 12:22:33 +02004331 NoAdaptUpInFpsAndResolutionIfBwEstimateIsLessThanMinBitrate) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01004332 test::ScopedKeyValueConfig field_trials(
4333 field_trials_,
Åsa Persson30ab0152019-08-27 12:22:33 +02004334 "WebRTC-Video-BalancedDegradationSettings/"
4335 "pixels:57600|129600|230400,fps:7|10|14,kbps:0|0|425,kbps_res:0|0|435/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02004336 SetupTest();
Åsa Persson30ab0152019-08-27 12:22:33 +02004337
Asa Persson606d3cb2021-10-04 10:07:11 +02004338 const DataRate kMinBitrate = DataRate::KilobitsPerSec(425);
4339 const DataRate kTooLowMinBitrate = DataRate::KilobitsPerSec(424);
4340 const DataRate kResolutionMinBitrate = DataRate::KilobitsPerSec(435);
4341 const DataRate kTooLowMinResolutionBitrate = DataRate::KilobitsPerSec(434);
4342 OnBitrateUpdated(kTooLowMinBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004343
Åsa Persson45b176f2019-09-30 11:19:05 +02004344 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004345 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson30ab0152019-08-27 12:22:33 +02004346 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4347
4348 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
4349 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004350 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004351 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson30ab0152019-08-27 12:22:33 +02004352 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4353
4354 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
4355 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004356 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004357 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004358 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4359
4360 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
4361 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004362 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004363 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004364 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4365
4366 // Trigger adapt up, expect no upscale (target bitrate < min bitrate).
4367 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004368 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02004369 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4370
4371 // Trigger adapt up, expect upscaled fps (target bitrate == min bitrate).
Asa Persson606d3cb2021-10-04 10:07:11 +02004372 OnBitrateUpdated(kMinBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004373 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004374 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004375 EXPECT_THAT(source_.sink_wants(), FpsGtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004376 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4377
4378 // Trigger adapt up, expect no upscale in res (target bitrate < min bitrate).
Asa Persson606d3cb2021-10-04 10:07:11 +02004379 OnBitrateUpdated(kTooLowMinResolutionBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004380 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004381 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02004382 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4383
4384 // Trigger adapt up, expect upscaled res (target bitrate == min bitrate).
Asa Persson606d3cb2021-10-04 10:07:11 +02004385 OnBitrateUpdated(kResolutionMinBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004386 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004387 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004388 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionGt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004389 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4390
Åsa Persson1b247f12019-08-14 17:26:39 +02004391 video_stream_encoder_->Stop();
4392}
4393
mflodmancc3d4422017-08-03 08:27:51 -07004394TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07004395 AdaptsResolutionOnOveruseAndLowQuality_MaintainFramerateMode) {
4396 const int kWidth = 1280;
4397 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02004398 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004399 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07004400
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004401 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004402 AdaptingFrameForwarder source(&time_controller_);
asaperssond0de2952017-04-21 01:47:31 -07004403 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07004404 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004405 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07004406
Åsa Persson8c1bf952018-09-13 10:42:19 +02004407 int64_t timestamp_ms = kFrameIntervalMs;
4408 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004409 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004410 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07004411 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4412 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4413 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4414 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4415
4416 // Trigger cpu adapt down, expect scaled down resolution (960x540).
mflodmancc3d4422017-08-03 08:27:51 -07004417 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004418 timestamp_ms += kFrameIntervalMs;
4419 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4420 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004421 EXPECT_THAT(source.sink_wants(),
4422 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07004423 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4424 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4425 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4426 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4427
4428 // Trigger cpu adapt down, expect scaled down resolution (640x360).
mflodmancc3d4422017-08-03 08:27:51 -07004429 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004430 timestamp_ms += kFrameIntervalMs;
4431 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4432 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004433 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssond0de2952017-04-21 01:47:31 -07004434 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4435 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4436 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4437 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4438
Jonathan Yubc771b72017-12-08 17:04:29 -08004439 // Trigger cpu adapt down, expect scaled down resolution (480x270).
mflodmancc3d4422017-08-03 08:27:51 -07004440 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004441 timestamp_ms += kFrameIntervalMs;
4442 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4443 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004444 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssond0de2952017-04-21 01:47:31 -07004445 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4446 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08004447 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07004448 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4449
Jonathan Yubc771b72017-12-08 17:04:29 -08004450 // Trigger quality adapt down, expect scaled down resolution (320x180).
mflodmancc3d4422017-08-03 08:27:51 -07004451 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004452 timestamp_ms += kFrameIntervalMs;
4453 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4454 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004455 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
Jonathan Yubc771b72017-12-08 17:04:29 -08004456 rtc::VideoSinkWants last_wants = source.sink_wants();
asaperssond0de2952017-04-21 01:47:31 -07004457 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4458 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4459 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4460 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4461
Jonathan Yubc771b72017-12-08 17:04:29 -08004462 // Trigger quality adapt down, expect no change (min resolution reached).
4463 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004464 timestamp_ms += kFrameIntervalMs;
4465 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4466 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004467 EXPECT_THAT(source.sink_wants(), FpsMax());
4468 EXPECT_EQ(source.sink_wants().max_pixel_count, last_wants.max_pixel_count);
Jonathan Yubc771b72017-12-08 17:04:29 -08004469 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4470 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4471 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4472 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4473
Evan Shrubsole64469032020-06-11 10:45:29 +02004474 // Trigger quality adapt up, expect upscaled resolution (480x270).
4475 video_stream_encoder_->TriggerQualityHigh();
4476 timestamp_ms += kFrameIntervalMs;
4477 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4478 WaitForEncodedFrame(timestamp_ms);
4479 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
4480 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4481 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4482 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4483 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4484
4485 // Trigger quality and cpu adapt up since both are most limited, expect
4486 // upscaled resolution (640x360).
Henrik Boström91aa7322020-04-28 12:24:33 +02004487 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole64469032020-06-11 10:45:29 +02004488 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004489 timestamp_ms += kFrameIntervalMs;
4490 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4491 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004492 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Jonathan Yubc771b72017-12-08 17:04:29 -08004493 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4494 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4495 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02004496 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
Jonathan Yubc771b72017-12-08 17:04:29 -08004497
Evan Shrubsole64469032020-06-11 10:45:29 +02004498 // Trigger quality and cpu adapt up since both are most limited, expect
4499 // upscaled resolution (960x540).
Henrik Boström91aa7322020-04-28 12:24:33 +02004500 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole64469032020-06-11 10:45:29 +02004501 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004502 timestamp_ms += kFrameIntervalMs;
4503 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4504 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004505 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssond0de2952017-04-21 01:47:31 -07004506 last_wants = source.sink_wants();
Evan Shrubsole64469032020-06-11 10:45:29 +02004507 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
asaperssond0de2952017-04-21 01:47:31 -07004508 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Evan Shrubsole64469032020-06-11 10:45:29 +02004509 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4510 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07004511
Evan Shrubsole64469032020-06-11 10:45:29 +02004512 // Trigger cpu adapt up, expect no change since not most limited (960x540).
4513 // However the stats will change since the CPU resource is no longer limited.
Henrik Boström91aa7322020-04-28 12:24:33 +02004514 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004515 timestamp_ms += kFrameIntervalMs;
4516 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4517 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004518 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(last_wants));
asaperssond0de2952017-04-21 01:47:31 -07004519 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4520 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08004521 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02004522 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07004523
4524 // Trigger quality adapt up, expect no restriction (1280x720).
mflodmancc3d4422017-08-03 08:27:51 -07004525 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004526 timestamp_ms += kFrameIntervalMs;
4527 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004528 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004529 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004530 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07004531 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4532 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08004533 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02004534 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
kthelgason5e13d412016-12-01 03:59:51 -08004535
mflodmancc3d4422017-08-03 08:27:51 -07004536 video_stream_encoder_->Stop();
kthelgason5e13d412016-12-01 03:59:51 -08004537}
4538
mflodmancc3d4422017-08-03 08:27:51 -07004539TEST_F(VideoStreamEncoderTest, CpuLimitedHistogramIsReported) {
asaperssonfab67072017-04-04 05:51:49 -07004540 const int kWidth = 640;
4541 const int kHeight = 360;
perkj803d97f2016-11-01 11:45:46 -07004542
Henrik Boström381d1092020-05-12 18:49:07 +02004543 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004544 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07004545
perkj803d97f2016-11-01 11:45:46 -07004546 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07004547 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004548 WaitForEncodedFrame(i);
perkj803d97f2016-11-01 11:45:46 -07004549 }
4550
mflodmancc3d4422017-08-03 08:27:51 -07004551 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07004552 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07004553 video_source_.IncomingCapturedFrame(CreateFrame(
4554 SendStatisticsProxy::kMinRequiredMetricsSamples + i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004555 WaitForEncodedFrame(SendStatisticsProxy::kMinRequiredMetricsSamples + i);
perkj803d97f2016-11-01 11:45:46 -07004556 }
4557
mflodmancc3d4422017-08-03 08:27:51 -07004558 video_stream_encoder_->Stop();
4559 video_stream_encoder_.reset();
perkj803d97f2016-11-01 11:45:46 -07004560 stats_proxy_.reset();
sprangf8ee65e2017-02-28 08:49:33 -08004561
Ying Wangef3998f2019-12-09 13:06:53 +01004562 EXPECT_METRIC_EQ(
4563 1, metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
4564 EXPECT_METRIC_EQ(
perkj803d97f2016-11-01 11:45:46 -07004565 1, metrics::NumEvents("WebRTC.Video.CpuLimitedResolutionInPercent", 50));
4566}
4567
mflodmancc3d4422017-08-03 08:27:51 -07004568TEST_F(VideoStreamEncoderTest,
4569 CpuLimitedHistogramIsNotReportedForDisabledDegradation) {
Henrik Boström381d1092020-05-12 18:49:07 +02004570 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004571 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf4e44af2017-04-19 02:01:06 -07004572 const int kWidth = 640;
4573 const int kHeight = 360;
4574
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004575 video_stream_encoder_->SetSource(&video_source_,
4576 webrtc::DegradationPreference::DISABLED);
asaperssonf4e44af2017-04-19 02:01:06 -07004577
4578 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
4579 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004580 WaitForEncodedFrame(i);
asaperssonf4e44af2017-04-19 02:01:06 -07004581 }
4582
mflodmancc3d4422017-08-03 08:27:51 -07004583 video_stream_encoder_->Stop();
4584 video_stream_encoder_.reset();
asaperssonf4e44af2017-04-19 02:01:06 -07004585 stats_proxy_.reset();
4586
4587 EXPECT_EQ(0,
4588 metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
4589}
4590
Per Kjellanderdcef6412020-10-07 15:09:05 +02004591TEST_F(VideoStreamEncoderTest, ReportsVideoBitrateAllocation) {
4592 ResetEncoder("FAKE", 1, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004593 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02004594 kVideoBitrateAllocation);
sprang57c2fff2017-01-16 06:24:02 -08004595
4596 const int kDefaultFps = 30;
Erik Språng566124a2018-04-23 12:32:22 +02004597 const VideoBitrateAllocation expected_bitrate =
Asa Persson606d3cb2021-10-04 10:07:11 +02004598 SimulcastRateAllocator(fake_encoder_.config())
4599 .Allocate(VideoBitrateAllocationParameters(kLowTargetBitrate.bps(),
Florent Castelli8bbdb5b2019-08-02 15:16:28 +02004600 kDefaultFps));
sprang57c2fff2017-01-16 06:24:02 -08004601
Henrik Boström381d1092020-05-12 18:49:07 +02004602 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004603 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
sprang57c2fff2017-01-16 06:24:02 -08004604
sprang57c2fff2017-01-16 06:24:02 -08004605 video_source_.IncomingCapturedFrame(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004606 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4607 WaitForEncodedFrame(CurrentTimeMs());
Per Kjellanderdcef6412020-10-07 15:09:05 +02004608 EXPECT_EQ(sink_.GetLastVideoBitrateAllocation(), expected_bitrate);
4609 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
4610
Erik Språngd7329ca2019-02-21 21:19:53 +01004611 // Check that encoder has been updated too, not just allocation observer.
Erik Språng9d69cbe2020-10-22 17:44:42 +02004612 EXPECT_TRUE(fake_encoder_.GetAndResetLastRateControlSettings().has_value());
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004613 AdvanceTime(TimeDelta::Seconds(1) / kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08004614
Per Kjellanderdcef6412020-10-07 15:09:05 +02004615 // VideoBitrateAllocation not updated on second frame.
sprang57c2fff2017-01-16 06:24:02 -08004616 video_source_.IncomingCapturedFrame(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004617 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4618 WaitForEncodedFrame(CurrentTimeMs());
Per Kjellanderdcef6412020-10-07 15:09:05 +02004619 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004620 AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08004621
Per Kjellanderdcef6412020-10-07 15:09:05 +02004622 // VideoBitrateAllocation updated after a process interval.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004623 const int64_t start_time_ms = CurrentTimeMs();
Per Kjellanderd0a8f512020-10-07 11:28:41 +02004624 while (CurrentTimeMs() - start_time_ms < 5 * kProcessIntervalMs) {
Erik Språngd7329ca2019-02-21 21:19:53 +01004625 video_source_.IncomingCapturedFrame(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004626 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4627 WaitForEncodedFrame(CurrentTimeMs());
4628 AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
Erik Språngd7329ca2019-02-21 21:19:53 +01004629 }
Per Kjellanderdcef6412020-10-07 15:09:05 +02004630 EXPECT_GT(sink_.number_of_bitrate_allocations(), 3);
Erik Språngd7329ca2019-02-21 21:19:53 +01004631
mflodmancc3d4422017-08-03 08:27:51 -07004632 video_stream_encoder_->Stop();
sprang57c2fff2017-01-16 06:24:02 -08004633}
4634
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004635TEST_F(VideoStreamEncoderTest, ReportsVideoLayersAllocationForVP8Simulcast) {
Per Kjellandera9434842020-10-15 17:53:22 +02004636 ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004637 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellandera9434842020-10-15 17:53:22 +02004638 kVideoLayersAllocation);
4639
4640 const int kDefaultFps = 30;
4641
4642 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004643 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Per Kjellandera9434842020-10-15 17:53:22 +02004644
4645 video_source_.IncomingCapturedFrame(
4646 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4647 WaitForEncodedFrame(CurrentTimeMs());
4648 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4649 VideoLayersAllocation last_layer_allocation =
4650 sink_.GetLastVideoLayersAllocation();
Asa Persson606d3cb2021-10-04 10:07:11 +02004651 // kLowTargetBitrate is only enough for one spatial layer.
Per Kjellandera9434842020-10-15 17:53:22 +02004652 ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 1u);
4653
4654 VideoBitrateAllocation bitrate_allocation =
Erik Språng9d69cbe2020-10-22 17:44:42 +02004655 fake_encoder_.GetAndResetLastRateControlSettings()->target_bitrate;
Per Kjellandera9434842020-10-15 17:53:22 +02004656 // Check that encoder has been updated too, not just allocation observer.
Asa Persson606d3cb2021-10-04 10:07:11 +02004657 EXPECT_EQ(bitrate_allocation.get_sum_bps(), kLowTargetBitrate.bps());
Per Kjellandera9434842020-10-15 17:53:22 +02004658 AdvanceTime(TimeDelta::Seconds(1) / kDefaultFps);
4659
Erik Språng9d69cbe2020-10-22 17:44:42 +02004660 // VideoLayersAllocation might be updated if frame rate changes.
Per Kjellandera9434842020-10-15 17:53:22 +02004661 int number_of_layers_allocation = 1;
4662 const int64_t start_time_ms = CurrentTimeMs();
4663 while (CurrentTimeMs() - start_time_ms < 10 * kProcessIntervalMs) {
4664 video_source_.IncomingCapturedFrame(
4665 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4666 WaitForEncodedFrame(CurrentTimeMs());
Per Kjellandera9434842020-10-15 17:53:22 +02004667 if (number_of_layers_allocation != sink_.number_of_layers_allocations()) {
4668 number_of_layers_allocation = sink_.number_of_layers_allocations();
4669 VideoLayersAllocation new_allocation =
4670 sink_.GetLastVideoLayersAllocation();
4671 ASSERT_EQ(new_allocation.active_spatial_layers.size(), 1u);
4672 EXPECT_NE(new_allocation.active_spatial_layers[0].frame_rate_fps,
4673 last_layer_allocation.active_spatial_layers[0].frame_rate_fps);
4674 EXPECT_EQ(new_allocation.active_spatial_layers[0]
4675 .target_bitrate_per_temporal_layer,
4676 last_layer_allocation.active_spatial_layers[0]
4677 .target_bitrate_per_temporal_layer);
4678 last_layer_allocation = new_allocation;
4679 }
4680 }
4681 EXPECT_LE(sink_.number_of_layers_allocations(), 3);
4682 video_stream_encoder_->Stop();
4683}
4684
4685TEST_F(VideoStreamEncoderTest,
Åsa Perssona7e34d32021-01-20 15:36:13 +01004686 ReportsVideoLayersAllocationForVP8WithMiddleLayerDisabled) {
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004687 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4688 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4689 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004690 VideoEncoderConfig video_encoder_config;
4691 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP8,
4692 /* num_streams*/ 3, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004693 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004694 video_encoder_config.content_type =
4695 VideoEncoderConfig::ContentType::kRealtimeVideo;
4696 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004697 rtc::make_ref_counted<VideoEncoderConfig::Vp8EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004698 VideoEncoder::GetDefaultVp8Settings());
4699 for (auto& layer : video_encoder_config.simulcast_layers) {
4700 layer.num_temporal_layers = 2;
4701 }
4702 // Simulcast layers are used for enabling/disabling streams.
4703 video_encoder_config.simulcast_layers[0].active = true;
4704 video_encoder_config.simulcast_layers[1].active = false;
4705 video_encoder_config.simulcast_layers[2].active = true;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004706 ConfigureEncoder(std::move(video_encoder_config),
4707 VideoStreamEncoder::BitrateAllocationCallbackType::
4708 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004709
4710 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004711 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004712
4713 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4714 WaitForEncodedFrame(CurrentTimeMs());
4715 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4716 VideoLayersAllocation last_layer_allocation =
4717 sink_.GetLastVideoLayersAllocation();
4718
4719 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4720 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4721 .target_bitrate_per_temporal_layer,
4722 SizeIs(2));
4723 EXPECT_LT(last_layer_allocation.active_spatial_layers[0].width, 1280);
4724 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 1280);
4725 video_stream_encoder_->Stop();
4726}
4727
4728TEST_F(VideoStreamEncoderTest,
Åsa Perssona7e34d32021-01-20 15:36:13 +01004729 ReportsVideoLayersAllocationForVP8WithMiddleAndHighestLayerDisabled) {
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004730 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4731 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4732 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004733 VideoEncoderConfig video_encoder_config;
4734 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP8,
4735 /* num_streams*/ 3, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004736 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004737 video_encoder_config.content_type =
4738 VideoEncoderConfig::ContentType::kRealtimeVideo;
4739 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004740 rtc::make_ref_counted<VideoEncoderConfig::Vp8EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004741 VideoEncoder::GetDefaultVp8Settings());
4742 for (auto& layer : video_encoder_config.simulcast_layers) {
4743 layer.num_temporal_layers = 2;
4744 }
4745 // Simulcast layers are used for enabling/disabling streams.
4746 video_encoder_config.simulcast_layers[0].active = true;
4747 video_encoder_config.simulcast_layers[1].active = false;
4748 video_encoder_config.simulcast_layers[2].active = false;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004749 ConfigureEncoder(std::move(video_encoder_config),
4750 VideoStreamEncoder::BitrateAllocationCallbackType::
4751 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004752
4753 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004754 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004755
4756 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4757 WaitForEncodedFrame(CurrentTimeMs());
4758 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4759 VideoLayersAllocation last_layer_allocation =
4760 sink_.GetLastVideoLayersAllocation();
4761
4762 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(1));
4763 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4764 .target_bitrate_per_temporal_layer,
4765 SizeIs(2));
4766 EXPECT_LT(last_layer_allocation.active_spatial_layers[0].width, 1280);
4767
4768 video_stream_encoder_->Stop();
4769}
4770
4771TEST_F(VideoStreamEncoderTest,
4772 ReportsVideoLayersAllocationForV9SvcWithTemporalLayerSupport) {
4773 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4774 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004775 VideoEncoderConfig video_encoder_config;
4776 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4777 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004778 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004779 video_encoder_config.content_type =
4780 VideoEncoderConfig::ContentType::kRealtimeVideo;
4781 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4782 vp9_settings.numberOfSpatialLayers = 2;
4783 vp9_settings.numberOfTemporalLayers = 2;
4784 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4785 vp9_settings.automaticResizeOn = false;
4786 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004787 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004788 vp9_settings);
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004789 ConfigureEncoder(std::move(video_encoder_config),
4790 VideoStreamEncoder::BitrateAllocationCallbackType::
4791 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004792
4793 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004794 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004795
4796 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4797 WaitForEncodedFrame(CurrentTimeMs());
4798 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4799 VideoLayersAllocation last_layer_allocation =
4800 sink_.GetLastVideoLayersAllocation();
4801
4802 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4803 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4804 .target_bitrate_per_temporal_layer,
4805 SizeIs(2));
4806 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 640);
4807 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].height, 360);
4808 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].frame_rate_fps, 30);
4809 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4810 .target_bitrate_per_temporal_layer,
4811 SizeIs(2));
4812 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 1280);
4813 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].height, 720);
4814 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].frame_rate_fps, 30);
4815
4816 // Since full SVC is used, expect the top layer to utilize the full target
4817 // rate.
4818 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1]
4819 .target_bitrate_per_temporal_layer[1],
Asa Persson606d3cb2021-10-04 10:07:11 +02004820 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004821 video_stream_encoder_->Stop();
4822}
4823
4824TEST_F(VideoStreamEncoderTest,
4825 ReportsVideoLayersAllocationForV9SvcWithoutTemporalLayerSupport) {
4826 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, false);
4827 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, false);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004828 VideoEncoderConfig video_encoder_config;
4829 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4830 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004831 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004832 video_encoder_config.content_type =
4833 VideoEncoderConfig::ContentType::kRealtimeVideo;
4834 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4835 vp9_settings.numberOfSpatialLayers = 2;
4836 vp9_settings.numberOfTemporalLayers = 2;
4837 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4838 vp9_settings.automaticResizeOn = false;
4839 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004840 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004841 vp9_settings);
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004842 ConfigureEncoder(std::move(video_encoder_config),
4843 VideoStreamEncoder::BitrateAllocationCallbackType::
4844 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004845
4846 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004847 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004848
4849 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4850 WaitForEncodedFrame(CurrentTimeMs());
4851 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4852 VideoLayersAllocation last_layer_allocation =
4853 sink_.GetLastVideoLayersAllocation();
4854
4855 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4856 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4857 .target_bitrate_per_temporal_layer,
4858 SizeIs(1));
4859 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4860 .target_bitrate_per_temporal_layer,
4861 SizeIs(1));
4862 // Since full SVC is used, expect the top layer to utilize the full target
4863 // rate.
4864 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1]
4865 .target_bitrate_per_temporal_layer[0],
Asa Persson606d3cb2021-10-04 10:07:11 +02004866 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004867 video_stream_encoder_->Stop();
4868}
4869
4870TEST_F(VideoStreamEncoderTest,
4871 ReportsVideoLayersAllocationForVP9KSvcWithTemporalLayerSupport) {
4872 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4873 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004874 VideoEncoderConfig video_encoder_config;
4875 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4876 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004877 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004878 video_encoder_config.content_type =
4879 VideoEncoderConfig::ContentType::kRealtimeVideo;
4880 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4881 vp9_settings.numberOfSpatialLayers = 2;
4882 vp9_settings.numberOfTemporalLayers = 2;
4883 vp9_settings.interLayerPred = InterLayerPredMode::kOnKeyPic;
4884 vp9_settings.automaticResizeOn = false;
4885 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004886 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004887 vp9_settings);
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004888 ConfigureEncoder(std::move(video_encoder_config),
4889 VideoStreamEncoder::BitrateAllocationCallbackType::
4890 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004891
4892 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004893 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004894
4895 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4896 WaitForEncodedFrame(CurrentTimeMs());
4897 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4898 VideoLayersAllocation last_layer_allocation =
4899 sink_.GetLastVideoLayersAllocation();
4900
4901 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4902 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4903 .target_bitrate_per_temporal_layer,
4904 SizeIs(2));
4905 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4906 .target_bitrate_per_temporal_layer,
4907 SizeIs(2));
4908 // Since KSVC is, spatial layers are independend except on key frames.
4909 EXPECT_LT(last_layer_allocation.active_spatial_layers[1]
4910 .target_bitrate_per_temporal_layer[1],
Asa Persson606d3cb2021-10-04 10:07:11 +02004911 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004912 video_stream_encoder_->Stop();
4913}
4914
4915TEST_F(VideoStreamEncoderTest,
4916 ReportsVideoLayersAllocationForV9SvcWithLowestLayerDisabled) {
4917 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4918 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4919 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004920 VideoEncoderConfig video_encoder_config;
4921 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4922 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004923 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004924 video_encoder_config.content_type =
4925 VideoEncoderConfig::ContentType::kRealtimeVideo;
4926 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4927 vp9_settings.numberOfSpatialLayers = 3;
4928 vp9_settings.numberOfTemporalLayers = 2;
4929 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4930 vp9_settings.automaticResizeOn = false;
4931 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004932 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004933 vp9_settings);
4934 // Simulcast layers are used for enabling/disabling streams.
4935 video_encoder_config.simulcast_layers.resize(3);
4936 video_encoder_config.simulcast_layers[0].active = false;
4937 video_encoder_config.simulcast_layers[1].active = true;
4938 video_encoder_config.simulcast_layers[2].active = true;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004939 ConfigureEncoder(std::move(video_encoder_config),
4940 VideoStreamEncoder::BitrateAllocationCallbackType::
4941 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004942
4943 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004944 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004945
4946 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4947 WaitForEncodedFrame(CurrentTimeMs());
4948 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4949 VideoLayersAllocation last_layer_allocation =
4950 sink_.GetLastVideoLayersAllocation();
4951
4952 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4953 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4954 .target_bitrate_per_temporal_layer,
4955 SizeIs(2));
4956 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 640);
4957 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].spatial_id, 0);
4958
4959 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 1280);
4960 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].spatial_id, 1);
4961 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4962 .target_bitrate_per_temporal_layer,
4963 SizeIs(2));
4964 // Since full SVC is used, expect the top layer to utilize the full target
4965 // rate.
4966 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1]
4967 .target_bitrate_per_temporal_layer[1],
Asa Persson606d3cb2021-10-04 10:07:11 +02004968 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004969 video_stream_encoder_->Stop();
4970}
4971
4972TEST_F(VideoStreamEncoderTest,
4973 ReportsVideoLayersAllocationForV9SvcWithHighestLayerDisabled) {
4974 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4975 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4976 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004977 VideoEncoderConfig video_encoder_config;
4978 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4979 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004980 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004981 video_encoder_config.content_type =
4982 VideoEncoderConfig::ContentType::kRealtimeVideo;
4983 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4984 vp9_settings.numberOfSpatialLayers = 3;
4985 vp9_settings.numberOfTemporalLayers = 2;
4986 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4987 vp9_settings.automaticResizeOn = false;
4988 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004989 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004990 vp9_settings);
4991 // Simulcast layers are used for enabling/disabling streams.
4992 video_encoder_config.simulcast_layers.resize(3);
4993 video_encoder_config.simulcast_layers[2].active = false;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004994 ConfigureEncoder(std::move(video_encoder_config),
4995 VideoStreamEncoder::BitrateAllocationCallbackType::
4996 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004997
4998 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004999 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005000
5001 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
5002 WaitForEncodedFrame(CurrentTimeMs());
5003 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
5004 VideoLayersAllocation last_layer_allocation =
5005 sink_.GetLastVideoLayersAllocation();
5006
5007 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
5008 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
5009 .target_bitrate_per_temporal_layer,
5010 SizeIs(2));
5011 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 320);
5012 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].spatial_id, 0);
5013
5014 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 640);
5015 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].spatial_id, 1);
5016 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
5017 .target_bitrate_per_temporal_layer,
5018 SizeIs(2));
5019 video_stream_encoder_->Stop();
5020}
5021
5022TEST_F(VideoStreamEncoderTest,
5023 ReportsVideoLayersAllocationForV9SvcWithAllButHighestLayerDisabled) {
5024 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
5025 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
5026 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005027 VideoEncoderConfig video_encoder_config;
5028 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
5029 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02005030 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005031 video_encoder_config.content_type =
5032 VideoEncoderConfig::ContentType::kRealtimeVideo;
5033 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5034 vp9_settings.numberOfSpatialLayers = 3;
5035 vp9_settings.numberOfTemporalLayers = 2;
5036 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
5037 vp9_settings.automaticResizeOn = false;
5038 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005039 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005040 vp9_settings);
5041 // Simulcast layers are used for enabling/disabling streams.
5042 video_encoder_config.simulcast_layers.resize(3);
5043 video_encoder_config.simulcast_layers[0].active = false;
5044 video_encoder_config.simulcast_layers[1].active = false;
5045 video_encoder_config.simulcast_layers[2].active = true;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005046 ConfigureEncoder(std::move(video_encoder_config),
5047 VideoStreamEncoder::BitrateAllocationCallbackType::
5048 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005049
5050 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005051 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005052
5053 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
5054 WaitForEncodedFrame(CurrentTimeMs());
5055 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
5056 VideoLayersAllocation last_layer_allocation =
5057 sink_.GetLastVideoLayersAllocation();
5058
5059 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(1));
5060 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
5061 .target_bitrate_per_temporal_layer,
5062 SizeIs(2));
5063 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 1280);
5064 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].spatial_id, 0);
5065 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0]
5066 .target_bitrate_per_temporal_layer[1],
Asa Persson606d3cb2021-10-04 10:07:11 +02005067 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005068 video_stream_encoder_->Stop();
5069}
5070
5071TEST_F(VideoStreamEncoderTest, ReportsVideoLayersAllocationForH264) {
5072 ResetEncoder("H264", 1, 1, 1, false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005073 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005074 kVideoLayersAllocation);
5075 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005076 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005077
5078 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
5079 WaitForEncodedFrame(CurrentTimeMs());
5080 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
5081 VideoLayersAllocation last_layer_allocation =
5082 sink_.GetLastVideoLayersAllocation();
5083
5084 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(1));
5085 ASSERT_THAT(last_layer_allocation.active_spatial_layers[0]
5086 .target_bitrate_per_temporal_layer,
5087 SizeIs(1));
5088 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0]
5089 .target_bitrate_per_temporal_layer[0],
Asa Persson606d3cb2021-10-04 10:07:11 +02005090 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005091 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 1280);
5092 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].height, 720);
5093 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].frame_rate_fps, 30);
5094 video_stream_encoder_->Stop();
5095}
5096
5097TEST_F(VideoStreamEncoderTest,
Per Kjellandera9434842020-10-15 17:53:22 +02005098 ReportsUpdatedVideoLayersAllocationWhenBweChanges) {
5099 ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005100 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellandera9434842020-10-15 17:53:22 +02005101 kVideoLayersAllocation);
5102
5103 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005104 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Per Kjellandera9434842020-10-15 17:53:22 +02005105
5106 video_source_.IncomingCapturedFrame(
5107 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
5108 WaitForEncodedFrame(CurrentTimeMs());
5109 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
5110 VideoLayersAllocation last_layer_allocation =
5111 sink_.GetLastVideoLayersAllocation();
Asa Persson606d3cb2021-10-04 10:07:11 +02005112 // kLowTargetBitrate is only enough for one spatial layer.
Per Kjellandera9434842020-10-15 17:53:22 +02005113 ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 1u);
5114 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0]
5115 .target_bitrate_per_temporal_layer[0],
Asa Persson606d3cb2021-10-04 10:07:11 +02005116 kLowTargetBitrate);
Per Kjellandera9434842020-10-15 17:53:22 +02005117
5118 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005119 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
5120 0, 0, 0);
Per Kjellandera9434842020-10-15 17:53:22 +02005121 video_source_.IncomingCapturedFrame(
5122 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
5123 WaitForEncodedFrame(CurrentTimeMs());
5124
5125 EXPECT_EQ(sink_.number_of_layers_allocations(), 2);
5126 last_layer_allocation = sink_.GetLastVideoLayersAllocation();
5127 ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 2u);
5128 EXPECT_GT(last_layer_allocation.active_spatial_layers[1]
5129 .target_bitrate_per_temporal_layer[0],
5130 DataRate::Zero());
5131
5132 video_stream_encoder_->Stop();
5133}
5134
Per Kjellander4190ce92020-12-15 17:24:55 +01005135TEST_F(VideoStreamEncoderTest,
5136 ReportsUpdatedVideoLayersAllocationWhenResolutionChanges) {
5137 ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005138 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellander4190ce92020-12-15 17:24:55 +01005139 kVideoLayersAllocation);
5140
5141 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005142 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
5143 0, 0, 0);
Per Kjellander4190ce92020-12-15 17:24:55 +01005144
5145 video_source_.IncomingCapturedFrame(
5146 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
5147 WaitForEncodedFrame(CurrentTimeMs());
5148 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
5149 ASSERT_THAT(sink_.GetLastVideoLayersAllocation().active_spatial_layers,
5150 SizeIs(2));
5151 EXPECT_EQ(sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].width,
5152 codec_width_);
5153 EXPECT_EQ(
5154 sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].height,
5155 codec_height_);
5156
5157 video_source_.IncomingCapturedFrame(
5158 CreateFrame(CurrentTimeMs(), codec_width_ / 2, codec_height_ / 2));
5159 WaitForEncodedFrame(CurrentTimeMs());
5160 EXPECT_EQ(sink_.number_of_layers_allocations(), 2);
5161 ASSERT_THAT(sink_.GetLastVideoLayersAllocation().active_spatial_layers,
5162 SizeIs(2));
5163 EXPECT_EQ(sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].width,
5164 codec_width_ / 2);
5165 EXPECT_EQ(
5166 sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].height,
5167 codec_height_ / 2);
5168
5169 video_stream_encoder_->Stop();
5170}
5171
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005172TEST_F(VideoStreamEncoderTest, TemporalLayersNotDisabledIfSupported) {
5173 // 2 TLs configured, temporal layers supported by encoder.
5174 const int kNumTemporalLayers = 2;
Per Kjellanderdcef6412020-10-07 15:09:05 +02005175 ResetEncoder("VP8", 1, kNumTemporalLayers, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005176 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02005177 kVideoBitrateAllocation);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005178 fake_encoder_.SetTemporalLayersSupported(0, true);
5179
5180 // Bitrate allocated across temporal layers.
Asa Persson606d3cb2021-10-04 10:07:11 +02005181 const int kTl0Bps = kTargetBitrate.bps() *
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005182 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01005183 kNumTemporalLayers, /*temporal_id*/ 0,
5184 /*base_heavy_tl3_alloc*/ false);
Asa Persson606d3cb2021-10-04 10:07:11 +02005185 const int kTl1Bps = kTargetBitrate.bps() *
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005186 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01005187 kNumTemporalLayers, /*temporal_id*/ 1,
5188 /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005189 VideoBitrateAllocation expected_bitrate;
5190 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTl0Bps);
5191 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kTl1Bps - kTl0Bps);
5192
5193 VerifyAllocatedBitrate(expected_bitrate);
5194 video_stream_encoder_->Stop();
5195}
5196
5197TEST_F(VideoStreamEncoderTest, TemporalLayersDisabledIfNotSupported) {
5198 // 2 TLs configured, temporal layers not supported by encoder.
Per Kjellanderdcef6412020-10-07 15:09:05 +02005199 ResetEncoder("VP8", 1, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005200 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02005201 kVideoBitrateAllocation);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005202 fake_encoder_.SetTemporalLayersSupported(0, false);
5203
5204 // Temporal layers not supported by the encoder.
5205 // Total bitrate should be at ti:0.
5206 VideoBitrateAllocation expected_bitrate;
Asa Persson606d3cb2021-10-04 10:07:11 +02005207 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTargetBitrate.bps());
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005208
5209 VerifyAllocatedBitrate(expected_bitrate);
5210 video_stream_encoder_->Stop();
5211}
5212
5213TEST_F(VideoStreamEncoderTest, VerifyBitrateAllocationForTwoStreams) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01005214 webrtc::test::ScopedKeyValueConfig field_trials(
5215 field_trials_,
Per Kjellanderdcef6412020-10-07 15:09:05 +02005216 "WebRTC-Video-QualityScalerSettings/"
5217 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
5218 // Reset encoder for field trials to take effect.
5219 ConfigureEncoder(video_encoder_config_.Copy());
5220
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005221 // 2 TLs configured, temporal layers only supported for first stream.
Per Kjellanderdcef6412020-10-07 15:09:05 +02005222 ResetEncoder("VP8", 2, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005223 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02005224 kVideoBitrateAllocation);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005225 fake_encoder_.SetTemporalLayersSupported(0, true);
5226 fake_encoder_.SetTemporalLayersSupported(1, false);
5227
5228 const int kS0Bps = 150000;
5229 const int kS0Tl0Bps =
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01005230 kS0Bps *
5231 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
5232 /*num_layers*/ 2, /*temporal_id*/ 0, /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005233 const int kS0Tl1Bps =
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01005234 kS0Bps *
5235 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
5236 /*num_layers*/ 2, /*temporal_id*/ 1, /*base_heavy_tl3_alloc*/ false);
Asa Persson606d3cb2021-10-04 10:07:11 +02005237 const int kS1Bps = kTargetBitrate.bps() - kS0Tl1Bps;
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005238 // Temporal layers not supported by si:1.
5239 VideoBitrateAllocation expected_bitrate;
5240 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kS0Tl0Bps);
5241 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kS0Tl1Bps - kS0Tl0Bps);
5242 expected_bitrate.SetBitrate(/*si*/ 1, /*ti*/ 0, kS1Bps);
5243
5244 VerifyAllocatedBitrate(expected_bitrate);
5245 video_stream_encoder_->Stop();
5246}
5247
Niels Möller7dc26b72017-12-06 10:27:48 +01005248TEST_F(VideoStreamEncoderTest, OveruseDetectorUpdatedOnReconfigureAndAdaption) {
5249 const int kFrameWidth = 1280;
5250 const int kFrameHeight = 720;
5251 const int kFramerate = 24;
5252
Henrik Boström381d1092020-05-12 18:49:07 +02005253 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005254 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller7dc26b72017-12-06 10:27:48 +01005255 test::FrameForwarder source;
5256 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005257 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 10:27:48 +01005258
5259 // Insert a single frame, triggering initial configuration.
5260 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
5261 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5262
5263 EXPECT_EQ(
5264 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5265 kDefaultFramerate);
5266
5267 // Trigger reconfigure encoder (without resetting the entire instance).
5268 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +02005269 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5270 video_encoder_config.simulcast_layers[0].max_framerate = kFramerate;
Asa Persson606d3cb2021-10-04 10:07:11 +02005271 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
Niels Möller7dc26b72017-12-06 10:27:48 +01005272 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02005273 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01005274 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5275
5276 // Detector should be updated with fps limit from codec config.
5277 EXPECT_EQ(
5278 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5279 kFramerate);
5280
5281 // Trigger overuse, max framerate should be reduced.
5282 VideoSendStream::Stats stats = stats_proxy_->GetStats();
5283 stats.input_frame_rate = kFramerate;
5284 stats_proxy_->SetMockStats(stats);
5285 video_stream_encoder_->TriggerCpuOveruse();
5286 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5287 int adapted_framerate =
5288 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
5289 EXPECT_LT(adapted_framerate, kFramerate);
5290
5291 // Trigger underuse, max framerate should go back to codec configured fps.
5292 // Set extra low fps, to make sure it's actually reset, not just incremented.
5293 stats = stats_proxy_->GetStats();
5294 stats.input_frame_rate = adapted_framerate / 2;
5295 stats_proxy_->SetMockStats(stats);
Henrik Boström91aa7322020-04-28 12:24:33 +02005296 video_stream_encoder_->TriggerCpuUnderuse();
Niels Möller7dc26b72017-12-06 10:27:48 +01005297 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5298 EXPECT_EQ(
5299 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5300 kFramerate);
5301
5302 video_stream_encoder_->Stop();
5303}
5304
5305TEST_F(VideoStreamEncoderTest,
5306 OveruseDetectorUpdatedRespectsFramerateAfterUnderuse) {
5307 const int kFrameWidth = 1280;
5308 const int kFrameHeight = 720;
5309 const int kLowFramerate = 15;
5310 const int kHighFramerate = 25;
5311
Henrik Boström381d1092020-05-12 18:49:07 +02005312 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005313 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller7dc26b72017-12-06 10:27:48 +01005314 test::FrameForwarder source;
5315 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005316 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 10:27:48 +01005317
5318 // Trigger initial configuration.
5319 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +02005320 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5321 video_encoder_config.simulcast_layers[0].max_framerate = kLowFramerate;
Asa Persson606d3cb2021-10-04 10:07:11 +02005322 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
Niels Möller7dc26b72017-12-06 10:27:48 +01005323 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
Åsa Persson17107062020-10-08 08:57:51 +02005324 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
Niels Möllerf1338562018-04-26 09:51:47 +02005325 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01005326 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5327
5328 EXPECT_EQ(
5329 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5330 kLowFramerate);
5331
5332 // Trigger overuse, max framerate should be reduced.
5333 VideoSendStream::Stats stats = stats_proxy_->GetStats();
5334 stats.input_frame_rate = kLowFramerate;
5335 stats_proxy_->SetMockStats(stats);
5336 video_stream_encoder_->TriggerCpuOveruse();
5337 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5338 int adapted_framerate =
5339 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
5340 EXPECT_LT(adapted_framerate, kLowFramerate);
5341
5342 // Reconfigure the encoder with a new (higher max framerate), max fps should
5343 // still respect the adaptation.
Åsa Persson17107062020-10-08 08:57:51 +02005344 video_encoder_config.simulcast_layers[0].max_framerate = kHighFramerate;
Niels Möller7dc26b72017-12-06 10:27:48 +01005345 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
5346 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02005347 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01005348 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5349
5350 EXPECT_EQ(
5351 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5352 adapted_framerate);
5353
5354 // Trigger underuse, max framerate should go back to codec configured fps.
5355 stats = stats_proxy_->GetStats();
5356 stats.input_frame_rate = adapted_framerate;
5357 stats_proxy_->SetMockStats(stats);
Henrik Boström91aa7322020-04-28 12:24:33 +02005358 video_stream_encoder_->TriggerCpuUnderuse();
Niels Möller7dc26b72017-12-06 10:27:48 +01005359 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5360 EXPECT_EQ(
5361 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5362 kHighFramerate);
5363
5364 video_stream_encoder_->Stop();
5365}
5366
mflodmancc3d4422017-08-03 08:27:51 -07005367TEST_F(VideoStreamEncoderTest,
5368 OveruseDetectorUpdatedOnDegradationPreferenceChange) {
sprangfda496a2017-06-15 04:21:07 -07005369 const int kFrameWidth = 1280;
5370 const int kFrameHeight = 720;
5371 const int kFramerate = 24;
5372
Henrik Boström381d1092020-05-12 18:49:07 +02005373 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005374 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprangfda496a2017-06-15 04:21:07 -07005375 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07005376 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005377 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangfda496a2017-06-15 04:21:07 -07005378
5379 // Trigger initial configuration.
5380 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +02005381 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5382 video_encoder_config.simulcast_layers[0].max_framerate = kFramerate;
Asa Persson606d3cb2021-10-04 10:07:11 +02005383 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
sprangfda496a2017-06-15 04:21:07 -07005384 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
mflodmancc3d4422017-08-03 08:27:51 -07005385 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02005386 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -07005387 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprangfda496a2017-06-15 04:21:07 -07005388
Niels Möller7dc26b72017-12-06 10:27:48 +01005389 EXPECT_EQ(
5390 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5391 kFramerate);
sprangfda496a2017-06-15 04:21:07 -07005392
5393 // Trigger overuse, max framerate should be reduced.
5394 VideoSendStream::Stats stats = stats_proxy_->GetStats();
5395 stats.input_frame_rate = kFramerate;
5396 stats_proxy_->SetMockStats(stats);
mflodmancc3d4422017-08-03 08:27:51 -07005397 video_stream_encoder_->TriggerCpuOveruse();
5398 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Niels Möller7dc26b72017-12-06 10:27:48 +01005399 int adapted_framerate =
5400 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
sprangfda496a2017-06-15 04:21:07 -07005401 EXPECT_LT(adapted_framerate, kFramerate);
5402
5403 // Change degradation preference to not enable framerate scaling. Target
5404 // framerate should be changed to codec defined limit.
Henrik Boström381d1092020-05-12 18:49:07 +02005405 video_stream_encoder_->SetSourceAndWaitForFramerateUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005406 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Niels Möller7dc26b72017-12-06 10:27:48 +01005407 EXPECT_EQ(
5408 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5409 kFramerate);
sprangfda496a2017-06-15 04:21:07 -07005410
mflodmancc3d4422017-08-03 08:27:51 -07005411 video_stream_encoder_->Stop();
sprangfda496a2017-06-15 04:21:07 -07005412}
5413
mflodmancc3d4422017-08-03 08:27:51 -07005414TEST_F(VideoStreamEncoderTest, DropsFramesAndScalesWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 05:51:49 -07005415 const int kTooLowBitrateForFrameSizeBps = 10000;
Henrik Boström381d1092020-05-12 18:49:07 +02005416 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005417 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5418 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5419 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07005420 const int kWidth = 640;
5421 const int kHeight = 360;
kthelgason2bc68642017-02-07 07:02:22 -08005422
asaperssonfab67072017-04-04 05:51:49 -07005423 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 07:02:22 -08005424
5425 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 07:06:52 -07005426 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08005427
5428 // Expect the sink_wants to specify a scaled frame.
Henrik Boström2671dac2020-05-19 16:29:09 +02005429 EXPECT_TRUE_WAIT(
5430 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
kthelgason2bc68642017-02-07 07:02:22 -08005431
sprangc5d62e22017-04-02 23:53:04 -07005432 int last_pixel_count = video_source_.sink_wants().max_pixel_count;
kthelgason2bc68642017-02-07 07:02:22 -08005433
asaperssonfab67072017-04-04 05:51:49 -07005434 // Next frame is scaled.
kthelgason2bc68642017-02-07 07:02:22 -08005435 video_source_.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07005436 CreateFrame(2, kWidth * 3 / 4, kHeight * 3 / 4));
kthelgason2bc68642017-02-07 07:02:22 -08005437
5438 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 07:06:52 -07005439 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08005440
Henrik Boström2671dac2020-05-19 16:29:09 +02005441 EXPECT_TRUE_WAIT(
5442 video_source_.sink_wants().max_pixel_count < last_pixel_count, 5000);
kthelgason2bc68642017-02-07 07:02:22 -08005443
mflodmancc3d4422017-08-03 08:27:51 -07005444 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08005445}
5446
mflodmancc3d4422017-08-03 08:27:51 -07005447TEST_F(VideoStreamEncoderTest,
5448 NumberOfDroppedFramesLimitedWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 05:51:49 -07005449 const int kTooLowBitrateForFrameSizeBps = 10000;
Henrik Boström381d1092020-05-12 18:49:07 +02005450 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005451 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5452 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5453 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07005454 const int kWidth = 640;
5455 const int kHeight = 360;
kthelgason2bc68642017-02-07 07:02:22 -08005456
5457 // We expect the n initial frames to get dropped.
5458 int i;
5459 for (i = 1; i <= kMaxInitialFramedrop; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07005460 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005461 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08005462 }
5463 // The n+1th frame should not be dropped, even though it's size is too large.
asaperssonfab67072017-04-04 05:51:49 -07005464 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005465 WaitForEncodedFrame(i);
kthelgason2bc68642017-02-07 07:02:22 -08005466
5467 // Expect the sink_wants to specify a scaled frame.
asaperssonfab67072017-04-04 05:51:49 -07005468 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason2bc68642017-02-07 07:02:22 -08005469
mflodmancc3d4422017-08-03 08:27:51 -07005470 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08005471}
5472
mflodmancc3d4422017-08-03 08:27:51 -07005473TEST_F(VideoStreamEncoderTest,
5474 InitialFrameDropOffWithMaintainResolutionPreference) {
asaperssonfab67072017-04-04 05:51:49 -07005475 const int kWidth = 640;
5476 const int kHeight = 360;
Henrik Boström381d1092020-05-12 18:49:07 +02005477 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005478 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
kthelgason2bc68642017-02-07 07:02:22 -08005479
5480 // Set degradation preference.
mflodmancc3d4422017-08-03 08:27:51 -07005481 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005482 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason2bc68642017-02-07 07:02:22 -08005483
asaperssonfab67072017-04-04 05:51:49 -07005484 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 07:02:22 -08005485 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 07:06:52 -07005486 WaitForEncodedFrame(1);
kthelgason2bc68642017-02-07 07:02:22 -08005487
mflodmancc3d4422017-08-03 08:27:51 -07005488 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08005489}
5490
mflodmancc3d4422017-08-03 08:27:51 -07005491TEST_F(VideoStreamEncoderTest, InitialFrameDropOffWhenEncoderDisabledScaling) {
asaperssonfab67072017-04-04 05:51:49 -07005492 const int kWidth = 640;
5493 const int kHeight = 360;
kthelgasonad9010c2017-02-14 00:46:51 -08005494 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 09:04:13 +02005495
5496 VideoEncoderConfig video_encoder_config;
5497 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5498 // Make format different, to force recreation of encoder.
5499 video_encoder_config.video_format.parameters["foo"] = "foo";
5500 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02005501 kMaxPayloadLength);
Henrik Boström381d1092020-05-12 18:49:07 +02005502 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005503 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07005504
kthelgasonb83797b2017-02-14 11:57:25 -08005505 // Force quality scaler reconfiguration by resetting the source.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005506 video_stream_encoder_->SetSource(&video_source_,
5507 webrtc::DegradationPreference::BALANCED);
kthelgasonad9010c2017-02-14 00:46:51 -08005508
asaperssonfab67072017-04-04 05:51:49 -07005509 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgasonad9010c2017-02-14 00:46:51 -08005510 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 07:06:52 -07005511 WaitForEncodedFrame(1);
kthelgasonad9010c2017-02-14 00:46:51 -08005512
mflodmancc3d4422017-08-03 08:27:51 -07005513 video_stream_encoder_->Stop();
kthelgasonad9010c2017-02-14 00:46:51 -08005514 fake_encoder_.SetQualityScaling(true);
5515}
5516
Åsa Persson139f4dc2019-08-02 09:29:58 +02005517TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenBweDrops) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01005518 webrtc::test::ScopedKeyValueConfig field_trials(
5519 field_trials_,
Åsa Persson139f4dc2019-08-02 09:29:58 +02005520 "WebRTC-Video-QualityScalerSettings/"
5521 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
5522 // Reset encoder for field trials to take effect.
5523 ConfigureEncoder(video_encoder_config_.Copy());
Asa Persson606d3cb2021-10-04 10:07:11 +02005524 const int kNotTooLowBitrateForFrameSizeBps = kTargetBitrate.bps() * 0.2;
5525 const int kTooLowBitrateForFrameSizeBps = kTargetBitrate.bps() * 0.19;
Åsa Persson139f4dc2019-08-02 09:29:58 +02005526 const int kWidth = 640;
5527 const int kHeight = 360;
5528
Henrik Boström381d1092020-05-12 18:49:07 +02005529 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005530 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005531 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5532 // Frame should not be dropped.
5533 WaitForEncodedFrame(1);
5534
Henrik Boström381d1092020-05-12 18:49:07 +02005535 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005536 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5537 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5538 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005539 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5540 // Frame should not be dropped.
5541 WaitForEncodedFrame(2);
5542
Henrik Boström381d1092020-05-12 18:49:07 +02005543 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005544 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5545 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5546 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005547 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5548 // Expect to drop this frame, the wait should time out.
5549 ExpectDroppedFrame();
5550
5551 // Expect the sink_wants to specify a scaled frame.
Henrik Boström2671dac2020-05-19 16:29:09 +02005552 EXPECT_TRUE_WAIT(
5553 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005554 video_stream_encoder_->Stop();
5555}
5556
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02005557TEST_F(VideoStreamEncoderTest,
5558 InitialFrameDropNotReactivatedWhenBweDropsWhenScalingDisabled) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01005559 webrtc::test::ScopedKeyValueConfig field_trials(
5560 field_trials_,
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02005561 "WebRTC-Video-QualityScalerSettings/"
5562 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
5563 fake_encoder_.SetQualityScaling(false);
5564 ConfigureEncoder(video_encoder_config_.Copy());
Asa Persson606d3cb2021-10-04 10:07:11 +02005565 const int kNotTooLowBitrateForFrameSizeBps = kTargetBitrate.bps() * 0.2;
5566 const int kTooLowBitrateForFrameSizeBps = kTargetBitrate.bps() * 0.19;
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02005567 const int kWidth = 640;
5568 const int kHeight = 360;
5569
5570 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005571 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02005572 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5573 // Frame should not be dropped.
5574 WaitForEncodedFrame(1);
5575
5576 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5577 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5578 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5579 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps), 0, 0, 0);
5580 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5581 // Frame should not be dropped.
5582 WaitForEncodedFrame(2);
5583
5584 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5585 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5586 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5587 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
5588 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5589 // Not dropped since quality scaling is disabled.
5590 WaitForEncodedFrame(3);
5591
5592 // Expect the sink_wants to specify a scaled frame.
Evan Shrubsole85728412020-08-25 13:12:12 +02005593 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02005594 EXPECT_THAT(video_source_.sink_wants(), ResolutionMax());
5595
5596 video_stream_encoder_->Stop();
5597}
5598
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005599TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenLayersChange) {
Asa Persson606d3cb2021-10-04 10:07:11 +02005600 const DataRate kLowTargetBitrate = DataRate::KilobitsPerSec(400);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005601 // Set simulcast.
5602 ResetEncoder("VP8", 3, 1, 1, false);
5603 fake_encoder_.SetQualityScaling(true);
5604 const int kWidth = 1280;
5605 const int kHeight = 720;
5606 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005607 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005608 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5609 // Frame should not be dropped.
5610 WaitForEncodedFrame(1);
5611
5612 // Trigger QVGA "singlecast"
5613 // Update the config.
5614 VideoEncoderConfig video_encoder_config;
5615 test::FillEncoderConfiguration(PayloadStringToCodecType("VP8"), 3,
5616 &video_encoder_config);
Åsa Persson7f354f82021-02-04 15:52:15 +01005617 video_encoder_config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005618 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Persson7f354f82021-02-04 15:52:15 +01005619 "VP8", /*max qp*/ 56, /*screencast*/ false,
5620 /*screenshare enabled*/ false);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005621 for (auto& layer : video_encoder_config.simulcast_layers) {
5622 layer.num_temporal_layers = 1;
5623 layer.max_framerate = kDefaultFramerate;
5624 }
Asa Persson606d3cb2021-10-04 10:07:11 +02005625 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005626 video_encoder_config.content_type =
5627 VideoEncoderConfig::ContentType::kRealtimeVideo;
5628
5629 video_encoder_config.simulcast_layers[0].active = true;
5630 video_encoder_config.simulcast_layers[1].active = false;
5631 video_encoder_config.simulcast_layers[2].active = false;
5632
5633 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5634 kMaxPayloadLength);
5635 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5636
5637 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5638 // Frame should not be dropped.
5639 WaitForEncodedFrame(2);
5640
5641 // Trigger HD "singlecast"
5642 video_encoder_config.simulcast_layers[0].active = false;
5643 video_encoder_config.simulcast_layers[1].active = false;
5644 video_encoder_config.simulcast_layers[2].active = true;
5645
5646 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5647 kMaxPayloadLength);
5648 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5649
5650 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5651 // Frame should be dropped because of initial frame drop.
5652 ExpectDroppedFrame();
5653
5654 // Expect the sink_wants to specify a scaled frame.
5655 EXPECT_TRUE_WAIT(
5656 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
5657 video_stream_encoder_->Stop();
5658}
5659
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005660TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenSVCLayersChange) {
Asa Persson606d3cb2021-10-04 10:07:11 +02005661 const DataRate kLowTargetBitrate = DataRate::KilobitsPerSec(400);
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005662 // Set simulcast.
5663 ResetEncoder("VP9", 1, 1, 3, false);
5664 fake_encoder_.SetQualityScaling(true);
5665 const int kWidth = 1280;
5666 const int kHeight = 720;
5667 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005668 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005669 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5670 // Frame should not be dropped.
5671 WaitForEncodedFrame(1);
5672
5673 // Trigger QVGA "singlecast"
5674 // Update the config.
5675 VideoEncoderConfig video_encoder_config;
5676 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5677 &video_encoder_config);
5678 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5679 vp9_settings.numberOfSpatialLayers = 3;
5680 // Since only one layer is active - automatic resize should be enabled.
5681 vp9_settings.automaticResizeOn = true;
5682 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005683 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005684 vp9_settings);
Asa Persson606d3cb2021-10-04 10:07:11 +02005685 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005686 video_encoder_config.content_type =
5687 VideoEncoderConfig::ContentType::kRealtimeVideo;
Artem Titovab30d722021-07-27 16:22:11 +02005688 // Currently simulcast layers `active` flags are used to inidicate
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005689 // which SVC layers are active.
5690 video_encoder_config.simulcast_layers.resize(3);
5691
5692 video_encoder_config.simulcast_layers[0].active = true;
5693 video_encoder_config.simulcast_layers[1].active = false;
5694 video_encoder_config.simulcast_layers[2].active = false;
5695
5696 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5697 kMaxPayloadLength);
5698 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5699
5700 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5701 // Frame should not be dropped.
5702 WaitForEncodedFrame(2);
5703
5704 // Trigger HD "singlecast"
5705 video_encoder_config.simulcast_layers[0].active = false;
5706 video_encoder_config.simulcast_layers[1].active = false;
5707 video_encoder_config.simulcast_layers[2].active = true;
5708
5709 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5710 kMaxPayloadLength);
5711 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5712
5713 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5714 // Frame should be dropped because of initial frame drop.
5715 ExpectDroppedFrame();
5716
5717 // Expect the sink_wants to specify a scaled frame.
5718 EXPECT_TRUE_WAIT(
5719 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
5720 video_stream_encoder_->Stop();
5721}
5722
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005723TEST_F(VideoStreamEncoderTest,
Åsa Perssonc91c4232021-02-01 09:20:05 +01005724 EncoderMaxAndMinBitratesUsedIfMiddleStreamActive) {
5725 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
5726 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
5727 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
5728 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
5729 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
5730 1280 * 720, 54 * 1000, 31 * 1000, 2500 * 1000);
5731 fake_encoder_.SetResolutionBitrateLimits(
5732 {kEncoderLimits270p, kEncoderLimits360p, kEncoderLimits720p});
5733
5734 VideoEncoderConfig video_encoder_config;
5735 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5736 &video_encoder_config);
5737 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5738 vp9_settings.numberOfSpatialLayers = 3;
5739 // Since only one layer is active - automatic resize should be enabled.
5740 vp9_settings.automaticResizeOn = true;
5741 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005742 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Åsa Perssonc91c4232021-02-01 09:20:05 +01005743 vp9_settings);
Asa Persson606d3cb2021-10-04 10:07:11 +02005744 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Åsa Perssonc91c4232021-02-01 09:20:05 +01005745 video_encoder_config.content_type =
5746 VideoEncoderConfig::ContentType::kRealtimeVideo;
5747 // Simulcast layers are used to indicate which spatial layers are active.
5748 video_encoder_config.simulcast_layers.resize(3);
5749 video_encoder_config.simulcast_layers[0].active = false;
5750 video_encoder_config.simulcast_layers[1].active = true;
5751 video_encoder_config.simulcast_layers[2].active = false;
5752
5753 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5754 kMaxPayloadLength);
5755 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5756
5757 // The encoder bitrate limits for 360p should be used.
5758 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5759 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02005760 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5761 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5762 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
5763 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5764 EXPECT_EQ(640, fake_encoder_.config().spatialLayers[0].width);
5765 EXPECT_EQ(360, fake_encoder_.config().spatialLayers[0].height);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005766 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005767 fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005768 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005769 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005770
5771 // The encoder bitrate limits for 270p should be used.
5772 video_source_.IncomingCapturedFrame(CreateFrame(2, 960, 540));
5773 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02005774 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5775 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5776 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
5777 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5778 EXPECT_EQ(480, fake_encoder_.config().spatialLayers[0].width);
5779 EXPECT_EQ(270, fake_encoder_.config().spatialLayers[0].height);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005780 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005781 fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005782 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005783 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005784
5785 video_stream_encoder_->Stop();
5786}
5787
5788TEST_F(VideoStreamEncoderTest,
Åsa Persson258e9892021-02-25 10:39:51 +01005789 DefaultMaxAndMinBitratesUsedIfMiddleStreamActive) {
5790 VideoEncoderConfig video_encoder_config;
5791 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5792 &video_encoder_config);
5793 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5794 vp9_settings.numberOfSpatialLayers = 3;
5795 // Since only one layer is active - automatic resize should be enabled.
5796 vp9_settings.automaticResizeOn = true;
5797 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005798 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Åsa Persson258e9892021-02-25 10:39:51 +01005799 vp9_settings);
Asa Persson606d3cb2021-10-04 10:07:11 +02005800 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Åsa Persson258e9892021-02-25 10:39:51 +01005801 video_encoder_config.content_type =
5802 VideoEncoderConfig::ContentType::kRealtimeVideo;
5803 // Simulcast layers are used to indicate which spatial layers are active.
5804 video_encoder_config.simulcast_layers.resize(3);
5805 video_encoder_config.simulcast_layers[0].active = false;
5806 video_encoder_config.simulcast_layers[1].active = true;
5807 video_encoder_config.simulcast_layers[2].active = false;
5808
5809 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5810 kMaxPayloadLength);
5811 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5812
5813 // The default bitrate limits for 360p should be used.
5814 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits360p =
Sergey Silkina86b29b2021-03-05 13:29:19 +01005815 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5816 kVideoCodecVP9, 640 * 360);
Åsa Persson258e9892021-02-25 10:39:51 +01005817 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5818 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02005819 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5820 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5821 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
5822 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5823 EXPECT_EQ(640, fake_encoder_.config().spatialLayers[0].width);
5824 EXPECT_EQ(360, fake_encoder_.config().spatialLayers[0].height);
Åsa Persson258e9892021-02-25 10:39:51 +01005825 EXPECT_EQ(static_cast<uint32_t>(kLimits360p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005826 fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005827 EXPECT_EQ(static_cast<uint32_t>(kLimits360p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005828 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005829
5830 // The default bitrate limits for 270p should be used.
5831 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits270p =
Sergey Silkina86b29b2021-03-05 13:29:19 +01005832 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5833 kVideoCodecVP9, 480 * 270);
Åsa Persson258e9892021-02-25 10:39:51 +01005834 video_source_.IncomingCapturedFrame(CreateFrame(2, 960, 540));
5835 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02005836 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5837 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5838 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
5839 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5840 EXPECT_EQ(480, fake_encoder_.config().spatialLayers[0].width);
5841 EXPECT_EQ(270, fake_encoder_.config().spatialLayers[0].height);
Åsa Persson258e9892021-02-25 10:39:51 +01005842 EXPECT_EQ(static_cast<uint32_t>(kLimits270p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005843 fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005844 EXPECT_EQ(static_cast<uint32_t>(kLimits270p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005845 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005846
5847 video_stream_encoder_->Stop();
5848}
5849
5850TEST_F(VideoStreamEncoderTest, DefaultMaxAndMinBitratesNotUsedIfDisabled) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01005851 webrtc::test::ScopedKeyValueConfig field_trials(
5852 field_trials_, "WebRTC-DefaultBitrateLimitsKillSwitch/Enabled/");
Åsa Persson258e9892021-02-25 10:39:51 +01005853 VideoEncoderConfig video_encoder_config;
5854 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5855 &video_encoder_config);
5856 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5857 vp9_settings.numberOfSpatialLayers = 3;
5858 // Since only one layer is active - automatic resize should be enabled.
5859 vp9_settings.automaticResizeOn = true;
5860 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005861 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Åsa Persson258e9892021-02-25 10:39:51 +01005862 vp9_settings);
Asa Persson606d3cb2021-10-04 10:07:11 +02005863 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Åsa Persson258e9892021-02-25 10:39:51 +01005864 video_encoder_config.content_type =
5865 VideoEncoderConfig::ContentType::kRealtimeVideo;
5866 // Simulcast layers are used to indicate which spatial layers are active.
5867 video_encoder_config.simulcast_layers.resize(3);
5868 video_encoder_config.simulcast_layers[0].active = false;
5869 video_encoder_config.simulcast_layers[1].active = true;
5870 video_encoder_config.simulcast_layers[2].active = false;
5871
5872 // Reset encoder for field trials to take effect.
5873 ConfigureEncoder(video_encoder_config.Copy());
5874
5875 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5876 kMaxPayloadLength);
5877 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5878
5879 // The default bitrate limits for 360p should not be used.
5880 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits360p =
Sergey Silkina86b29b2021-03-05 13:29:19 +01005881 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5882 kVideoCodecVP9, 640 * 360);
Åsa Persson258e9892021-02-25 10:39:51 +01005883 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5884 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02005885 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5886 EXPECT_EQ(fake_encoder_.config().codecType, kVideoCodecVP9);
5887 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
5888 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5889 EXPECT_EQ(640, fake_encoder_.config().spatialLayers[0].width);
5890 EXPECT_EQ(360, fake_encoder_.config().spatialLayers[0].height);
Åsa Persson258e9892021-02-25 10:39:51 +01005891 EXPECT_NE(static_cast<uint32_t>(kLimits360p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005892 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005893
5894 video_stream_encoder_->Stop();
5895}
5896
5897TEST_F(VideoStreamEncoderTest, SinglecastBitrateLimitsNotUsedForOneStream) {
5898 ResetEncoder("VP9", /*num_streams=*/1, /*num_temporal_layers=*/1,
5899 /*num_spatial_layers=*/1, /*screenshare=*/false);
5900
5901 // The default singlecast bitrate limits for 720p should not be used.
5902 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits720p =
Sergey Silkina86b29b2021-03-05 13:29:19 +01005903 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5904 kVideoCodecVP9, 1280 * 720);
Åsa Persson258e9892021-02-25 10:39:51 +01005905 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5906 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02005907 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5908 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5909 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 1);
5910 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5911 EXPECT_EQ(1280, fake_encoder_.config().spatialLayers[0].width);
5912 EXPECT_EQ(720, fake_encoder_.config().spatialLayers[0].height);
Åsa Persson258e9892021-02-25 10:39:51 +01005913 EXPECT_NE(static_cast<uint32_t>(kLimits720p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005914 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005915
5916 video_stream_encoder_->Stop();
5917}
5918
5919TEST_F(VideoStreamEncoderTest,
Åsa Perssonc91c4232021-02-01 09:20:05 +01005920 EncoderMaxAndMinBitratesNotUsedIfLowestStreamActive) {
5921 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits180p(
5922 320 * 180, 34 * 1000, 12 * 1000, 1234 * 1000);
5923 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
5924 1280 * 720, 54 * 1000, 31 * 1000, 2500 * 1000);
5925 fake_encoder_.SetResolutionBitrateLimits(
5926 {kEncoderLimits180p, kEncoderLimits720p});
5927
5928 VideoEncoderConfig video_encoder_config;
5929 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5930 &video_encoder_config);
5931 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5932 vp9_settings.numberOfSpatialLayers = 3;
5933 // Since only one layer is active - automatic resize should be enabled.
5934 vp9_settings.automaticResizeOn = true;
5935 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005936 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Åsa Perssonc91c4232021-02-01 09:20:05 +01005937 vp9_settings);
Asa Persson606d3cb2021-10-04 10:07:11 +02005938 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Åsa Perssonc91c4232021-02-01 09:20:05 +01005939 video_encoder_config.content_type =
5940 VideoEncoderConfig::ContentType::kRealtimeVideo;
5941 // Simulcast layers are used to indicate which spatial layers are active.
5942 video_encoder_config.simulcast_layers.resize(3);
5943 video_encoder_config.simulcast_layers[0].active = true;
5944 video_encoder_config.simulcast_layers[1].active = false;
5945 video_encoder_config.simulcast_layers[2].active = false;
5946
5947 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5948 kMaxPayloadLength);
5949 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5950
5951 // Limits not applied on lowest stream, limits for 180p should not be used.
5952 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5953 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02005954 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5955 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5956 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 3);
5957 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5958 EXPECT_EQ(320, fake_encoder_.config().spatialLayers[0].width);
5959 EXPECT_EQ(180, fake_encoder_.config().spatialLayers[0].height);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005960 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits180p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005961 fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005962 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits180p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005963 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005964
5965 video_stream_encoder_->Stop();
5966}
5967
5968TEST_F(VideoStreamEncoderTest,
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005969 InitialFrameDropActivatesWhenResolutionIncreases) {
5970 const int kWidth = 640;
5971 const int kHeight = 360;
5972
5973 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005974 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005975 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth / 2, kHeight / 2));
5976 // Frame should not be dropped.
5977 WaitForEncodedFrame(1);
5978
5979 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005980 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005981 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth / 2, kHeight / 2));
5982 // Frame should not be dropped, bitrate not too low for frame.
5983 WaitForEncodedFrame(2);
5984
5985 // Incoming resolution increases.
5986 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5987 // Expect to drop this frame, bitrate too low for frame.
5988 ExpectDroppedFrame();
5989
5990 // Expect the sink_wants to specify a scaled frame.
5991 EXPECT_TRUE_WAIT(
5992 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
5993 video_stream_encoder_->Stop();
5994}
5995
5996TEST_F(VideoStreamEncoderTest, InitialFrameDropIsNotReactivatedWhenAdaptingUp) {
5997 const int kWidth = 640;
5998 const int kHeight = 360;
5999 // So that quality scaling doesn't happen by itself.
6000 fake_encoder_.SetQp(kQpHigh);
6001
6002 AdaptingFrameForwarder source(&time_controller_);
6003 source.set_adaptation_enabled(true);
6004 video_stream_encoder_->SetSource(
6005 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
6006
6007 int timestamp = 1;
6008
6009 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006010 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01006011 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
6012 WaitForEncodedFrame(timestamp);
6013 timestamp += 9000;
6014 // Long pause to disable all first BWE drop logic.
6015 AdvanceTime(TimeDelta::Millis(1000));
6016
6017 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006018 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01006019 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
6020 // Not dropped frame, as initial frame drop is disabled by now.
6021 WaitForEncodedFrame(timestamp);
6022 timestamp += 9000;
6023 AdvanceTime(TimeDelta::Millis(100));
6024
6025 // Quality adaptation down.
6026 video_stream_encoder_->TriggerQualityLow();
6027
6028 // Adaptation has an effect.
6029 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < kWidth * kHeight,
6030 5000);
6031
6032 // Frame isn't dropped as initial frame dropper is disabled.
6033 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
6034 WaitForEncodedFrame(timestamp);
6035 timestamp += 9000;
6036 AdvanceTime(TimeDelta::Millis(100));
6037
6038 // Quality adaptation up.
6039 video_stream_encoder_->TriggerQualityHigh();
6040
6041 // Adaptation has an effect.
6042 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count > kWidth * kHeight,
6043 5000);
6044
6045 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
6046 // Frame should not be dropped, as initial framedropper is off.
6047 WaitForEncodedFrame(timestamp);
6048
6049 video_stream_encoder_->Stop();
6050}
6051
Åsa Persson7f354f82021-02-04 15:52:15 +01006052TEST_F(VideoStreamEncoderTest,
6053 FrameDroppedWhenResolutionIncreasesAndLinkAllocationIsLow) {
6054 const int kMinStartBps360p = 222000;
6055 fake_encoder_.SetResolutionBitrateLimits(
6056 {VideoEncoder::ResolutionBitrateLimits(320 * 180, 0, 30000, 400000),
6057 VideoEncoder::ResolutionBitrateLimits(640 * 360, kMinStartBps360p, 30000,
6058 800000)});
6059
6060 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
6061 DataRate::BitsPerSec(kMinStartBps360p - 1), // target_bitrate
6062 DataRate::BitsPerSec(kMinStartBps360p - 1), // stable_target_bitrate
6063 DataRate::BitsPerSec(kMinStartBps360p - 1), // link_allocation
6064 0, 0, 0);
6065 // Frame should not be dropped, bitrate not too low for frame.
6066 video_source_.IncomingCapturedFrame(CreateFrame(1, 320, 180));
6067 WaitForEncodedFrame(1);
6068
6069 // Incoming resolution increases, initial frame drop activates.
6070 // Frame should be dropped, link allocation too low for frame.
6071 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
6072 ExpectDroppedFrame();
6073
6074 // Expect sink_wants to specify a scaled frame.
6075 EXPECT_TRUE_WAIT(video_source_.sink_wants().max_pixel_count < 640 * 360,
6076 5000);
6077 video_stream_encoder_->Stop();
6078}
6079
6080TEST_F(VideoStreamEncoderTest,
6081 FrameNotDroppedWhenResolutionIncreasesAndLinkAllocationIsHigh) {
6082 const int kMinStartBps360p = 222000;
6083 fake_encoder_.SetResolutionBitrateLimits(
6084 {VideoEncoder::ResolutionBitrateLimits(320 * 180, 0, 30000, 400000),
6085 VideoEncoder::ResolutionBitrateLimits(640 * 360, kMinStartBps360p, 30000,
6086 800000)});
6087
6088 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
6089 DataRate::BitsPerSec(kMinStartBps360p - 1), // target_bitrate
6090 DataRate::BitsPerSec(kMinStartBps360p - 1), // stable_target_bitrate
6091 DataRate::BitsPerSec(kMinStartBps360p), // link_allocation
6092 0, 0, 0);
6093 // Frame should not be dropped, bitrate not too low for frame.
6094 video_source_.IncomingCapturedFrame(CreateFrame(1, 320, 180));
6095 WaitForEncodedFrame(1);
6096
6097 // Incoming resolution increases, initial frame drop activates.
6098 // Frame should be dropped, link allocation not too low for frame.
6099 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
6100 WaitForEncodedFrame(2);
6101
6102 video_stream_encoder_->Stop();
6103}
6104
Åsa Perssone644a032019-11-08 15:56:00 +01006105TEST_F(VideoStreamEncoderTest, RampsUpInQualityWhenBwIsHigh) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01006106 webrtc::test::ScopedKeyValueConfig field_trials(
6107 field_trials_,
Åsa Persson06defc42021-09-10 15:28:48 +02006108 "WebRTC-Video-QualityRampupSettings/"
6109 "min_pixels:921600,min_duration_ms:2000/");
6110
6111 const int kWidth = 1280;
6112 const int kHeight = 720;
6113 const int kFps = 10;
6114 max_framerate_ = kFps;
Åsa Perssone644a032019-11-08 15:56:00 +01006115
6116 // Reset encoder for field trials to take effect.
6117 VideoEncoderConfig config = video_encoder_config_.Copy();
Asa Persson606d3cb2021-10-04 10:07:11 +02006118 config.max_bitrate_bps = kTargetBitrate.bps();
Evan Shrubsoledff79252020-04-16 11:34:32 +02006119 DataRate max_bitrate = DataRate::BitsPerSec(config.max_bitrate_bps);
Åsa Perssone644a032019-11-08 15:56:00 +01006120 ConfigureEncoder(std::move(config));
6121 fake_encoder_.SetQp(kQpLow);
6122
6123 // Enable MAINTAIN_FRAMERATE preference.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006124 AdaptingFrameForwarder source(&time_controller_);
Åsa Perssone644a032019-11-08 15:56:00 +01006125 source.set_adaptation_enabled(true);
6126 video_stream_encoder_->SetSource(&source,
6127 DegradationPreference::MAINTAIN_FRAMERATE);
6128
6129 // Start at low bitrate.
Asa Persson606d3cb2021-10-04 10:07:11 +02006130 const DataRate kLowBitrate = DataRate::KilobitsPerSec(200);
Henrik Boström381d1092020-05-12 18:49:07 +02006131 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006132 kLowBitrate, kLowBitrate, kLowBitrate, 0, 0, 0);
Åsa Perssone644a032019-11-08 15:56:00 +01006133
6134 // Expect first frame to be dropped and resolution to be limited.
Åsa Persson06defc42021-09-10 15:28:48 +02006135 const int64_t kFrameIntervalMs = 1000 / kFps;
Åsa Perssone644a032019-11-08 15:56:00 +01006136 int64_t timestamp_ms = kFrameIntervalMs;
6137 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6138 ExpectDroppedFrame();
Henrik Boström2671dac2020-05-19 16:29:09 +02006139 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < kWidth * kHeight,
6140 5000);
Åsa Perssone644a032019-11-08 15:56:00 +01006141
6142 // Increase bitrate to encoder max.
Henrik Boström381d1092020-05-12 18:49:07 +02006143 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
6144 max_bitrate, max_bitrate, max_bitrate, 0, 0, 0);
Åsa Perssone644a032019-11-08 15:56:00 +01006145
Artem Titovab30d722021-07-27 16:22:11 +02006146 // Insert frames and advance `min_duration_ms`.
Åsa Persson06defc42021-09-10 15:28:48 +02006147 const int64_t start_bw_high_ms = CurrentTimeMs();
Åsa Perssone644a032019-11-08 15:56:00 +01006148 for (size_t i = 1; i <= 10; i++) {
6149 timestamp_ms += kFrameIntervalMs;
6150 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6151 WaitForEncodedFrame(timestamp_ms);
6152 }
Åsa Persson06defc42021-09-10 15:28:48 +02006153
6154 // Advance to `min_duration_ms` - 1, frame should not trigger high BW.
6155 int64_t elapsed_bw_high_ms = CurrentTimeMs() - start_bw_high_ms;
6156 AdvanceTime(TimeDelta::Millis(2000 - elapsed_bw_high_ms - 1));
6157 timestamp_ms += kFrameIntervalMs;
6158 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6159 WaitForEncodedFrame(timestamp_ms);
Åsa Perssone644a032019-11-08 15:56:00 +01006160 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6161 EXPECT_LT(source.sink_wants().max_pixel_count, kWidth * kHeight);
6162
Åsa Persson06defc42021-09-10 15:28:48 +02006163 // Frame should trigger high BW and release quality limitation.
Åsa Perssone644a032019-11-08 15:56:00 +01006164 timestamp_ms += kFrameIntervalMs;
6165 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6166 WaitForEncodedFrame(timestamp_ms);
Henrik Boström381d1092020-05-12 18:49:07 +02006167 // The ramp-up code involves the adaptation queue, give it time to execute.
6168 // TODO(hbos): Can we await an appropriate event instead?
Evan Shrubsole85728412020-08-25 13:12:12 +02006169 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006170 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
Åsa Perssone644a032019-11-08 15:56:00 +01006171
6172 // Frame should not be adapted.
6173 timestamp_ms += kFrameIntervalMs;
6174 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6175 WaitForEncodedFrame(kWidth, kHeight);
6176 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6177
6178 video_stream_encoder_->Stop();
6179}
6180
mflodmancc3d4422017-08-03 08:27:51 -07006181TEST_F(VideoStreamEncoderTest,
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02006182 QualityScalerAdaptationsRemovedWhenQualityScalingDisabled) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01006183 webrtc::test::ScopedKeyValueConfig field_trials(
6184 field_trials_, "WebRTC-Video-QualityScaling/Disabled/");
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006185 AdaptingFrameForwarder source(&time_controller_);
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02006186 source.set_adaptation_enabled(true);
6187 video_stream_encoder_->SetSource(&source,
6188 DegradationPreference::MAINTAIN_FRAMERATE);
6189 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006190 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02006191 fake_encoder_.SetQp(kQpHigh + 1);
6192 const int kWidth = 1280;
6193 const int kHeight = 720;
6194 const int64_t kFrameIntervalMs = 100;
6195 int64_t timestamp_ms = kFrameIntervalMs;
6196 for (size_t i = 1; i <= 100; i++) {
6197 timestamp_ms += kFrameIntervalMs;
6198 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6199 WaitForEncodedFrame(timestamp_ms);
6200 }
6201 // Wait for QualityScaler, which will wait for 2000*2.5 ms until checking QP
6202 // for the first time.
6203 // TODO(eshr): We should avoid these waits by using threads with simulated
6204 // time.
6205 EXPECT_TRUE_WAIT(stats_proxy_->GetStats().bw_limited_resolution,
6206 2000 * 2.5 * 2);
6207 timestamp_ms += kFrameIntervalMs;
6208 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6209 WaitForEncodedFrame(timestamp_ms);
6210 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
6211 EXPECT_THAT(source.sink_wants(), WantsMaxPixels(Lt(kWidth * kHeight)));
6212 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6213
6214 // Disable Quality scaling by turning off scaler on the encoder and
6215 // reconfiguring.
6216 fake_encoder_.SetQualityScaling(false);
6217 video_stream_encoder_->ConfigureEncoder(video_encoder_config_.Copy(),
6218 kMaxPayloadLength);
6219 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Markus Handell28c71802021-11-08 10:11:55 +01006220 AdvanceTime(TimeDelta::Zero());
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02006221 // Since we turned off the quality scaler, the adaptations made by it are
6222 // removed.
6223 EXPECT_THAT(source.sink_wants(), ResolutionMax());
6224 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6225
6226 video_stream_encoder_->Stop();
6227}
6228
6229TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07006230 ResolutionNotAdaptedForTooSmallFrame_MaintainFramerateMode) {
6231 const int kTooSmallWidth = 10;
6232 const int kTooSmallHeight = 10;
Henrik Boström381d1092020-05-12 18:49:07 +02006233 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006234 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07006235
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006236 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asaperssond0de2952017-04-21 01:47:31 -07006237 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07006238 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006239 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006240 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
asaperssond0de2952017-04-21 01:47:31 -07006241 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6242
6243 // Trigger adapt down, too small frame, expect no change.
6244 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07006245 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 08:27:51 -07006246 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006247 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07006248 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6249 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6250
mflodmancc3d4422017-08-03 08:27:51 -07006251 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07006252}
6253
mflodmancc3d4422017-08-03 08:27:51 -07006254TEST_F(VideoStreamEncoderTest,
6255 ResolutionNotAdaptedForTooSmallFrame_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07006256 const int kTooSmallWidth = 10;
6257 const int kTooSmallHeight = 10;
6258 const int kFpsLimit = 7;
Henrik Boström381d1092020-05-12 18:49:07 +02006259 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006260 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07006261
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006262 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07006263 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006264 video_stream_encoder_->SetSource(&source,
6265 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006266 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
asaperssonf7e294d2017-06-13 23:25:22 -07006267 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6268 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6269
6270 // Trigger adapt down, expect limited framerate.
6271 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07006272 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 08:27:51 -07006273 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006274 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07006275 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6276 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6277 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6278
6279 // Trigger adapt down, too small frame, expect no change.
6280 source.IncomingCapturedFrame(CreateFrame(2, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07006281 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07006282 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006283 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07006284 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6285 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6286 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6287
mflodmancc3d4422017-08-03 08:27:51 -07006288 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006289}
6290
mflodmancc3d4422017-08-03 08:27:51 -07006291TEST_F(VideoStreamEncoderTest, FailingInitEncodeDoesntCauseCrash) {
asapersson02465b82017-04-10 01:12:52 -07006292 fake_encoder_.ForceInitEncodeFailure(true);
Henrik Boström381d1092020-05-12 18:49:07 +02006293 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006294 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möllerf1338562018-04-26 09:51:47 +02006295 ResetEncoder("VP8", 2, 1, 1, false);
asapersson02465b82017-04-10 01:12:52 -07006296 const int kFrameWidth = 1280;
6297 const int kFrameHeight = 720;
6298 video_source_.IncomingCapturedFrame(
6299 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006300 ExpectDroppedFrame();
mflodmancc3d4422017-08-03 08:27:51 -07006301 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07006302}
6303
sprangb1ca0732017-02-01 08:38:12 -08006304// TODO(sprang): Extend this with fps throttling and any "balanced" extensions.
mflodmancc3d4422017-08-03 08:27:51 -07006305TEST_F(VideoStreamEncoderTest,
6306 AdaptsResolutionOnOveruse_MaintainFramerateMode) {
Henrik Boström381d1092020-05-12 18:49:07 +02006307 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006308 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprangb1ca0732017-02-01 08:38:12 -08006309
6310 const int kFrameWidth = 1280;
6311 const int kFrameHeight = 720;
6312 // Enabled default VideoAdapter downscaling. First step is 3/4, not 3/5 as
mflodmancc3d4422017-08-03 08:27:51 -07006313 // requested by
6314 // VideoStreamEncoder::VideoSourceProxy::RequestResolutionLowerThan().
sprangb1ca0732017-02-01 08:38:12 -08006315 video_source_.set_adaptation_enabled(true);
6316
6317 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02006318 CreateFrame(1 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006319 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 08:38:12 -08006320
6321 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 08:27:51 -07006322 video_stream_encoder_->TriggerCpuOveruse();
sprangb1ca0732017-02-01 08:38:12 -08006323 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02006324 CreateFrame(2 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006325 WaitForEncodedFrame((kFrameWidth * 3) / 4, (kFrameHeight * 3) / 4);
sprangb1ca0732017-02-01 08:38:12 -08006326
asaperssonfab67072017-04-04 05:51:49 -07006327 // Trigger CPU normal use, return to original resolution.
Henrik Boström91aa7322020-04-28 12:24:33 +02006328 video_stream_encoder_->TriggerCpuUnderuse();
sprangb1ca0732017-02-01 08:38:12 -08006329 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02006330 CreateFrame(3 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006331 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 08:38:12 -08006332
mflodmancc3d4422017-08-03 08:27:51 -07006333 video_stream_encoder_->Stop();
sprangb1ca0732017-02-01 08:38:12 -08006334}
sprangfe627f32017-03-29 08:24:59 -07006335
mflodmancc3d4422017-08-03 08:27:51 -07006336TEST_F(VideoStreamEncoderTest,
6337 AdaptsFramerateOnOveruse_MaintainResolutionMode) {
sprangc5d62e22017-04-02 23:53:04 -07006338 const int kFrameWidth = 1280;
6339 const int kFrameHeight = 720;
sprangc5d62e22017-04-02 23:53:04 -07006340
Henrik Boström381d1092020-05-12 18:49:07 +02006341 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006342 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07006343 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006344 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07006345 video_source_.set_adaptation_enabled(true);
6346
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006347 int64_t timestamp_ms = CurrentTimeMs();
sprangc5d62e22017-04-02 23:53:04 -07006348
6349 video_source_.IncomingCapturedFrame(
6350 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006351 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-02 23:53:04 -07006352
6353 // Try to trigger overuse. No fps estimate available => no effect.
mflodmancc3d4422017-08-03 08:27:51 -07006354 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07006355
6356 // Insert frames for one second to get a stable estimate.
sprang4847ae62017-06-27 07:06:52 -07006357 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006358 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006359 video_source_.IncomingCapturedFrame(
6360 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006361 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-02 23:53:04 -07006362 }
6363
6364 // Trigger CPU overuse, reduce framerate by 2/3.
mflodmancc3d4422017-08-03 08:27:51 -07006365 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07006366 int num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07006367 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006368 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006369 video_source_.IncomingCapturedFrame(
6370 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006371 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07006372 ++num_frames_dropped;
6373 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01006374 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07006375 }
6376 }
6377
sprang4847ae62017-06-27 07:06:52 -07006378 // Add some slack to account for frames dropped by the frame dropper.
6379 const int kErrorMargin = 1;
6380 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-02 23:53:04 -07006381 kErrorMargin);
6382
6383 // Trigger CPU overuse, reduce framerate by 2/3 again.
mflodmancc3d4422017-08-03 08:27:51 -07006384 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07006385 num_frames_dropped = 0;
Åsa Persson8c1bf952018-09-13 10:42:19 +02006386 for (int i = 0; i <= max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006387 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006388 video_source_.IncomingCapturedFrame(
6389 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006390 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07006391 ++num_frames_dropped;
6392 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01006393 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07006394 }
6395 }
sprang4847ae62017-06-27 07:06:52 -07006396 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 4 / 9),
sprangc5d62e22017-04-02 23:53:04 -07006397 kErrorMargin);
6398
6399 // Go back up one step.
Henrik Boström91aa7322020-04-28 12:24:33 +02006400 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07006401 num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07006402 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006403 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006404 video_source_.IncomingCapturedFrame(
6405 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006406 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07006407 ++num_frames_dropped;
6408 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01006409 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07006410 }
6411 }
sprang4847ae62017-06-27 07:06:52 -07006412 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-02 23:53:04 -07006413 kErrorMargin);
6414
6415 // Go back up to original mode.
Henrik Boström91aa7322020-04-28 12:24:33 +02006416 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07006417 num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07006418 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006419 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006420 video_source_.IncomingCapturedFrame(
6421 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006422 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07006423 ++num_frames_dropped;
6424 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01006425 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07006426 }
6427 }
6428 EXPECT_NEAR(num_frames_dropped, 0, kErrorMargin);
6429
mflodmancc3d4422017-08-03 08:27:51 -07006430 video_stream_encoder_->Stop();
sprangc5d62e22017-04-02 23:53:04 -07006431}
6432
mflodmancc3d4422017-08-03 08:27:51 -07006433TEST_F(VideoStreamEncoderTest, DoesntAdaptDownPastMinFramerate) {
sprangc5d62e22017-04-02 23:53:04 -07006434 const int kFramerateFps = 5;
6435 const int kFrameIntervalMs = rtc::kNumMillisecsPerSec / kFramerateFps;
sprangc5d62e22017-04-02 23:53:04 -07006436 const int kFrameWidth = 1280;
6437 const int kFrameHeight = 720;
6438
sprang4847ae62017-06-27 07:06:52 -07006439 // Reconfigure encoder with two temporal layers and screensharing, which will
6440 // disable frame dropping and make testing easier.
Niels Möllerf1338562018-04-26 09:51:47 +02006441 ResetEncoder("VP8", 1, 2, 1, true);
sprang4847ae62017-06-27 07:06:52 -07006442
Henrik Boström381d1092020-05-12 18:49:07 +02006443 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006444 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07006445 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006446 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07006447 video_source_.set_adaptation_enabled(true);
6448
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006449 int64_t timestamp_ms = CurrentTimeMs();
sprangc5d62e22017-04-02 23:53:04 -07006450
6451 // Trigger overuse as much as we can.
Jonathan Yubc771b72017-12-08 17:04:29 -08006452 rtc::VideoSinkWants last_wants;
6453 do {
6454 last_wants = video_source_.sink_wants();
6455
sprangc5d62e22017-04-02 23:53:04 -07006456 // Insert frames to get a new fps estimate...
6457 for (int j = 0; j < kFramerateFps; ++j) {
6458 video_source_.IncomingCapturedFrame(
6459 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
Jonathan Yubc771b72017-12-08 17:04:29 -08006460 if (video_source_.last_sent_width()) {
6461 sink_.WaitForEncodedFrame(timestamp_ms);
6462 }
sprangc5d62e22017-04-02 23:53:04 -07006463 timestamp_ms += kFrameIntervalMs;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006464 AdvanceTime(TimeDelta::Millis(kFrameIntervalMs));
sprangc5d62e22017-04-02 23:53:04 -07006465 }
6466 // ...and then try to adapt again.
mflodmancc3d4422017-08-03 08:27:51 -07006467 video_stream_encoder_->TriggerCpuOveruse();
Jonathan Yubc771b72017-12-08 17:04:29 -08006468 } while (video_source_.sink_wants().max_framerate_fps <
6469 last_wants.max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07006470
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006471 EXPECT_THAT(video_source_.sink_wants(),
6472 FpsMatchesResolutionMax(Eq(kMinFramerateFps)));
asaperssonf7e294d2017-06-13 23:25:22 -07006473
mflodmancc3d4422017-08-03 08:27:51 -07006474 video_stream_encoder_->Stop();
sprangc5d62e22017-04-02 23:53:04 -07006475}
asaperssonf7e294d2017-06-13 23:25:22 -07006476
mflodmancc3d4422017-08-03 08:27:51 -07006477TEST_F(VideoStreamEncoderTest,
6478 AdaptsResolutionAndFramerateForLowQuality_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07006479 const int kWidth = 1280;
6480 const int kHeight = 720;
6481 const int64_t kFrameIntervalMs = 150;
6482 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02006483 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006484 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07006485
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006486 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006487 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07006488 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006489 video_stream_encoder_->SetSource(&source,
6490 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07006491 timestamp_ms += kFrameIntervalMs;
6492 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006493 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006494 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006495 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6496 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6497 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6498
6499 // Trigger adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006500 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006501 timestamp_ms += kFrameIntervalMs;
6502 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006503 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006504 EXPECT_THAT(source.sink_wants(),
6505 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07006506 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6507 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6508 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6509
6510 // Trigger adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006511 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006512 timestamp_ms += kFrameIntervalMs;
6513 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006514 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006515 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006516 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6517 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6518 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6519
6520 // Trigger adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006521 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006522 timestamp_ms += kFrameIntervalMs;
6523 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006524 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006525 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006526 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6527 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6528 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6529
6530 // Trigger adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006531 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006532 timestamp_ms += kFrameIntervalMs;
6533 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006534 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006535 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006536 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6537 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6538 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6539
6540 // Restrict bitrate, trigger adapt down, expect reduced fps (480x270@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07006541 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006542 timestamp_ms += kFrameIntervalMs;
6543 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006544 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006545 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006546 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6547 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6548 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6549
6550 // Trigger adapt down, expect scaled down resolution (320x180@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07006551 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006552 timestamp_ms += kFrameIntervalMs;
6553 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006554 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006555 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006556 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6557 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6558 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6559
6560 // Trigger adapt down, expect reduced fps (320x180@7fps).
mflodmancc3d4422017-08-03 08:27:51 -07006561 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006562 timestamp_ms += kFrameIntervalMs;
6563 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006564 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006565 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006566 rtc::VideoSinkWants last_wants = source.sink_wants();
6567 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6568 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6569 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6570
6571 // Trigger adapt down, min resolution reached, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006572 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006573 timestamp_ms += kFrameIntervalMs;
6574 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006575 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006576 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(last_wants));
asaperssonf7e294d2017-06-13 23:25:22 -07006577 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6578 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6579 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6580
Åsa Perssonf5f7e8e2021-06-09 22:55:38 +02006581 // Trigger adapt up, expect increased fps (320x180@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07006582 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006583 timestamp_ms += kFrameIntervalMs;
6584 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006585 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006586 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006587 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6588 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6589 EXPECT_EQ(8, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6590
6591 // Trigger adapt up, expect upscaled resolution (480x270@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07006592 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006593 timestamp_ms += kFrameIntervalMs;
6594 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006595 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006596 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006597 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6598 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6599 EXPECT_EQ(9, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6600
6601 // Increase bitrate, trigger adapt up, expect increased fps (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006602 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006603 timestamp_ms += kFrameIntervalMs;
6604 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006605 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006606 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006607 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6608 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6609 EXPECT_EQ(10, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6610
6611 // Trigger adapt up, expect upscaled resolution (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006612 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006613 timestamp_ms += kFrameIntervalMs;
6614 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006615 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006616 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006617 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6618 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6619 EXPECT_EQ(11, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6620
6621 // Trigger adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006622 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006623 timestamp_ms += kFrameIntervalMs;
6624 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006625 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006626 EXPECT_THAT(source.sink_wants(), FpsMax());
6627 EXPECT_EQ(source.sink_wants().max_pixel_count,
6628 source.last_wants().max_pixel_count);
asaperssonf7e294d2017-06-13 23:25:22 -07006629 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6630 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6631 EXPECT_EQ(12, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6632
6633 // Trigger adapt up, expect upscaled resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006634 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006635 timestamp_ms += kFrameIntervalMs;
6636 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006637 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006638 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006639 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6640 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6641 EXPECT_EQ(13, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6642
Åsa Persson30ab0152019-08-27 12:22:33 +02006643 // Trigger adapt up, expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006644 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006645 timestamp_ms += kFrameIntervalMs;
6646 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006647 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006648 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006649 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006650 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6651 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6652 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6653
6654 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006655 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006656 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006657 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6658
mflodmancc3d4422017-08-03 08:27:51 -07006659 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006660}
6661
mflodmancc3d4422017-08-03 08:27:51 -07006662TEST_F(VideoStreamEncoderTest, AdaptWithTwoReasonsAndDifferentOrder_Framerate) {
asaperssonf7e294d2017-06-13 23:25:22 -07006663 const int kWidth = 1280;
6664 const int kHeight = 720;
6665 const int64_t kFrameIntervalMs = 150;
6666 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02006667 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006668 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07006669
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006670 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006671 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07006672 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006673 video_stream_encoder_->SetSource(&source,
6674 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07006675 timestamp_ms += kFrameIntervalMs;
6676 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006677 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006678 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006679 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6680 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6681 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6682 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6683 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6684 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6685
6686 // Trigger cpu adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006687 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07006688 timestamp_ms += kFrameIntervalMs;
6689 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006690 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006691 EXPECT_THAT(source.sink_wants(),
6692 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07006693 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6694 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6695 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6696 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6697 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6698 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6699
6700 // Trigger cpu adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006701 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07006702 timestamp_ms += kFrameIntervalMs;
6703 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006704 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006705 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006706 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6707 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6708 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6709 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6710 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6711 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6712
6713 // Trigger quality adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006714 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006715 timestamp_ms += kFrameIntervalMs;
6716 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006717 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006718 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
Evan Shrubsole64469032020-06-11 10:45:29 +02006719 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
asaperssonf7e294d2017-06-13 23:25:22 -07006720 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6721 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6722 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6723 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6724 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6725
Evan Shrubsole64469032020-06-11 10:45:29 +02006726 // Trigger cpu adapt up, expect no change since QP is most limited.
6727 {
6728 // Store current sink wants since we expect no change and if there is no
6729 // change then last_wants() is not updated.
6730 auto previous_sink_wants = source.sink_wants();
6731 video_stream_encoder_->TriggerCpuUnderuse();
6732 timestamp_ms += kFrameIntervalMs;
6733 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6734 WaitForEncodedFrame(timestamp_ms);
6735 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(previous_sink_wants));
6736 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6737 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6738 }
6739
6740 // Trigger quality adapt up, expect increased fps (640x360@30fps).
6741 video_stream_encoder_->TriggerQualityHigh();
6742 timestamp_ms += kFrameIntervalMs;
6743 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6744 WaitForEncodedFrame(timestamp_ms);
6745 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
6746 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6747 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6748 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6749 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6750 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6751 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6752
6753 // Trigger quality adapt up and Cpu adapt up since both are most limited,
6754 // expect increased resolution (960x540@30fps).
6755 video_stream_encoder_->TriggerQualityHigh();
Henrik Boström91aa7322020-04-28 12:24:33 +02006756 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07006757 timestamp_ms += kFrameIntervalMs;
6758 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006759 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole64469032020-06-11 10:45:29 +02006760 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006761 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6762 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6763 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6764 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6765 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006766 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006767
Evan Shrubsole64469032020-06-11 10:45:29 +02006768 // Trigger quality adapt up and Cpu adapt up since both are most limited,
6769 // expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006770 video_stream_encoder_->TriggerQualityHigh();
Henrik Boström91aa7322020-04-28 12:24:33 +02006771 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07006772 timestamp_ms += kFrameIntervalMs;
6773 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006774 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006775 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006776 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006777 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6778 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6779 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6780 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6781 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006782 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006783
6784 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006785 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006786 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006787 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006788 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006789
mflodmancc3d4422017-08-03 08:27:51 -07006790 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006791}
6792
mflodmancc3d4422017-08-03 08:27:51 -07006793TEST_F(VideoStreamEncoderTest,
6794 AdaptWithTwoReasonsAndDifferentOrder_Resolution) {
asaperssonf7e294d2017-06-13 23:25:22 -07006795 const int kWidth = 640;
6796 const int kHeight = 360;
6797 const int kFpsLimit = 15;
6798 const int64_t kFrameIntervalMs = 150;
6799 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02006800 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006801 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07006802
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006803 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006804 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07006805 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006806 video_stream_encoder_->SetSource(&source,
6807 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07006808 timestamp_ms += kFrameIntervalMs;
6809 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006810 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006811 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006812 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6813 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6814 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6815 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6816 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6817 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6818
6819 // Trigger cpu adapt down, expect scaled down framerate (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006820 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07006821 timestamp_ms += kFrameIntervalMs;
6822 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006823 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006824 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07006825 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6826 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6827 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6828 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
6829 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6830 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6831
6832 // Trigger quality adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006833 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006834 timestamp_ms += kFrameIntervalMs;
6835 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006836 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006837 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006838 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Evan Shrubsole64469032020-06-11 10:45:29 +02006839 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
asaperssonf7e294d2017-06-13 23:25:22 -07006840 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6841 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
6842 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6843 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6844
Evan Shrubsole64469032020-06-11 10:45:29 +02006845 // Trigger cpu adapt up, expect no change because quality is most limited.
6846 {
6847 auto previous_sink_wants = source.sink_wants();
6848 // Store current sink wants since we expect no change ind if there is no
6849 // change then last__wants() is not updated.
6850 video_stream_encoder_->TriggerCpuUnderuse();
6851 timestamp_ms += kFrameIntervalMs;
6852 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6853 WaitForEncodedFrame(timestamp_ms);
6854 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(previous_sink_wants));
6855 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6856 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6857 }
6858
6859 // Trigger quality adapt up, expect upscaled resolution (640x360@15fps).
6860 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006861 timestamp_ms += kFrameIntervalMs;
6862 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006863 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006864 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006865 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6866 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6867 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Evan Shrubsole64469032020-06-11 10:45:29 +02006868 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
6869 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6870 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006871
Evan Shrubsole64469032020-06-11 10:45:29 +02006872 // Trigger quality and cpu adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006873 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole64469032020-06-11 10:45:29 +02006874 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07006875 timestamp_ms += kFrameIntervalMs;
6876 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006877 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006878 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006879 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6880 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6881 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6882 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6883 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006884 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006885
6886 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006887 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006888 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006889 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006890 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006891
mflodmancc3d4422017-08-03 08:27:51 -07006892 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006893}
6894
mflodmancc3d4422017-08-03 08:27:51 -07006895TEST_F(VideoStreamEncoderTest, AcceptsFullHdAdaptedDownSimulcastFrames) {
ilnik6b826ef2017-06-16 06:53:48 -07006896 const int kFrameWidth = 1920;
6897 const int kFrameHeight = 1080;
6898 // 3/4 of 1920.
6899 const int kAdaptedFrameWidth = 1440;
6900 // 3/4 of 1080 rounded down to multiple of 4.
6901 const int kAdaptedFrameHeight = 808;
6902 const int kFramerate = 24;
6903
Henrik Boström381d1092020-05-12 18:49:07 +02006904 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006905 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
ilnik6b826ef2017-06-16 06:53:48 -07006906 // Trigger reconfigure encoder (without resetting the entire instance).
6907 VideoEncoderConfig video_encoder_config;
Åsa Persson17b29b92020-10-17 12:57:58 +02006908 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
6909 video_encoder_config.simulcast_layers[0].max_framerate = kFramerate;
Asa Persson606d3cb2021-10-04 10:07:11 +02006910 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
ilnik6b826ef2017-06-16 06:53:48 -07006911 video_encoder_config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02006912 rtc::make_ref_counted<CroppingVideoStreamFactory>();
mflodmancc3d4422017-08-03 08:27:51 -07006913 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02006914 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -07006915 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
ilnik6b826ef2017-06-16 06:53:48 -07006916
6917 video_source_.set_adaptation_enabled(true);
6918
6919 video_source_.IncomingCapturedFrame(
6920 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006921 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
ilnik6b826ef2017-06-16 06:53:48 -07006922
6923 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 08:27:51 -07006924 video_stream_encoder_->TriggerCpuOveruse();
ilnik6b826ef2017-06-16 06:53:48 -07006925 video_source_.IncomingCapturedFrame(
6926 CreateFrame(2, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006927 WaitForEncodedFrame(kAdaptedFrameWidth, kAdaptedFrameHeight);
ilnik6b826ef2017-06-16 06:53:48 -07006928
mflodmancc3d4422017-08-03 08:27:51 -07006929 video_stream_encoder_->Stop();
ilnik6b826ef2017-06-16 06:53:48 -07006930}
6931
mflodmancc3d4422017-08-03 08:27:51 -07006932TEST_F(VideoStreamEncoderTest, PeriodicallyUpdatesChannelParameters) {
sprang4847ae62017-06-27 07:06:52 -07006933 const int kFrameWidth = 1280;
6934 const int kFrameHeight = 720;
6935 const int kLowFps = 2;
6936 const int kHighFps = 30;
6937
Henrik Boström381d1092020-05-12 18:49:07 +02006938 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006939 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07006940
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006941 int64_t timestamp_ms = CurrentTimeMs();
sprang4847ae62017-06-27 07:06:52 -07006942 max_framerate_ = kLowFps;
6943
6944 // Insert 2 seconds of 2fps video.
6945 for (int i = 0; i < kLowFps * 2; ++i) {
6946 video_source_.IncomingCapturedFrame(
6947 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6948 WaitForEncodedFrame(timestamp_ms);
6949 timestamp_ms += 1000 / kLowFps;
6950 }
6951
6952 // Make sure encoder is updated with new target.
Henrik Boström381d1092020-05-12 18:49:07 +02006953 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006954 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07006955 video_source_.IncomingCapturedFrame(
6956 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6957 WaitForEncodedFrame(timestamp_ms);
6958 timestamp_ms += 1000 / kLowFps;
6959
6960 EXPECT_EQ(kLowFps, fake_encoder_.GetConfiguredInputFramerate());
6961
6962 // Insert 30fps frames for just a little more than the forced update period.
Niels Möllerfe407b72019-09-10 10:48:48 +02006963 const int kVcmTimerIntervalFrames = (kProcessIntervalMs * kHighFps) / 1000;
sprang4847ae62017-06-27 07:06:52 -07006964 const int kFrameIntervalMs = 1000 / kHighFps;
6965 max_framerate_ = kHighFps;
6966 for (int i = 0; i < kVcmTimerIntervalFrames + 2; ++i) {
6967 video_source_.IncomingCapturedFrame(
6968 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6969 // Wait for encoded frame, but skip ahead if it doesn't arrive as it might
6970 // be dropped if the encoder hans't been updated with the new higher target
6971 // framerate yet, causing it to overshoot the target bitrate and then
6972 // suffering the wrath of the media optimizer.
6973 TimedWaitForEncodedFrame(timestamp_ms, 2 * kFrameIntervalMs);
6974 timestamp_ms += kFrameIntervalMs;
6975 }
6976
6977 // Don expect correct measurement just yet, but it should be higher than
6978 // before.
6979 EXPECT_GT(fake_encoder_.GetConfiguredInputFramerate(), kLowFps);
6980
mflodmancc3d4422017-08-03 08:27:51 -07006981 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 07:06:52 -07006982}
6983
mflodmancc3d4422017-08-03 08:27:51 -07006984TEST_F(VideoStreamEncoderTest, DoesNotUpdateBitrateAllocationWhenSuspended) {
sprang4847ae62017-06-27 07:06:52 -07006985 const int kFrameWidth = 1280;
6986 const int kFrameHeight = 720;
Per Kjellanderdcef6412020-10-07 15:09:05 +02006987 ResetEncoder("FAKE", 1, 1, 1, false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01006988 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02006989 kVideoBitrateAllocation);
sprang4847ae62017-06-27 07:06:52 -07006990
Henrik Boström381d1092020-05-12 18:49:07 +02006991 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006992 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07006993 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprang4847ae62017-06-27 07:06:52 -07006994
6995 // Insert a first video frame, causes another bitrate update.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006996 int64_t timestamp_ms = CurrentTimeMs();
sprang4847ae62017-06-27 07:06:52 -07006997 video_source_.IncomingCapturedFrame(
6998 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6999 WaitForEncodedFrame(timestamp_ms);
Per Kjellanderdcef6412020-10-07 15:09:05 +02007000 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
sprang4847ae62017-06-27 07:06:52 -07007001
7002 // Next, simulate video suspension due to pacer queue overrun.
Henrik Boström381d1092020-05-12 18:49:07 +02007003 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007004 DataRate::Zero(), DataRate::Zero(), DataRate::Zero(), 0, 1, 0);
sprang4847ae62017-06-27 07:06:52 -07007005
7006 // Skip ahead until a new periodic parameter update should have occured.
Niels Möllerfe407b72019-09-10 10:48:48 +02007007 timestamp_ms += kProcessIntervalMs;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007008 AdvanceTime(TimeDelta::Millis(kProcessIntervalMs));
sprang4847ae62017-06-27 07:06:52 -07007009
Per Kjellanderdcef6412020-10-07 15:09:05 +02007010 // No more allocations has been made.
sprang4847ae62017-06-27 07:06:52 -07007011 video_source_.IncomingCapturedFrame(
7012 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7013 ExpectDroppedFrame();
Per Kjellanderdcef6412020-10-07 15:09:05 +02007014 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
sprang4847ae62017-06-27 07:06:52 -07007015
mflodmancc3d4422017-08-03 08:27:51 -07007016 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 07:06:52 -07007017}
ilnik6b826ef2017-06-16 06:53:48 -07007018
Niels Möller4db138e2018-04-19 09:04:13 +02007019TEST_F(VideoStreamEncoderTest,
7020 DefaultCpuAdaptationThresholdsForSoftwareEncoder) {
7021 const int kFrameWidth = 1280;
7022 const int kFrameHeight = 720;
7023 const CpuOveruseOptions default_options;
Henrik Boström381d1092020-05-12 18:49:07 +02007024 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007025 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller4db138e2018-04-19 09:04:13 +02007026 video_source_.IncomingCapturedFrame(
7027 CreateFrame(1, kFrameWidth, kFrameHeight));
7028 WaitForEncodedFrame(1);
7029 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7030 .low_encode_usage_threshold_percent,
7031 default_options.low_encode_usage_threshold_percent);
7032 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7033 .high_encode_usage_threshold_percent,
7034 default_options.high_encode_usage_threshold_percent);
7035 video_stream_encoder_->Stop();
7036}
7037
7038TEST_F(VideoStreamEncoderTest,
7039 HigherCpuAdaptationThresholdsForHardwareEncoder) {
7040 const int kFrameWidth = 1280;
7041 const int kFrameHeight = 720;
7042 CpuOveruseOptions hardware_options;
7043 hardware_options.low_encode_usage_threshold_percent = 150;
7044 hardware_options.high_encode_usage_threshold_percent = 200;
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01007045 fake_encoder_.SetIsHardwareAccelerated(true);
Niels Möller4db138e2018-04-19 09:04:13 +02007046
Henrik Boström381d1092020-05-12 18:49:07 +02007047 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007048 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller4db138e2018-04-19 09:04:13 +02007049 video_source_.IncomingCapturedFrame(
7050 CreateFrame(1, kFrameWidth, kFrameHeight));
7051 WaitForEncodedFrame(1);
7052 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7053 .low_encode_usage_threshold_percent,
7054 hardware_options.low_encode_usage_threshold_percent);
7055 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7056 .high_encode_usage_threshold_percent,
7057 hardware_options.high_encode_usage_threshold_percent);
7058 video_stream_encoder_->Stop();
7059}
7060
Jakob Ivarsson461b1d92021-01-22 16:27:43 +01007061TEST_F(VideoStreamEncoderTest,
7062 CpuAdaptationThresholdsUpdatesWhenHardwareAccelerationChange) {
7063 const int kFrameWidth = 1280;
7064 const int kFrameHeight = 720;
7065
7066 const CpuOveruseOptions default_options;
7067 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007068 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Jakob Ivarsson461b1d92021-01-22 16:27:43 +01007069 video_source_.IncomingCapturedFrame(
7070 CreateFrame(1, kFrameWidth, kFrameHeight));
7071 WaitForEncodedFrame(1);
7072 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7073 .low_encode_usage_threshold_percent,
7074 default_options.low_encode_usage_threshold_percent);
7075 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7076 .high_encode_usage_threshold_percent,
7077 default_options.high_encode_usage_threshold_percent);
7078
7079 CpuOveruseOptions hardware_options;
7080 hardware_options.low_encode_usage_threshold_percent = 150;
7081 hardware_options.high_encode_usage_threshold_percent = 200;
7082 fake_encoder_.SetIsHardwareAccelerated(true);
7083
7084 video_source_.IncomingCapturedFrame(
7085 CreateFrame(2, kFrameWidth, kFrameHeight));
7086 WaitForEncodedFrame(2);
7087
7088 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7089 .low_encode_usage_threshold_percent,
7090 hardware_options.low_encode_usage_threshold_percent);
7091 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7092 .high_encode_usage_threshold_percent,
7093 hardware_options.high_encode_usage_threshold_percent);
7094
7095 video_stream_encoder_->Stop();
7096}
7097
Niels Möller6bb5ab92019-01-11 11:11:10 +01007098TEST_F(VideoStreamEncoderTest, DropsFramesWhenEncoderOvershoots) {
7099 const int kFrameWidth = 320;
7100 const int kFrameHeight = 240;
7101 const int kFps = 30;
Asa Persson606d3cb2021-10-04 10:07:11 +02007102 const DataRate kTargetBitrate = DataRate::KilobitsPerSec(120);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007103 const int kNumFramesInRun = kFps * 5; // Runs of five seconds.
7104
Henrik Boström381d1092020-05-12 18:49:07 +02007105 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007106 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007107
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007108 int64_t timestamp_ms = CurrentTimeMs();
Niels Möller6bb5ab92019-01-11 11:11:10 +01007109 max_framerate_ = kFps;
7110
7111 // Insert 3 seconds of video, verify number of drops with normal bitrate.
7112 fake_encoder_.SimulateOvershoot(1.0);
7113 int num_dropped = 0;
7114 for (int i = 0; i < kNumFramesInRun; ++i) {
7115 video_source_.IncomingCapturedFrame(
7116 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7117 // Wait up to two frame durations for a frame to arrive.
7118 if (!TimedWaitForEncodedFrame(timestamp_ms, 2 * 1000 / kFps)) {
7119 ++num_dropped;
7120 }
7121 timestamp_ms += 1000 / kFps;
7122 }
7123
Erik Språnga8d48ab2019-02-08 14:17:40 +01007124 // Framerate should be measured to be near the expected target rate.
7125 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
7126
7127 // Frame drops should be within 5% of expected 0%.
7128 EXPECT_NEAR(num_dropped, 0, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007129
7130 // Make encoder produce frames at double the expected bitrate during 3 seconds
7131 // of video, verify number of drops. Rate needs to be slightly changed in
7132 // order to force the rate to be reconfigured.
Erik Språng7ca375c2019-02-06 16:20:17 +01007133 double overshoot_factor = 2.0;
Erik Språng9d69cbe2020-10-22 17:44:42 +02007134 const RateControlSettings trials =
7135 RateControlSettings::ParseFromFieldTrials();
7136 if (trials.UseEncoderBitrateAdjuster()) {
Erik Språng7ca375c2019-02-06 16:20:17 +01007137 // With bitrate adjuster, when need to overshoot even more to trigger
Erik Språng9d69cbe2020-10-22 17:44:42 +02007138 // frame dropping since the adjuter will try to just lower the target
7139 // bitrate rather than drop frames. If network headroom can be used, it
7140 // doesn't push back as hard so we don't need quite as much overshoot.
7141 // These numbers are unfortunately a bit magical but there's not trivial
7142 // way to algebraically infer them.
Erik Språng73cf80a2021-03-24 11:10:09 +01007143 overshoot_factor = 3.0;
Erik Språng7ca375c2019-02-06 16:20:17 +01007144 }
7145 fake_encoder_.SimulateOvershoot(overshoot_factor);
Henrik Boström381d1092020-05-12 18:49:07 +02007146 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007147 kTargetBitrate + DataRate::KilobitsPerSec(1),
7148 kTargetBitrate + DataRate::KilobitsPerSec(1),
7149 kTargetBitrate + DataRate::KilobitsPerSec(1), 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007150 num_dropped = 0;
7151 for (int i = 0; i < kNumFramesInRun; ++i) {
7152 video_source_.IncomingCapturedFrame(
7153 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7154 // Wait up to two frame durations for a frame to arrive.
7155 if (!TimedWaitForEncodedFrame(timestamp_ms, 2 * 1000 / kFps)) {
7156 ++num_dropped;
7157 }
7158 timestamp_ms += 1000 / kFps;
7159 }
7160
Henrik Boström381d1092020-05-12 18:49:07 +02007161 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007162 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Erik Språnga8d48ab2019-02-08 14:17:40 +01007163
7164 // Target framerate should be still be near the expected target, despite
7165 // the frame drops.
7166 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
7167
7168 // Frame drops should be within 5% of expected 50%.
7169 EXPECT_NEAR(num_dropped, kNumFramesInRun / 2, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007170
7171 video_stream_encoder_->Stop();
7172}
7173
7174TEST_F(VideoStreamEncoderTest, ConfiguresCorrectFrameRate) {
7175 const int kFrameWidth = 320;
7176 const int kFrameHeight = 240;
7177 const int kActualInputFps = 24;
Asa Persson606d3cb2021-10-04 10:07:11 +02007178 const DataRate kTargetBitrate = DataRate::KilobitsPerSec(120);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007179
7180 ASSERT_GT(max_framerate_, kActualInputFps);
7181
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007182 int64_t timestamp_ms = CurrentTimeMs();
Niels Möller6bb5ab92019-01-11 11:11:10 +01007183 max_framerate_ = kActualInputFps;
Henrik Boström381d1092020-05-12 18:49:07 +02007184 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007185 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007186
7187 // Insert 3 seconds of video, with an input fps lower than configured max.
7188 for (int i = 0; i < kActualInputFps * 3; ++i) {
7189 video_source_.IncomingCapturedFrame(
7190 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7191 // Wait up to two frame durations for a frame to arrive.
7192 WaitForEncodedFrame(timestamp_ms);
7193 timestamp_ms += 1000 / kActualInputFps;
7194 }
7195
7196 EXPECT_NEAR(kActualInputFps, fake_encoder_.GetLastFramerate(), 1);
7197
7198 video_stream_encoder_->Stop();
7199}
7200
Markus Handell9a478b52021-11-18 16:07:01 +01007201TEST_F(VideoStreamEncoderTest, AccumulatesUpdateRectOnDroppedFrames) {
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01007202 VideoFrame::UpdateRect rect;
Markus Handell9a478b52021-11-18 16:07:01 +01007203 test::FrameForwarder source;
7204 video_stream_encoder_->SetSource(&source,
7205 DegradationPreference::MAINTAIN_FRAMERATE);
Henrik Boström381d1092020-05-12 18:49:07 +02007206 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007207 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01007208
Markus Handell9a478b52021-11-18 16:07:01 +01007209 source.IncomingCapturedFrame(CreateFrameWithUpdatedPixel(1, nullptr, 0));
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01007210 WaitForEncodedFrame(1);
7211 // On the very first frame full update should be forced.
7212 rect = fake_encoder_.GetLastUpdateRect();
7213 EXPECT_EQ(rect.offset_x, 0);
7214 EXPECT_EQ(rect.offset_y, 0);
7215 EXPECT_EQ(rect.height, codec_height_);
7216 EXPECT_EQ(rect.width, codec_width_);
Markus Handell9a478b52021-11-18 16:07:01 +01007217 // Frame with NTP timestamp 2 will be dropped due to outstanding frames
7218 // scheduled for processing during encoder queue processing of frame 2.
7219 source.IncomingCapturedFrame(CreateFrameWithUpdatedPixel(2, nullptr, 1));
7220 source.IncomingCapturedFrame(CreateFrameWithUpdatedPixel(3, nullptr, 10));
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01007221 WaitForEncodedFrame(3);
7222 // Updates to pixels 1 and 10 should be accumulated to one 10x1 rect.
7223 rect = fake_encoder_.GetLastUpdateRect();
7224 EXPECT_EQ(rect.offset_x, 1);
7225 EXPECT_EQ(rect.offset_y, 0);
7226 EXPECT_EQ(rect.width, 10);
7227 EXPECT_EQ(rect.height, 1);
7228
Markus Handell9a478b52021-11-18 16:07:01 +01007229 source.IncomingCapturedFrame(CreateFrameWithUpdatedPixel(4, nullptr, 0));
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01007230 WaitForEncodedFrame(4);
7231 // Previous frame was encoded, so no accumulation should happen.
7232 rect = fake_encoder_.GetLastUpdateRect();
7233 EXPECT_EQ(rect.offset_x, 0);
7234 EXPECT_EQ(rect.offset_y, 0);
7235 EXPECT_EQ(rect.width, 1);
7236 EXPECT_EQ(rect.height, 1);
7237
7238 video_stream_encoder_->Stop();
7239}
7240
Erik Språngd7329ca2019-02-21 21:19:53 +01007241TEST_F(VideoStreamEncoderTest, SetsFrameTypes) {
Henrik Boström381d1092020-05-12 18:49:07 +02007242 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007243 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01007244
7245 // First frame is always keyframe.
7246 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7247 WaitForEncodedFrame(1);
Niels Möller8f7ce222019-03-21 15:43:58 +01007248 EXPECT_THAT(
7249 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007250 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007251
7252 // Insert delta frame.
7253 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
7254 WaitForEncodedFrame(2);
Niels Möller8f7ce222019-03-21 15:43:58 +01007255 EXPECT_THAT(
7256 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007257 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007258
7259 // Request next frame be a key-frame.
7260 video_stream_encoder_->SendKeyFrame();
7261 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
7262 WaitForEncodedFrame(3);
Niels Möller8f7ce222019-03-21 15:43:58 +01007263 EXPECT_THAT(
7264 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007265 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007266
7267 video_stream_encoder_->Stop();
7268}
7269
7270TEST_F(VideoStreamEncoderTest, SetsFrameTypesSimulcast) {
7271 // Setup simulcast with three streams.
7272 ResetEncoder("VP8", 3, 1, 1, false);
Henrik Boström381d1092020-05-12 18:49:07 +02007273 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007274 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
7275 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01007276 // Wait for all three layers before triggering event.
7277 sink_.SetNumExpectedLayers(3);
7278
7279 // First frame is always keyframe.
7280 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7281 WaitForEncodedFrame(1);
7282 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007283 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7284 VideoFrameType::kVideoFrameKey,
7285 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007286
7287 // Insert delta frame.
7288 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
7289 WaitForEncodedFrame(2);
7290 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007291 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
7292 VideoFrameType::kVideoFrameDelta,
7293 VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007294
7295 // Request next frame be a key-frame.
7296 // Only first stream is configured to produce key-frame.
7297 video_stream_encoder_->SendKeyFrame();
7298 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
7299 WaitForEncodedFrame(3);
Sergey Silkine62a08a2019-05-13 13:45:39 +02007300
7301 // TODO(webrtc:10615): Map keyframe request to spatial layer. Currently
7302 // keyframe request on any layer triggers keyframe on all layers.
Erik Språngd7329ca2019-02-21 21:19:53 +01007303 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007304 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
Sergey Silkine62a08a2019-05-13 13:45:39 +02007305 VideoFrameType::kVideoFrameKey,
7306 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007307
7308 video_stream_encoder_->Stop();
7309}
7310
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007311TEST_F(VideoStreamEncoderTest, DoesNotRewriteH264BitstreamWithOptimalSps) {
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007312 // SPS contains VUI with restrictions on the maximum number of reordered
7313 // pictures, there is no need to rewrite the bitstream to enable faster
7314 // decoding.
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007315 ResetEncoder("H264", 1, 1, 1, false);
7316
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007317 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007318 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007319 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007320
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007321 fake_encoder_.SetEncodedImageData(
Asa Persson606d3cb2021-10-04 10:07:11 +02007322 EncodedImageBuffer::Create(kOptimalSps, sizeof(kOptimalSps)));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007323
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007324 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7325 WaitForEncodedFrame(1);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007326
7327 EXPECT_THAT(sink_.GetLastEncodedImageData(),
Asa Persson606d3cb2021-10-04 10:07:11 +02007328 testing::ElementsAreArray(kOptimalSps));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007329
7330 video_stream_encoder_->Stop();
7331}
7332
7333TEST_F(VideoStreamEncoderTest, RewritesH264BitstreamWithNonOptimalSps) {
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007334 // SPS does not contain VUI, the bitstream is will be rewritten with added
7335 // VUI with restrictions on the maximum number of reordered pictures to
7336 // enable faster decoding.
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007337 uint8_t original_sps[] = {0, 0, 0, 1, H264::NaluType::kSps,
7338 0x00, 0x00, 0x03, 0x03, 0xF4,
7339 0x05, 0x03, 0xC7, 0xC0};
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007340 ResetEncoder("H264", 1, 1, 1, false);
7341
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007342 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007343 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007344 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007345
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007346 fake_encoder_.SetEncodedImageData(
7347 EncodedImageBuffer::Create(original_sps, sizeof(original_sps)));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007348
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007349 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7350 WaitForEncodedFrame(1);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007351
7352 EXPECT_THAT(sink_.GetLastEncodedImageData(),
Asa Persson606d3cb2021-10-04 10:07:11 +02007353 testing::ElementsAreArray(kOptimalSps));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007354
7355 video_stream_encoder_->Stop();
7356}
7357
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007358TEST_F(VideoStreamEncoderTest, CopiesVideoFrameMetadataAfterDownscale) {
7359 const int kFrameWidth = 1280;
7360 const int kFrameHeight = 720;
Asa Persson606d3cb2021-10-04 10:07:11 +02007361 const DataRate kTargetBitrate =
7362 DataRate::KilobitsPerSec(300); // Too low for HD resolution.
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007363
Henrik Boström381d1092020-05-12 18:49:07 +02007364 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007365 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007366 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7367
7368 // Insert a first video frame. It should be dropped because of downscale in
7369 // resolution.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007370 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007371 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7372 frame.set_rotation(kVideoRotation_270);
7373 video_source_.IncomingCapturedFrame(frame);
7374
7375 ExpectDroppedFrame();
7376
7377 // Second frame is downscaled.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007378 timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007379 frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
7380 frame.set_rotation(kVideoRotation_90);
7381 video_source_.IncomingCapturedFrame(frame);
7382
7383 WaitForEncodedFrame(timestamp_ms);
7384 sink_.CheckLastFrameRotationMatches(kVideoRotation_90);
7385
7386 // Insert another frame, also downscaled.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007387 timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007388 frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
7389 frame.set_rotation(kVideoRotation_180);
7390 video_source_.IncomingCapturedFrame(frame);
7391
7392 WaitForEncodedFrame(timestamp_ms);
7393 sink_.CheckLastFrameRotationMatches(kVideoRotation_180);
7394
7395 video_stream_encoder_->Stop();
7396}
7397
Erik Språng5056af02019-09-02 15:53:11 +02007398TEST_F(VideoStreamEncoderTest, BandwidthAllocationLowerBound) {
7399 const int kFrameWidth = 320;
7400 const int kFrameHeight = 180;
7401
7402 // Initial rate.
Henrik Boström381d1092020-05-12 18:49:07 +02007403 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007404 /*target_bitrate=*/DataRate::KilobitsPerSec(300),
7405 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(300),
7406 /*link_allocation=*/DataRate::KilobitsPerSec(300),
Erik Språng5056af02019-09-02 15:53:11 +02007407 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007408 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007409 /*cwnd_reduce_ratio=*/0);
Erik Språng5056af02019-09-02 15:53:11 +02007410
7411 // Insert a first video frame so that encoder gets configured.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007412 int64_t timestamp_ms = CurrentTimeMs();
Erik Språng5056af02019-09-02 15:53:11 +02007413 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7414 frame.set_rotation(kVideoRotation_270);
7415 video_source_.IncomingCapturedFrame(frame);
7416 WaitForEncodedFrame(timestamp_ms);
7417
7418 // Set a target rate below the minimum allowed by the codec settings.
Asa Persson606d3cb2021-10-04 10:07:11 +02007419 VideoCodec codec_config = fake_encoder_.config();
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007420 DataRate min_rate = DataRate::KilobitsPerSec(codec_config.minBitrate);
7421 DataRate target_rate = min_rate - DataRate::KilobitsPerSec(1);
Henrik Boström381d1092020-05-12 18:49:07 +02007422 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Erik Språng5056af02019-09-02 15:53:11 +02007423 /*target_bitrate=*/target_rate,
Florent Castellia8336d32019-09-09 13:36:55 +02007424 /*stable_target_bitrate=*/target_rate,
Erik Språng5056af02019-09-02 15:53:11 +02007425 /*link_allocation=*/target_rate,
7426 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007427 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007428 /*cwnd_reduce_ratio=*/0);
Erik Språng5056af02019-09-02 15:53:11 +02007429 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7430
7431 // Target bitrate and bandwidth allocation should both be capped at min_rate.
7432 auto rate_settings = fake_encoder_.GetAndResetLastRateControlSettings();
7433 ASSERT_TRUE(rate_settings.has_value());
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007434 DataRate allocation_sum =
7435 DataRate::BitsPerSec(rate_settings->bitrate.get_sum_bps());
Erik Språng5056af02019-09-02 15:53:11 +02007436 EXPECT_EQ(min_rate, allocation_sum);
7437 EXPECT_EQ(rate_settings->bandwidth_allocation, min_rate);
7438
7439 video_stream_encoder_->Stop();
7440}
7441
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01007442TEST_F(VideoStreamEncoderTest, EncoderRatesPropagatedOnReconfigure) {
Henrik Boström381d1092020-05-12 18:49:07 +02007443 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007444 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02007445 // Capture a frame and wait for it to synchronize with the encoder thread.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007446 int64_t timestamp_ms = CurrentTimeMs();
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02007447 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
7448 WaitForEncodedFrame(1);
7449
7450 auto prev_rate_settings = fake_encoder_.GetAndResetLastRateControlSettings();
7451 ASSERT_TRUE(prev_rate_settings.has_value());
7452 EXPECT_EQ(static_cast<int>(prev_rate_settings->framerate_fps),
7453 kDefaultFramerate);
7454
7455 // Send 1s of video to ensure the framerate is stable at kDefaultFramerate.
7456 for (int i = 0; i < 2 * kDefaultFramerate; i++) {
7457 timestamp_ms += 1000 / kDefaultFramerate;
7458 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
7459 WaitForEncodedFrame(timestamp_ms);
7460 }
7461 EXPECT_EQ(static_cast<int>(fake_encoder_.GetLastFramerate()),
7462 kDefaultFramerate);
7463 // Capture larger frame to trigger a reconfigure.
7464 codec_height_ *= 2;
7465 codec_width_ *= 2;
7466 timestamp_ms += 1000 / kDefaultFramerate;
7467 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
7468 WaitForEncodedFrame(timestamp_ms);
7469
7470 EXPECT_EQ(2, sink_.number_of_reconfigurations());
7471 auto current_rate_settings =
7472 fake_encoder_.GetAndResetLastRateControlSettings();
7473 // Ensure we have actually reconfigured twice
7474 // The rate settings should have been set again even though
7475 // they haven't changed.
7476 ASSERT_TRUE(current_rate_settings.has_value());
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007477 EXPECT_EQ(prev_rate_settings, current_rate_settings);
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02007478
7479 video_stream_encoder_->Stop();
7480}
7481
philipeld9cc8c02019-09-16 14:53:40 +02007482struct MockEncoderSwitchRequestCallback : public EncoderSwitchRequestCallback {
Danil Chapovalov91fdc602020-05-14 19:17:51 +02007483 MOCK_METHOD(void, RequestEncoderFallback, (), (override));
Danil Chapovalov91fdc602020-05-14 19:17:51 +02007484 MOCK_METHOD(void,
7485 RequestEncoderSwitch,
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007486 (const webrtc::SdpVideoFormat& format,
7487 bool allow_default_fallback),
Danil Chapovalov91fdc602020-05-14 19:17:51 +02007488 (override));
philipeld9cc8c02019-09-16 14:53:40 +02007489};
7490
philipel9b058032020-02-10 11:30:00 +01007491TEST_F(VideoStreamEncoderTest, EncoderSelectorCurrentEncoderIsSignaled) {
7492 constexpr int kDontCare = 100;
7493 StrictMock<MockEncoderSelector> encoder_selector;
7494 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7495 &fake_encoder_, &encoder_selector);
7496 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7497
7498 // Reset encoder for new configuration to take effect.
7499 ConfigureEncoder(video_encoder_config_.Copy());
7500
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007501 EXPECT_CALL(encoder_selector, OnCurrentEncoder);
philipel9b058032020-02-10 11:30:00 +01007502
7503 video_source_.IncomingCapturedFrame(
7504 CreateFrame(kDontCare, kDontCare, kDontCare));
Markus Handell28c71802021-11-08 10:11:55 +01007505 AdvanceTime(TimeDelta::Zero());
philipel9b058032020-02-10 11:30:00 +01007506 video_stream_encoder_->Stop();
7507
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007508 // The encoders produced by the VideoEncoderProxyFactory have a pointer back
philipel9b058032020-02-10 11:30:00 +01007509 // to it's factory, so in order for the encoder instance in the
Artem Titovab30d722021-07-27 16:22:11 +02007510 // `video_stream_encoder_` to be destroyed before the `encoder_factory` we
7511 // reset the `video_stream_encoder_` here.
philipel9b058032020-02-10 11:30:00 +01007512 video_stream_encoder_.reset();
7513}
7514
7515TEST_F(VideoStreamEncoderTest, EncoderSelectorBitrateSwitch) {
7516 constexpr int kDontCare = 100;
7517
7518 NiceMock<MockEncoderSelector> encoder_selector;
7519 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7520 video_send_config_.encoder_settings.encoder_switch_request_callback =
7521 &switch_callback;
7522 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7523 &fake_encoder_, &encoder_selector);
7524 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7525
7526 // Reset encoder for new configuration to take effect.
7527 ConfigureEncoder(video_encoder_config_.Copy());
7528
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007529 ON_CALL(encoder_selector, OnAvailableBitrate)
philipel9b058032020-02-10 11:30:00 +01007530 .WillByDefault(Return(SdpVideoFormat("AV1")));
7531 EXPECT_CALL(switch_callback,
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007532 RequestEncoderSwitch(Field(&SdpVideoFormat::name, "AV1"),
7533 /*allow_default_fallback=*/false));
philipel9b058032020-02-10 11:30:00 +01007534
Henrik Boström381d1092020-05-12 18:49:07 +02007535 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007536 /*target_bitrate=*/DataRate::KilobitsPerSec(50),
7537 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(kDontCare),
7538 /*link_allocation=*/DataRate::KilobitsPerSec(kDontCare),
philipel9b058032020-02-10 11:30:00 +01007539 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007540 /*round_trip_time_ms=*/0,
philipel9b058032020-02-10 11:30:00 +01007541 /*cwnd_reduce_ratio=*/0);
Markus Handell28c71802021-11-08 10:11:55 +01007542 AdvanceTime(TimeDelta::Zero());
philipel9b058032020-02-10 11:30:00 +01007543
7544 video_stream_encoder_->Stop();
7545}
7546
7547TEST_F(VideoStreamEncoderTest, EncoderSelectorBrokenEncoderSwitch) {
7548 constexpr int kSufficientBitrateToNotDrop = 1000;
7549 constexpr int kDontCare = 100;
7550
7551 NiceMock<MockVideoEncoder> video_encoder;
7552 NiceMock<MockEncoderSelector> encoder_selector;
7553 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7554 video_send_config_.encoder_settings.encoder_switch_request_callback =
7555 &switch_callback;
7556 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7557 &video_encoder, &encoder_selector);
7558 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7559
7560 // Reset encoder for new configuration to take effect.
7561 ConfigureEncoder(video_encoder_config_.Copy());
7562
7563 // The VideoStreamEncoder needs some bitrate before it can start encoding,
7564 // setting some bitrate so that subsequent calls to WaitForEncodedFrame does
7565 // not fail.
Henrik Boström381d1092020-05-12 18:49:07 +02007566 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007567 /*target_bitrate=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
7568 /*stable_target_bitrate=*/
7569 DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
7570 /*link_allocation=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
philipel9b058032020-02-10 11:30:00 +01007571 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007572 /*round_trip_time_ms=*/0,
philipel9b058032020-02-10 11:30:00 +01007573 /*cwnd_reduce_ratio=*/0);
7574
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007575 ON_CALL(video_encoder, Encode)
philipel9b058032020-02-10 11:30:00 +01007576 .WillByDefault(Return(WEBRTC_VIDEO_CODEC_ENCODER_FAILURE));
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007577 ON_CALL(encoder_selector, OnEncoderBroken)
philipel9b058032020-02-10 11:30:00 +01007578 .WillByDefault(Return(SdpVideoFormat("AV2")));
7579
7580 rtc::Event encode_attempted;
7581 EXPECT_CALL(switch_callback,
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007582 RequestEncoderSwitch(Field(&SdpVideoFormat::name, "AV2"),
7583 /*allow_default_fallback=*/true))
7584 .WillOnce([&encode_attempted]() { encode_attempted.Set(); });
philipel9b058032020-02-10 11:30:00 +01007585
7586 video_source_.IncomingCapturedFrame(CreateFrame(1, kDontCare, kDontCare));
7587 encode_attempted.Wait(3000);
7588
Markus Handell28c71802021-11-08 10:11:55 +01007589 AdvanceTime(TimeDelta::Zero());
Tomas Gunnarssond41c2a62020-09-21 15:56:42 +02007590
philipel9b058032020-02-10 11:30:00 +01007591 video_stream_encoder_->Stop();
7592
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007593 // The encoders produced by the VideoEncoderProxyFactory have a pointer back
7594 // to it's factory, so in order for the encoder instance in the
7595 // `video_stream_encoder_` to be destroyed before the `encoder_factory` we
7596 // reset the `video_stream_encoder_` here.
7597 video_stream_encoder_.reset();
7598}
7599
7600TEST_F(VideoStreamEncoderTest, SwitchEncoderOnInitFailureWithEncoderSelector) {
7601 NiceMock<MockVideoEncoder> video_encoder;
7602 NiceMock<MockEncoderSelector> encoder_selector;
7603 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7604 video_send_config_.encoder_settings.encoder_switch_request_callback =
7605 &switch_callback;
7606 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7607 &video_encoder, &encoder_selector);
7608 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7609
7610 // Reset encoder for new configuration to take effect.
7611 ConfigureEncoder(video_encoder_config_.Copy());
7612
7613 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
7614 kTargetBitrate, kTargetBitrate, kTargetBitrate, /*fraction_lost=*/0,
7615 /*round_trip_time_ms=*/0,
7616 /*cwnd_reduce_ratio=*/0);
7617 ASSERT_EQ(0, sink_.number_of_reconfigurations());
7618
7619 ON_CALL(video_encoder, InitEncode(_, _))
7620 .WillByDefault(Return(WEBRTC_VIDEO_CODEC_ENCODER_FAILURE));
7621 ON_CALL(encoder_selector, OnEncoderBroken)
7622 .WillByDefault(Return(SdpVideoFormat("AV2")));
7623
7624 rtc::Event encode_attempted;
7625 EXPECT_CALL(switch_callback,
7626 RequestEncoderSwitch(Field(&SdpVideoFormat::name, "AV2"),
7627 /*allow_default_fallback=*/true))
7628 .WillOnce([&encode_attempted]() { encode_attempted.Set(); });
7629
7630 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7631 encode_attempted.Wait(3000);
7632
7633 AdvanceTime(TimeDelta::Zero());
7634
7635 video_stream_encoder_->Stop();
7636
7637 // The encoders produced by the VideoEncoderProxyFactory have a pointer back
7638 // to it's factory, so in order for the encoder instance in the
7639 // `video_stream_encoder_` to be destroyed before the `encoder_factory` we
7640 // reset the `video_stream_encoder_` here.
7641 video_stream_encoder_.reset();
7642}
7643
7644TEST_F(VideoStreamEncoderTest,
7645 SwitchEncoderOnInitFailureWithoutEncoderSelector) {
7646 NiceMock<MockVideoEncoder> video_encoder;
7647 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7648 video_send_config_.encoder_settings.encoder_switch_request_callback =
7649 &switch_callback;
7650 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7651 &video_encoder, /*encoder_selector=*/nullptr);
7652 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7653
7654 // Reset encoder for new configuration to take effect.
7655 ConfigureEncoder(video_encoder_config_.Copy());
7656
7657 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
7658 kTargetBitrate, kTargetBitrate, kTargetBitrate, /*fraction_lost=*/0,
7659 /*round_trip_time_ms=*/0,
7660 /*cwnd_reduce_ratio=*/0);
7661 ASSERT_EQ(0, sink_.number_of_reconfigurations());
7662
7663 ON_CALL(video_encoder, InitEncode(_, _))
7664 .WillByDefault(Return(WEBRTC_VIDEO_CODEC_ENCODER_FAILURE));
7665
7666 rtc::Event encode_attempted;
7667 EXPECT_CALL(switch_callback,
7668 RequestEncoderSwitch(Field(&SdpVideoFormat::name, "VP8"),
7669 /*allow_default_fallback=*/true))
7670 .WillOnce([&encode_attempted]() { encode_attempted.Set(); });
7671
7672 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7673 encode_attempted.Wait(3000);
7674
7675 AdvanceTime(TimeDelta::Zero());
7676
7677 video_stream_encoder_->Stop();
7678
7679 // The encoders produced by the VideoEncoderProxyFactory have a pointer back
philipel9b058032020-02-10 11:30:00 +01007680 // to it's factory, so in order for the encoder instance in the
Artem Titovab30d722021-07-27 16:22:11 +02007681 // `video_stream_encoder_` to be destroyed before the `encoder_factory` we
7682 // reset the `video_stream_encoder_` here.
philipel9b058032020-02-10 11:30:00 +01007683 video_stream_encoder_.reset();
7684}
7685
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007686TEST_F(VideoStreamEncoderTest,
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01007687 AllocationPropagatedToEncoderWhenTargetRateChanged) {
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007688 const int kFrameWidth = 320;
7689 const int kFrameHeight = 180;
7690
7691 // Set initial rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007692 auto rate = DataRate::KilobitsPerSec(100);
Henrik Boström381d1092020-05-12 18:49:07 +02007693 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007694 /*target_bitrate=*/rate,
7695 /*stable_target_bitrate=*/rate,
7696 /*link_allocation=*/rate,
7697 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007698 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007699 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007700
7701 // Insert a first video frame so that encoder gets configured.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007702 int64_t timestamp_ms = CurrentTimeMs();
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007703 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7704 frame.set_rotation(kVideoRotation_270);
7705 video_source_.IncomingCapturedFrame(frame);
7706 WaitForEncodedFrame(timestamp_ms);
7707 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
7708
7709 // Change of target bitrate propagates to the encoder.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007710 auto new_stable_rate = rate - DataRate::KilobitsPerSec(5);
Henrik Boström381d1092020-05-12 18:49:07 +02007711 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007712 /*target_bitrate=*/new_stable_rate,
7713 /*stable_target_bitrate=*/new_stable_rate,
7714 /*link_allocation=*/rate,
7715 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007716 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007717 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007718 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7719 EXPECT_EQ(2, fake_encoder_.GetNumSetRates());
7720 video_stream_encoder_->Stop();
7721}
7722
7723TEST_F(VideoStreamEncoderTest,
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01007724 AllocationNotPropagatedToEncoderWhenTargetRateUnchanged) {
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007725 const int kFrameWidth = 320;
7726 const int kFrameHeight = 180;
7727
7728 // Set initial rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007729 auto rate = DataRate::KilobitsPerSec(100);
Henrik Boström381d1092020-05-12 18:49:07 +02007730 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007731 /*target_bitrate=*/rate,
7732 /*stable_target_bitrate=*/rate,
7733 /*link_allocation=*/rate,
7734 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007735 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007736 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007737
7738 // Insert a first video frame so that encoder gets configured.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007739 int64_t timestamp_ms = CurrentTimeMs();
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007740 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7741 frame.set_rotation(kVideoRotation_270);
7742 video_source_.IncomingCapturedFrame(frame);
7743 WaitForEncodedFrame(timestamp_ms);
7744 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
7745
7746 // Set a higher target rate without changing the link_allocation. Should not
7747 // reset encoder's rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007748 auto new_stable_rate = rate - DataRate::KilobitsPerSec(5);
Henrik Boström381d1092020-05-12 18:49:07 +02007749 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007750 /*target_bitrate=*/rate,
7751 /*stable_target_bitrate=*/new_stable_rate,
7752 /*link_allocation=*/rate,
7753 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007754 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007755 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007756 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7757 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
7758 video_stream_encoder_->Stop();
7759}
7760
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007761TEST_F(VideoStreamEncoderTest, AutomaticAnimationDetection) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01007762 test::ScopedKeyValueConfig field_trials(
7763 field_trials_,
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007764 "WebRTC-AutomaticAnimationDetectionScreenshare/"
7765 "enabled:true,min_fps:20,min_duration_ms:1000,min_area_ratio:0.8/");
7766 const int kFramerateFps = 30;
7767 const int kWidth = 1920;
7768 const int kHeight = 1080;
7769 const int kNumFrames = 2 * kFramerateFps; // >1 seconds of frames.
7770 // Works on screenshare mode.
7771 ResetEncoder("VP8", 1, 1, 1, /*screenshare*/ true);
7772 // We rely on the automatic resolution adaptation, but we handle framerate
7773 // adaptation manually by mocking the stats proxy.
7774 video_source_.set_adaptation_enabled(true);
7775
7776 // BALANCED degradation preference is required for this feature.
Henrik Boström381d1092020-05-12 18:49:07 +02007777 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007778 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007779 video_stream_encoder_->SetSource(&video_source_,
7780 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02007781 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007782
7783 VideoFrame frame = CreateFrame(1, kWidth, kHeight);
7784 frame.set_update_rect(VideoFrame::UpdateRect{0, 0, kWidth, kHeight});
7785
7786 // Pass enough frames with the full update to trigger animation detection.
7787 for (int i = 0; i < kNumFrames; ++i) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007788 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007789 frame.set_ntp_time_ms(timestamp_ms);
7790 frame.set_timestamp_us(timestamp_ms * 1000);
7791 video_source_.IncomingCapturedFrame(frame);
7792 WaitForEncodedFrame(timestamp_ms);
7793 }
7794
7795 // Resolution should be limited.
7796 rtc::VideoSinkWants expected;
7797 expected.max_framerate_fps = kFramerateFps;
7798 expected.max_pixel_count = 1280 * 720 + 1;
Evan Shrubsole5fd40602020-05-25 16:19:54 +02007799 EXPECT_THAT(video_source_.sink_wants(), FpsEqResolutionLt(expected));
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007800
7801 // Pass one frame with no known update.
7802 // Resolution cap should be removed immediately.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007803 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007804 frame.set_ntp_time_ms(timestamp_ms);
7805 frame.set_timestamp_us(timestamp_ms * 1000);
7806 frame.clear_update_rect();
7807
7808 video_source_.IncomingCapturedFrame(frame);
7809 WaitForEncodedFrame(timestamp_ms);
7810
7811 // Resolution should be unlimited now.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02007812 EXPECT_THAT(video_source_.sink_wants(),
7813 FpsMatchesResolutionMax(Eq(kFramerateFps)));
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007814
7815 video_stream_encoder_->Stop();
7816}
7817
Ilya Nikolaevskiy09eb6e22020-06-05 12:36:32 +02007818TEST_F(VideoStreamEncoderTest, ConfiguresVp9SvcAtOddResolutions) {
7819 const int kWidth = 720; // 540p adapted down.
7820 const int kHeight = 405;
7821 const int kNumFrames = 3;
7822 // Works on screenshare mode.
7823 ResetEncoder("VP9", /*num_streams=*/1, /*num_temporal_layers=*/1,
7824 /*num_spatial_layers=*/2, /*screenshare=*/true);
7825
7826 video_source_.set_adaptation_enabled(true);
7827
7828 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007829 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy09eb6e22020-06-05 12:36:32 +02007830
7831 VideoFrame frame = CreateFrame(1, kWidth, kHeight);
7832
7833 // Pass enough frames with the full update to trigger animation detection.
7834 for (int i = 0; i < kNumFrames; ++i) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007835 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiy09eb6e22020-06-05 12:36:32 +02007836 frame.set_ntp_time_ms(timestamp_ms);
7837 frame.set_timestamp_us(timestamp_ms * 1000);
7838 video_source_.IncomingCapturedFrame(frame);
7839 WaitForEncodedFrame(timestamp_ms);
7840 }
7841
7842 video_stream_encoder_->Stop();
7843}
7844
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007845TEST_F(VideoStreamEncoderTest, EncoderResetAccordingToParameterChange) {
7846 const float downscale_factors[] = {4.0, 2.0, 1.0};
7847 const int number_layers =
7848 sizeof(downscale_factors) / sizeof(downscale_factors[0]);
7849 VideoEncoderConfig config;
7850 test::FillEncoderConfiguration(kVideoCodecVP8, number_layers, &config);
7851 for (int i = 0; i < number_layers; ++i) {
7852 config.simulcast_layers[i].scale_resolution_down_by = downscale_factors[i];
7853 config.simulcast_layers[i].active = true;
7854 }
7855 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02007856 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007857 "VP8", /*max qp*/ 56, /*screencast*/ false,
7858 /*screenshare enabled*/ false);
7859 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007860 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
7861 0, 0, 0);
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007862
7863 // First initialization.
7864 // Encoder should be initialized. Next frame should be key frame.
7865 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7866 sink_.SetNumExpectedLayers(number_layers);
7867 int64_t timestamp_ms = kFrameIntervalMs;
7868 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7869 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02007870 EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007871 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7872 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7873 VideoFrameType::kVideoFrameKey,
7874 VideoFrameType::kVideoFrameKey}));
7875
7876 // Disable top layer.
7877 // Encoder shouldn't be re-initialized. Next frame should be delta frame.
7878 config.simulcast_layers[number_layers - 1].active = false;
7879 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7880 sink_.SetNumExpectedLayers(number_layers - 1);
7881 timestamp_ms += kFrameIntervalMs;
7882 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7883 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02007884 EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007885 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7886 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
7887 VideoFrameType::kVideoFrameDelta,
7888 VideoFrameType::kVideoFrameDelta}));
7889
7890 // Re-enable top layer.
7891 // Encoder should be re-initialized. Next frame should be key frame.
7892 config.simulcast_layers[number_layers - 1].active = true;
7893 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7894 sink_.SetNumExpectedLayers(number_layers);
7895 timestamp_ms += kFrameIntervalMs;
7896 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7897 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02007898 EXPECT_EQ(2, fake_encoder_.GetNumInitializations());
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007899 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7900 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7901 VideoFrameType::kVideoFrameKey,
7902 VideoFrameType::kVideoFrameKey}));
7903
7904 // Top layer max rate change.
7905 // Encoder shouldn't be re-initialized. Next frame should be delta frame.
7906 config.simulcast_layers[number_layers - 1].max_bitrate_bps -= 100;
7907 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7908 sink_.SetNumExpectedLayers(number_layers);
7909 timestamp_ms += kFrameIntervalMs;
7910 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7911 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02007912 EXPECT_EQ(2, fake_encoder_.GetNumInitializations());
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007913 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7914 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
7915 VideoFrameType::kVideoFrameDelta,
7916 VideoFrameType::kVideoFrameDelta}));
7917
7918 // Top layer resolution change.
7919 // Encoder should be re-initialized. Next frame should be key frame.
7920 config.simulcast_layers[number_layers - 1].scale_resolution_down_by += 0.1;
7921 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7922 sink_.SetNumExpectedLayers(number_layers);
7923 timestamp_ms += kFrameIntervalMs;
7924 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7925 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02007926 EXPECT_EQ(3, fake_encoder_.GetNumInitializations());
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007927 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7928 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7929 VideoFrameType::kVideoFrameKey,
7930 VideoFrameType::kVideoFrameKey}));
7931 video_stream_encoder_->Stop();
7932}
7933
Henrik Boström1124ed12021-02-25 10:30:39 +01007934TEST_F(VideoStreamEncoderTest, EncoderResolutionsExposedInSinglecast) {
7935 const int kFrameWidth = 1280;
7936 const int kFrameHeight = 720;
7937
7938 SetUp();
7939 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007940 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Henrik Boström1124ed12021-02-25 10:30:39 +01007941
7942 // Capturing a frame should reconfigure the encoder and expose the encoder
7943 // resolution, which is the same as the input frame.
7944 int64_t timestamp_ms = kFrameIntervalMs;
7945 video_source_.IncomingCapturedFrame(
7946 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7947 WaitForEncodedFrame(timestamp_ms);
7948 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7949 EXPECT_THAT(video_source_.sink_wants().resolutions,
7950 ::testing::ElementsAreArray(
7951 {rtc::VideoSinkWants::FrameSize(kFrameWidth, kFrameHeight)}));
7952
7953 video_stream_encoder_->Stop();
7954}
7955
7956TEST_F(VideoStreamEncoderTest, EncoderResolutionsExposedInSimulcast) {
7957 // Pick downscale factors such that we never encode at full resolution - this
7958 // is an interesting use case. The frame resolution influences the encoder
Artem Titovab30d722021-07-27 16:22:11 +02007959 // resolutions, but if no layer has `scale_resolution_down_by` == 1 then the
Henrik Boström1124ed12021-02-25 10:30:39 +01007960 // encoder should not ask for the frame resolution. This allows video frames
7961 // to have the appearence of one resolution but optimize its internal buffers
7962 // for what is actually encoded.
7963 const size_t kNumSimulcastLayers = 3u;
7964 const float kDownscaleFactors[] = {8.0, 4.0, 2.0};
7965 const int kFrameWidth = 1280;
7966 const int kFrameHeight = 720;
7967 const rtc::VideoSinkWants::FrameSize kLayer0Size(
7968 kFrameWidth / kDownscaleFactors[0], kFrameHeight / kDownscaleFactors[0]);
7969 const rtc::VideoSinkWants::FrameSize kLayer1Size(
7970 kFrameWidth / kDownscaleFactors[1], kFrameHeight / kDownscaleFactors[1]);
7971 const rtc::VideoSinkWants::FrameSize kLayer2Size(
7972 kFrameWidth / kDownscaleFactors[2], kFrameHeight / kDownscaleFactors[2]);
7973
7974 VideoEncoderConfig config;
7975 test::FillEncoderConfiguration(kVideoCodecVP8, kNumSimulcastLayers, &config);
7976 for (size_t i = 0; i < kNumSimulcastLayers; ++i) {
7977 config.simulcast_layers[i].scale_resolution_down_by = kDownscaleFactors[i];
7978 config.simulcast_layers[i].active = true;
7979 }
7980 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02007981 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Henrik Boström1124ed12021-02-25 10:30:39 +01007982 "VP8", /*max qp*/ 56, /*screencast*/ false,
7983 /*screenshare enabled*/ false);
7984 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007985 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
7986 0, 0, 0);
Henrik Boström1124ed12021-02-25 10:30:39 +01007987
7988 // Capture a frame with all layers active.
7989 int64_t timestamp_ms = kFrameIntervalMs;
7990 sink_.SetNumExpectedLayers(kNumSimulcastLayers);
7991 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7992 video_source_.IncomingCapturedFrame(
7993 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7994 WaitForEncodedFrame(timestamp_ms);
7995 // Expect encoded resolutions to match the expected simulcast layers.
7996 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7997 EXPECT_THAT(
7998 video_source_.sink_wants().resolutions,
7999 ::testing::ElementsAreArray({kLayer0Size, kLayer1Size, kLayer2Size}));
8000
8001 // Capture a frame with one of the layers inactive.
8002 timestamp_ms += kFrameIntervalMs;
8003 config.simulcast_layers[2].active = false;
8004 sink_.SetNumExpectedLayers(kNumSimulcastLayers - 1);
8005 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8006 video_source_.IncomingCapturedFrame(
8007 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8008 WaitForEncodedFrame(timestamp_ms);
8009
8010 // Expect encoded resolutions to match the expected simulcast layers.
8011 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8012 EXPECT_THAT(video_source_.sink_wants().resolutions,
8013 ::testing::ElementsAreArray({kLayer0Size, kLayer1Size}));
8014
8015 // Capture a frame with all but one layer turned off.
8016 timestamp_ms += kFrameIntervalMs;
8017 config.simulcast_layers[1].active = false;
8018 sink_.SetNumExpectedLayers(kNumSimulcastLayers - 2);
8019 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8020 video_source_.IncomingCapturedFrame(
8021 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8022 WaitForEncodedFrame(timestamp_ms);
8023
8024 // Expect encoded resolutions to match the expected simulcast layers.
8025 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8026 EXPECT_THAT(video_source_.sink_wants().resolutions,
8027 ::testing::ElementsAreArray({kLayer0Size}));
8028
8029 video_stream_encoder_->Stop();
8030}
8031
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008032TEST_F(VideoStreamEncoderTest, QpPresent_QpKept) {
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008033 ResetEncoder("VP8", 1, 1, 1, false);
8034
Niels Möller8b692902021-06-14 12:04:57 +02008035 // Force encoder reconfig.
8036 video_source_.IncomingCapturedFrame(
8037 CreateFrame(1, codec_width_, codec_height_));
8038 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8039
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008040 // Set QP on encoded frame and pass the frame to encode complete callback.
8041 // Since QP is present QP parsing won't be triggered and the original value
8042 // should be kept.
8043 EncodedImage encoded_image;
8044 encoded_image.qp_ = 123;
8045 encoded_image.SetEncodedData(EncodedImageBuffer::Create(
8046 kCodedFrameVp8Qp25, sizeof(kCodedFrameVp8Qp25)));
8047 CodecSpecificInfo codec_info;
8048 codec_info.codecType = kVideoCodecVP8;
8049 fake_encoder_.InjectEncodedImage(encoded_image, &codec_info);
8050 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
8051 EXPECT_EQ(sink_.GetLastEncodedImage().qp_, 123);
8052 video_stream_encoder_->Stop();
8053}
8054
8055TEST_F(VideoStreamEncoderTest, QpAbsent_QpParsed) {
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008056 ResetEncoder("VP8", 1, 1, 1, false);
8057
Niels Möller8b692902021-06-14 12:04:57 +02008058 // Force encoder reconfig.
8059 video_source_.IncomingCapturedFrame(
8060 CreateFrame(1, codec_width_, codec_height_));
8061 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8062
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008063 // Pass an encoded frame without QP to encode complete callback. QP should be
8064 // parsed and set.
8065 EncodedImage encoded_image;
8066 encoded_image.qp_ = -1;
8067 encoded_image.SetEncodedData(EncodedImageBuffer::Create(
8068 kCodedFrameVp8Qp25, sizeof(kCodedFrameVp8Qp25)));
8069 CodecSpecificInfo codec_info;
8070 codec_info.codecType = kVideoCodecVP8;
8071 fake_encoder_.InjectEncodedImage(encoded_image, &codec_info);
8072 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
8073 EXPECT_EQ(sink_.GetLastEncodedImage().qp_, 25);
8074 video_stream_encoder_->Stop();
8075}
8076
8077TEST_F(VideoStreamEncoderTest, QpAbsentParsingDisabled_QpAbsent) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01008078 webrtc::test::ScopedKeyValueConfig field_trials(
8079 field_trials_, "WebRTC-QpParsingKillSwitch/Enabled/");
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008080
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008081 ResetEncoder("VP8", 1, 1, 1, false);
8082
Niels Möller8b692902021-06-14 12:04:57 +02008083 // Force encoder reconfig.
8084 video_source_.IncomingCapturedFrame(
8085 CreateFrame(1, codec_width_, codec_height_));
8086 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8087
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008088 EncodedImage encoded_image;
8089 encoded_image.qp_ = -1;
8090 encoded_image.SetEncodedData(EncodedImageBuffer::Create(
8091 kCodedFrameVp8Qp25, sizeof(kCodedFrameVp8Qp25)));
8092 CodecSpecificInfo codec_info;
8093 codec_info.codecType = kVideoCodecVP8;
8094 fake_encoder_.InjectEncodedImage(encoded_image, &codec_info);
8095 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
8096 EXPECT_EQ(sink_.GetLastEncodedImage().qp_, -1);
8097 video_stream_encoder_->Stop();
8098}
8099
Sergey Silkind19e3b92021-03-16 10:05:30 +00008100TEST_F(VideoStreamEncoderTest,
8101 QualityScalingNotAllowed_QualityScalingDisabled) {
8102 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8103
8104 // Disable scaling settings in encoder info.
8105 fake_encoder_.SetQualityScaling(false);
8106 // Disable quality scaling in encoder config.
8107 video_encoder_config.is_quality_scaling_allowed = false;
8108 ConfigureEncoder(std::move(video_encoder_config));
8109
8110 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008111 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkind19e3b92021-03-16 10:05:30 +00008112
8113 test::FrameForwarder source;
8114 video_stream_encoder_->SetSource(
8115 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8116 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8117 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8118
8119 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8120 WaitForEncodedFrame(1);
8121 video_stream_encoder_->TriggerQualityLow();
8122 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8123
8124 video_stream_encoder_->Stop();
8125}
8126
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08008127TEST_F(VideoStreamEncoderTest, QualityScalingNotAllowed_IsQpTrustedSetTrue) {
8128 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8129
8130 // Disable scaling settings in encoder info.
8131 fake_encoder_.SetQualityScaling(false);
8132 // Set QP trusted in encoder info.
8133 fake_encoder_.SetIsQpTrusted(true);
8134 // Enable quality scaling in encoder config.
8135 video_encoder_config.is_quality_scaling_allowed = false;
8136 ConfigureEncoder(std::move(video_encoder_config));
8137
8138 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008139 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08008140
8141 test::FrameForwarder source;
8142 video_stream_encoder_->SetSource(
8143 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8144 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8145 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8146
8147 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8148 WaitForEncodedFrame(1);
8149 video_stream_encoder_->TriggerQualityLow();
8150 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8151
8152 video_stream_encoder_->Stop();
8153}
8154
Shuhai Pengf2707702021-09-29 17:19:44 +08008155TEST_F(VideoStreamEncoderTest,
8156 QualityScalingNotAllowedAndQPIsTrusted_BandwidthScalerDisable) {
8157 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8158
8159 // Disable scaling settings in encoder info.
8160 fake_encoder_.SetQualityScaling(false);
8161 // Set QP trusted in encoder info.
8162 fake_encoder_.SetIsQpTrusted(true);
8163 // Enable quality scaling in encoder config.
8164 video_encoder_config.is_quality_scaling_allowed = false;
8165 ConfigureEncoder(std::move(video_encoder_config));
8166
8167 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008168 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008169
8170 test::FrameForwarder source;
8171 video_stream_encoder_->SetSource(
8172 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8173 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8174 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8175
8176 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8177 WaitForEncodedFrame(1);
8178 video_stream_encoder_->TriggerQualityLow();
8179 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8180
8181 video_stream_encoder_->Stop();
8182}
8183
8184TEST_F(VideoStreamEncoderTest,
8185 QualityScalingNotAllowedAndQPIsNotTrusted_BandwidthScalerDisable) {
8186 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8187
8188 // Disable scaling settings in encoder info.
8189 fake_encoder_.SetQualityScaling(false);
8190 // Set QP trusted in encoder info.
8191 fake_encoder_.SetIsQpTrusted(false);
8192 // Enable quality scaling in encoder config.
8193 video_encoder_config.is_quality_scaling_allowed = false;
8194 ConfigureEncoder(std::move(video_encoder_config));
8195
8196 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008197 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008198
8199 test::FrameForwarder source;
8200 video_stream_encoder_->SetSource(
8201 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8202 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8203 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8204
8205 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8206 WaitForEncodedFrame(1);
8207 video_stream_encoder_->TriggerQualityLow();
8208 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8209
8210 video_stream_encoder_->Stop();
8211}
8212
8213TEST_F(VideoStreamEncoderTest, EncoderProvideLimitsWhenQPIsNotTrusted) {
8214 // Set QP trusted in encoder info.
8215 fake_encoder_.SetIsQpTrusted(false);
8216
8217 const int MinEncBitrateKbps = 30;
8218 const int MaxEncBitrateKbps = 100;
8219 const int MinStartBitrateKbp = 50;
8220 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
8221 /*frame_size_pixels=*/codec_width_ * codec_height_,
8222 /*min_start_bitrate_bps=*/MinStartBitrateKbp,
8223 /*min_bitrate_bps=*/MinEncBitrateKbps * 1000,
8224 /*max_bitrate_bps=*/MaxEncBitrateKbps * 1000);
8225
8226 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008227 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008228
8229 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
8230
8231 VideoEncoderConfig video_encoder_config;
8232 test::FillEncoderConfiguration(kVideoCodecH264, 1, &video_encoder_config);
8233 video_encoder_config.max_bitrate_bps = MaxEncBitrateKbps * 1000;
8234 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
8235 MinEncBitrateKbps * 1000;
8236 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
8237 kMaxPayloadLength);
8238
8239 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
8240 WaitForEncodedFrame(1);
8241 EXPECT_EQ(
8242 MaxEncBitrateKbps,
8243 static_cast<int>(bitrate_allocator_factory_.codec_config().maxBitrate));
8244 EXPECT_EQ(
8245 MinEncBitrateKbps,
8246 static_cast<int>(bitrate_allocator_factory_.codec_config().minBitrate));
8247
8248 video_stream_encoder_->Stop();
8249}
8250
8251TEST_F(VideoStreamEncoderTest, EncoderDoesnotProvideLimitsWhenQPIsNotTrusted) {
8252 // Set QP trusted in encoder info.
8253 fake_encoder_.SetIsQpTrusted(false);
8254
8255 absl::optional<VideoEncoder::ResolutionBitrateLimits> suitable_bitrate_limit =
8256 EncoderInfoSettings::
8257 GetSinglecastBitrateLimitForResolutionWhenQpIsUntrusted(
8258 codec_width_ * codec_height_,
8259 EncoderInfoSettings::
8260 GetDefaultSinglecastBitrateLimitsWhenQpIsUntrusted());
8261 EXPECT_TRUE(suitable_bitrate_limit.has_value());
8262
8263 const int MaxEncBitrate = suitable_bitrate_limit->max_bitrate_bps;
8264 const int MinEncBitrate = suitable_bitrate_limit->min_bitrate_bps;
8265 const int TargetEncBitrate = MaxEncBitrate;
8266 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8267 DataRate::BitsPerSec(TargetEncBitrate),
8268 DataRate::BitsPerSec(TargetEncBitrate),
8269 DataRate::BitsPerSec(TargetEncBitrate), 0, 0, 0);
8270
8271 VideoEncoderConfig video_encoder_config;
8272 test::FillEncoderConfiguration(kVideoCodecH264, 1, &video_encoder_config);
8273 video_encoder_config.max_bitrate_bps = MaxEncBitrate;
8274 video_encoder_config.simulcast_layers[0].min_bitrate_bps = MinEncBitrate;
8275 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
8276 kMaxPayloadLength);
8277
8278 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
8279 WaitForEncodedFrame(1);
8280 EXPECT_EQ(
8281 MaxEncBitrate / 1000,
8282 static_cast<int>(bitrate_allocator_factory_.codec_config().maxBitrate));
8283 EXPECT_EQ(
8284 MinEncBitrate / 1000,
8285 static_cast<int>(bitrate_allocator_factory_.codec_config().minBitrate));
8286
8287 video_stream_encoder_->Stop();
8288}
8289
Sergey Silkind19e3b92021-03-16 10:05:30 +00008290#if !defined(WEBRTC_IOS)
8291// TODO(bugs.webrtc.org/12401): Disabled because WebRTC-Video-QualityScaling is
8292// disabled by default on iOS.
8293TEST_F(VideoStreamEncoderTest, QualityScalingAllowed_QualityScalingEnabled) {
8294 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8295
8296 // Disable scaling settings in encoder info.
8297 fake_encoder_.SetQualityScaling(false);
8298 // Enable quality scaling in encoder config.
8299 video_encoder_config.is_quality_scaling_allowed = true;
8300 ConfigureEncoder(std::move(video_encoder_config));
8301
8302 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008303 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkind19e3b92021-03-16 10:05:30 +00008304
8305 test::FrameForwarder source;
8306 video_stream_encoder_->SetSource(
8307 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8308 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8309 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8310
8311 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8312 WaitForEncodedFrame(1);
8313 video_stream_encoder_->TriggerQualityLow();
8314 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8315
8316 video_stream_encoder_->Stop();
8317}
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08008318
8319TEST_F(VideoStreamEncoderTest, QualityScalingAllowed_IsQpTrustedSetTrue) {
8320 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8321
8322 // Disable scaling settings in encoder info.
8323 fake_encoder_.SetQualityScaling(false);
8324 // Set QP trusted in encoder info.
8325 fake_encoder_.SetIsQpTrusted(true);
8326 // Enable quality scaling in encoder config.
8327 video_encoder_config.is_quality_scaling_allowed = true;
8328 ConfigureEncoder(std::move(video_encoder_config));
8329
8330 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008331 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08008332
8333 test::FrameForwarder source;
8334 video_stream_encoder_->SetSource(
8335 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8336 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8337 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8338
8339 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8340 WaitForEncodedFrame(1);
8341 video_stream_encoder_->TriggerQualityLow();
8342 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8343
8344 video_stream_encoder_->Stop();
8345}
Shuhai Pengf2707702021-09-29 17:19:44 +08008346
8347TEST_F(VideoStreamEncoderTest, QualityScalingAllowed_IsQpTrustedSetFalse) {
8348 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8349
8350 // Disable scaling settings in encoder info.
8351 fake_encoder_.SetQualityScaling(false);
8352 // Set QP not trusted in encoder info.
8353 fake_encoder_.SetIsQpTrusted(false);
8354 // Enable quality scaling in encoder config.
8355 video_encoder_config.is_quality_scaling_allowed = true;
8356 ConfigureEncoder(std::move(video_encoder_config));
8357
8358 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008359 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008360
8361 test::FrameForwarder source;
8362 video_stream_encoder_->SetSource(
8363 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8364 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8365 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8366
8367 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8368 WaitForEncodedFrame(1);
8369 video_stream_encoder_->TriggerQualityLow();
8370 // When quality_scaler doesn't work and is_quality_scaling_allowed is
8371 // true,the bandwidth_quality_scaler_ works,so bw_limited_resolution is true.
8372 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8373
8374 video_stream_encoder_->Stop();
8375}
8376
8377TEST_F(VideoStreamEncoderTest,
8378 QualityScalingAllowedAndQPIsTrusted_BandwidthScalerDisable) {
8379 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8380
8381 // Disable scaling settings in encoder info.
8382 fake_encoder_.SetQualityScaling(false);
8383 // Set QP trusted in encoder info.
8384 fake_encoder_.SetIsQpTrusted(true);
8385 // Enable quality scaling in encoder config.
8386 video_encoder_config.is_quality_scaling_allowed = true;
8387 ConfigureEncoder(std::move(video_encoder_config));
8388
8389 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008390 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008391
8392 test::FrameForwarder source;
8393 video_stream_encoder_->SetSource(
8394 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8395 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8396 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8397
8398 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8399 WaitForEncodedFrame(1);
8400 video_stream_encoder_->TriggerQualityLow();
8401 // bandwidth_quality_scaler isn't working, but quality_scaler is working.
8402 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8403
8404 video_stream_encoder_->Stop();
8405}
8406
8407TEST_F(VideoStreamEncoderTest,
8408 QualityScalingAllowedAndQPIsNotTrusted_BandwidthScalerEnabled) {
8409 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8410
8411 // Disable scaling settings in encoder info.
8412 fake_encoder_.SetQualityScaling(false);
8413 // Set QP trusted in encoder info.
8414 fake_encoder_.SetIsQpTrusted(false);
8415 // Enable quality scaling in encoder config.
8416 video_encoder_config.is_quality_scaling_allowed = true;
8417 ConfigureEncoder(std::move(video_encoder_config));
8418
8419 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008420 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008421
8422 test::FrameForwarder source;
8423 video_stream_encoder_->SetSource(
8424 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8425 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8426 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8427
8428 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8429 WaitForEncodedFrame(1);
8430 video_stream_encoder_->TriggerQualityLow();
8431 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8432
8433 video_stream_encoder_->Stop();
8434}
8435
Sergey Silkind19e3b92021-03-16 10:05:30 +00008436#endif
8437
Henrik Boström56db9ff2021-03-24 09:06:45 +01008438// Test parameters: (VideoCodecType codec, bool allow_i420_conversion)
8439class VideoStreamEncoderWithRealEncoderTest
8440 : public VideoStreamEncoderTest,
8441 public ::testing::WithParamInterface<std::pair<VideoCodecType, bool>> {
8442 public:
8443 VideoStreamEncoderWithRealEncoderTest()
8444 : VideoStreamEncoderTest(),
8445 codec_type_(std::get<0>(GetParam())),
8446 allow_i420_conversion_(std::get<1>(GetParam())) {}
8447
8448 void SetUp() override {
8449 VideoStreamEncoderTest::SetUp();
8450 std::unique_ptr<VideoEncoder> encoder;
8451 switch (codec_type_) {
8452 case kVideoCodecVP8:
8453 encoder = VP8Encoder::Create();
8454 break;
8455 case kVideoCodecVP9:
8456 encoder = VP9Encoder::Create();
8457 break;
8458 case kVideoCodecAV1:
philipel95701502022-01-18 18:47:52 +01008459 encoder = CreateLibaomAv1EncoderIfSupported();
Henrik Boström56db9ff2021-03-24 09:06:45 +01008460 break;
8461 case kVideoCodecH264:
8462 encoder =
8463 H264Encoder::Create(cricket::VideoCodec(cricket::kH264CodecName));
8464 break;
8465 case kVideoCodecMultiplex:
8466 mock_encoder_factory_for_multiplex_ =
8467 std::make_unique<MockVideoEncoderFactory>();
8468 EXPECT_CALL(*mock_encoder_factory_for_multiplex_, Die);
8469 EXPECT_CALL(*mock_encoder_factory_for_multiplex_, CreateVideoEncoder)
8470 .WillRepeatedly([] { return VP8Encoder::Create(); });
8471 encoder = std::make_unique<MultiplexEncoderAdapter>(
8472 mock_encoder_factory_for_multiplex_.get(), SdpVideoFormat("VP8"),
8473 false);
8474 break;
8475 default:
Artem Titovd3251962021-11-15 16:57:07 +01008476 RTC_DCHECK_NOTREACHED();
Henrik Boström56db9ff2021-03-24 09:06:45 +01008477 }
8478 ConfigureEncoderAndBitrate(codec_type_, std::move(encoder));
8479 }
8480
8481 void TearDown() override {
8482 video_stream_encoder_->Stop();
Artem Titovab30d722021-07-27 16:22:11 +02008483 // Ensure `video_stream_encoder_` is destroyed before
8484 // `encoder_proxy_factory_`.
Henrik Boström56db9ff2021-03-24 09:06:45 +01008485 video_stream_encoder_.reset();
8486 VideoStreamEncoderTest::TearDown();
8487 }
8488
8489 protected:
8490 void ConfigureEncoderAndBitrate(VideoCodecType codec_type,
8491 std::unique_ptr<VideoEncoder> encoder) {
8492 // Configure VSE to use the encoder.
8493 encoder_ = std::move(encoder);
8494 encoder_proxy_factory_ = std::make_unique<test::VideoEncoderProxyFactory>(
8495 encoder_.get(), &encoder_selector_);
8496 video_send_config_.encoder_settings.encoder_factory =
8497 encoder_proxy_factory_.get();
8498 VideoEncoderConfig video_encoder_config;
8499 test::FillEncoderConfiguration(codec_type, 1, &video_encoder_config);
8500 video_encoder_config_ = video_encoder_config.Copy();
8501 ConfigureEncoder(video_encoder_config_.Copy());
8502
8503 // Set bitrate to ensure frame is not dropped.
8504 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008505 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Henrik Boström56db9ff2021-03-24 09:06:45 +01008506 }
8507
8508 const VideoCodecType codec_type_;
8509 const bool allow_i420_conversion_;
8510 NiceMock<MockEncoderSelector> encoder_selector_;
8511 std::unique_ptr<test::VideoEncoderProxyFactory> encoder_proxy_factory_;
8512 std::unique_ptr<VideoEncoder> encoder_;
8513 std::unique_ptr<MockVideoEncoderFactory> mock_encoder_factory_for_multiplex_;
8514};
8515
8516TEST_P(VideoStreamEncoderWithRealEncoderTest, EncoderMapsNativeI420) {
8517 auto native_i420_frame = test::CreateMappableNativeFrame(
8518 1, VideoFrameBuffer::Type::kI420, codec_width_, codec_height_);
8519 video_source_.IncomingCapturedFrame(native_i420_frame);
8520 WaitForEncodedFrame(codec_width_, codec_height_);
8521
8522 auto mappable_native_buffer =
8523 test::GetMappableNativeBufferFromVideoFrame(native_i420_frame);
8524 std::vector<rtc::scoped_refptr<VideoFrameBuffer>> mapped_frame_buffers =
8525 mappable_native_buffer->GetMappedFramedBuffers();
8526 ASSERT_EQ(mapped_frame_buffers.size(), 1u);
8527 EXPECT_EQ(mapped_frame_buffers[0]->width(), codec_width_);
8528 EXPECT_EQ(mapped_frame_buffers[0]->height(), codec_height_);
8529 EXPECT_EQ(mapped_frame_buffers[0]->type(), VideoFrameBuffer::Type::kI420);
8530}
8531
8532TEST_P(VideoStreamEncoderWithRealEncoderTest, EncoderMapsNativeNV12) {
8533 auto native_nv12_frame = test::CreateMappableNativeFrame(
8534 1, VideoFrameBuffer::Type::kNV12, codec_width_, codec_height_);
8535 video_source_.IncomingCapturedFrame(native_nv12_frame);
8536 WaitForEncodedFrame(codec_width_, codec_height_);
8537
8538 auto mappable_native_buffer =
8539 test::GetMappableNativeBufferFromVideoFrame(native_nv12_frame);
8540 std::vector<rtc::scoped_refptr<VideoFrameBuffer>> mapped_frame_buffers =
8541 mappable_native_buffer->GetMappedFramedBuffers();
8542 ASSERT_EQ(mapped_frame_buffers.size(), 1u);
8543 EXPECT_EQ(mapped_frame_buffers[0]->width(), codec_width_);
8544 EXPECT_EQ(mapped_frame_buffers[0]->height(), codec_height_);
8545 EXPECT_EQ(mapped_frame_buffers[0]->type(), VideoFrameBuffer::Type::kNV12);
8546
8547 if (!allow_i420_conversion_) {
8548 EXPECT_FALSE(mappable_native_buffer->DidConvertToI420());
8549 }
8550}
8551
Erik Språng7444b192021-06-02 14:02:13 +02008552TEST_P(VideoStreamEncoderWithRealEncoderTest, HandlesLayerToggling) {
8553 if (codec_type_ == kVideoCodecMultiplex) {
8554 // Multiplex codec here uses wrapped mock codecs, ignore for this test.
8555 return;
8556 }
8557
8558 const size_t kNumSpatialLayers = 3u;
8559 const float kDownscaleFactors[] = {4.0, 2.0, 1.0};
8560 const int kFrameWidth = 1280;
8561 const int kFrameHeight = 720;
8562 const rtc::VideoSinkWants::FrameSize kLayer0Size(
8563 kFrameWidth / kDownscaleFactors[0], kFrameHeight / kDownscaleFactors[0]);
8564 const rtc::VideoSinkWants::FrameSize kLayer1Size(
8565 kFrameWidth / kDownscaleFactors[1], kFrameHeight / kDownscaleFactors[1]);
8566 const rtc::VideoSinkWants::FrameSize kLayer2Size(
8567 kFrameWidth / kDownscaleFactors[2], kFrameHeight / kDownscaleFactors[2]);
8568
8569 VideoEncoderConfig config;
8570 if (codec_type_ == VideoCodecType::kVideoCodecVP9) {
8571 test::FillEncoderConfiguration(codec_type_, 1, &config);
Asa Persson606d3cb2021-10-04 10:07:11 +02008572 config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Erik Språng7444b192021-06-02 14:02:13 +02008573 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
8574 vp9_settings.numberOfSpatialLayers = kNumSpatialLayers;
8575 vp9_settings.numberOfTemporalLayers = 3;
8576 vp9_settings.automaticResizeOn = false;
8577 config.encoder_specific_settings =
8578 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
8579 vp9_settings);
8580 config.spatial_layers = GetSvcConfig(kFrameWidth, kFrameHeight,
8581 /*fps=*/30.0,
8582 /*first_active_layer=*/0,
8583 /*num_spatial_layers=*/3,
8584 /*num_temporal_layers=*/3,
8585 /*is_screenshare=*/false);
8586 } else if (codec_type_ == VideoCodecType::kVideoCodecAV1) {
8587 test::FillEncoderConfiguration(codec_type_, 1, &config);
Asa Persson606d3cb2021-10-04 10:07:11 +02008588 config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Erik Språng7444b192021-06-02 14:02:13 +02008589 config.spatial_layers = GetSvcConfig(kFrameWidth, kFrameHeight,
8590 /*fps=*/30.0,
8591 /*first_active_layer=*/0,
8592 /*num_spatial_layers=*/3,
8593 /*num_temporal_layers=*/3,
8594 /*is_screenshare=*/false);
8595 config.simulcast_layers[0].scalability_mode = "L3T3_KEY";
8596 } else {
8597 // Simulcast for VP8/H264.
8598 test::FillEncoderConfiguration(codec_type_, kNumSpatialLayers, &config);
8599 for (size_t i = 0; i < kNumSpatialLayers; ++i) {
8600 config.simulcast_layers[i].scale_resolution_down_by =
8601 kDownscaleFactors[i];
8602 config.simulcast_layers[i].active = true;
8603 }
8604 if (codec_type_ == VideoCodecType::kVideoCodecH264) {
8605 // Turn off frame dropping to prevent flakiness.
8606 VideoCodecH264 h264_settings = VideoEncoder::GetDefaultH264Settings();
8607 h264_settings.frameDroppingOn = false;
8608 config.encoder_specific_settings = rtc::make_ref_counted<
8609 VideoEncoderConfig::H264EncoderSpecificSettings>(h264_settings);
8610 }
8611 }
8612
8613 auto set_layer_active = [&](int layer_idx, bool active) {
8614 if (codec_type_ == VideoCodecType::kVideoCodecVP9 ||
8615 codec_type_ == VideoCodecType::kVideoCodecAV1) {
8616 config.spatial_layers[layer_idx].active = active;
8617 } else {
8618 config.simulcast_layers[layer_idx].active = active;
8619 }
8620 };
8621
8622 config.video_stream_factory =
8623 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
8624 CodecTypeToPayloadString(codec_type_), /*max qp*/ 56,
8625 /*screencast*/ false,
8626 /*screenshare enabled*/ false);
8627 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008628 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
8629 0, 0, 0);
Erik Språng7444b192021-06-02 14:02:13 +02008630
8631 // Capture a frame with all layers active.
8632 sink_.SetNumExpectedLayers(kNumSpatialLayers);
8633 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8634 int64_t timestamp_ms = kFrameIntervalMs;
8635 video_source_.IncomingCapturedFrame(
8636 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8637
8638 WaitForEncodedFrame(kLayer2Size.width, kLayer2Size.height);
8639 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8640
8641 // Capture a frame with one of the layers inactive.
8642 set_layer_active(2, false);
8643 sink_.SetNumExpectedLayers(kNumSpatialLayers - 1);
8644 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8645 timestamp_ms += kFrameIntervalMs;
8646 video_source_.IncomingCapturedFrame(
8647 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8648 WaitForEncodedFrame(kLayer1Size.width, kLayer1Size.height);
8649
8650 // New target bitrates signaled based on lower resolution.
8651 DataRate kTwoLayerBitrate = DataRate::KilobitsPerSec(833);
8652 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8653 kTwoLayerBitrate, kTwoLayerBitrate, kTwoLayerBitrate, 0, 0, 0);
8654 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8655
8656 // Re-enable the top layer.
8657 set_layer_active(2, true);
8658 sink_.SetNumExpectedLayers(kNumSpatialLayers);
8659 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8660 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8661
8662 // Bitrate target adjusted back up to enable HD layer...
8663 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8664 DataRate::KilobitsPerSec(1800), DataRate::KilobitsPerSec(1800),
8665 DataRate::KilobitsPerSec(1800), 0, 0, 0);
8666 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8667
8668 // ...then add a new frame.
8669 timestamp_ms += kFrameIntervalMs;
8670 video_source_.IncomingCapturedFrame(
8671 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8672 WaitForEncodedFrame(kLayer2Size.width, kLayer2Size.height);
8673 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8674
8675 video_stream_encoder_->Stop();
8676}
8677
Henrik Boström56db9ff2021-03-24 09:06:45 +01008678std::string TestParametersVideoCodecAndAllowI420ConversionToString(
8679 testing::TestParamInfo<std::pair<VideoCodecType, bool>> info) {
8680 VideoCodecType codec_type = std::get<0>(info.param);
8681 bool allow_i420_conversion = std::get<1>(info.param);
8682 std::string str;
8683 switch (codec_type) {
8684 case kVideoCodecGeneric:
8685 str = "Generic";
8686 break;
8687 case kVideoCodecVP8:
8688 str = "VP8";
8689 break;
8690 case kVideoCodecVP9:
8691 str = "VP9";
8692 break;
8693 case kVideoCodecAV1:
8694 str = "AV1";
8695 break;
8696 case kVideoCodecH264:
8697 str = "H264";
8698 break;
8699 case kVideoCodecMultiplex:
8700 str = "Multiplex";
8701 break;
8702 default:
Artem Titovd3251962021-11-15 16:57:07 +01008703 RTC_DCHECK_NOTREACHED();
Henrik Boström56db9ff2021-03-24 09:06:45 +01008704 }
8705 str += allow_i420_conversion ? "_AllowToI420" : "_DisallowToI420";
8706 return str;
8707}
8708
8709constexpr std::pair<VideoCodecType, bool> kVP8DisallowConversion =
8710 std::make_pair(kVideoCodecVP8, /*allow_i420_conversion=*/false);
8711constexpr std::pair<VideoCodecType, bool> kVP9DisallowConversion =
8712 std::make_pair(kVideoCodecVP9, /*allow_i420_conversion=*/false);
8713constexpr std::pair<VideoCodecType, bool> kAV1AllowConversion =
Andrey Logvinfef00262022-03-09 11:37:17 +00008714 std::make_pair(kVideoCodecAV1, /*allow_i420_conversion=*/true);
Henrik Boström56db9ff2021-03-24 09:06:45 +01008715constexpr std::pair<VideoCodecType, bool> kMultiplexDisallowConversion =
8716 std::make_pair(kVideoCodecMultiplex, /*allow_i420_conversion=*/false);
8717#if defined(WEBRTC_USE_H264)
8718constexpr std::pair<VideoCodecType, bool> kH264AllowConversion =
8719 std::make_pair(kVideoCodecH264, /*allow_i420_conversion=*/true);
8720
8721// The windows compiler does not tolerate #if statements inside the
8722// INSTANTIATE_TEST_SUITE_P() macro, so we have to have two definitions (with
8723// and without H264).
8724INSTANTIATE_TEST_SUITE_P(
8725 All,
8726 VideoStreamEncoderWithRealEncoderTest,
8727 ::testing::Values(kVP8DisallowConversion,
8728 kVP9DisallowConversion,
8729 kAV1AllowConversion,
8730 kMultiplexDisallowConversion,
8731 kH264AllowConversion),
8732 TestParametersVideoCodecAndAllowI420ConversionToString);
8733#else
8734INSTANTIATE_TEST_SUITE_P(
8735 All,
8736 VideoStreamEncoderWithRealEncoderTest,
8737 ::testing::Values(kVP8DisallowConversion,
8738 kVP9DisallowConversion,
8739 kAV1AllowConversion,
8740 kMultiplexDisallowConversion),
8741 TestParametersVideoCodecAndAllowI420ConversionToString);
8742#endif
8743
Åsa Persson4d4f62f2021-09-12 16:14:48 +02008744class ReconfigureEncoderTest : public VideoStreamEncoderTest {
8745 protected:
8746 void RunTest(const std::vector<VideoStream>& configs,
8747 const int expected_num_init_encode) {
8748 ConfigureEncoder(configs[0]);
Asa Persson606d3cb2021-10-04 10:07:11 +02008749 OnBitrateUpdated(kTargetBitrate);
Åsa Persson4d4f62f2021-09-12 16:14:48 +02008750 InsertFrameAndWaitForEncoded();
8751 EXPECT_EQ(1, sink_.number_of_reconfigurations());
8752 ExpectEqual(bitrate_allocator_factory_.codec_config(), configs[0]);
Asa Persson606d3cb2021-10-04 10:07:11 +02008753 EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
8754 ExpectEqual(fake_encoder_.config(), configs[0]);
Åsa Persson4d4f62f2021-09-12 16:14:48 +02008755
8756 // Reconfigure encoder, the encoder should only be reconfigured if needed.
8757 ConfigureEncoder(configs[1]);
8758 InsertFrameAndWaitForEncoded();
8759 EXPECT_EQ(2, sink_.number_of_reconfigurations());
8760 ExpectEqual(bitrate_allocator_factory_.codec_config(), configs[1]);
Asa Persson606d3cb2021-10-04 10:07:11 +02008761 EXPECT_EQ(expected_num_init_encode, fake_encoder_.GetNumInitializations());
Åsa Persson4d4f62f2021-09-12 16:14:48 +02008762 if (expected_num_init_encode > 1)
Asa Persson606d3cb2021-10-04 10:07:11 +02008763 ExpectEqual(fake_encoder_.config(), configs[1]);
Åsa Persson4d4f62f2021-09-12 16:14:48 +02008764
8765 video_stream_encoder_->Stop();
8766 }
8767
8768 void ConfigureEncoder(const VideoStream& stream) {
8769 VideoEncoderConfig config;
8770 test::FillEncoderConfiguration(kVideoCodecVP8, /*num_streams=*/1, &config);
8771 config.max_bitrate_bps = stream.max_bitrate_bps;
8772 config.simulcast_layers[0] = stream;
8773 config.video_stream_factory =
8774 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
8775 /*codec_name=*/"VP8", /*max_qp=*/0, /*is_screenshare=*/false,
8776 /*conference_mode=*/false);
8777 video_stream_encoder_->ConfigureEncoder(std::move(config),
8778 kMaxPayloadLength);
8779 }
8780
8781 void OnBitrateUpdated(DataRate bitrate) {
8782 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8783 bitrate, bitrate, bitrate, 0, 0, 0);
8784 }
8785
8786 void InsertFrameAndWaitForEncoded() {
8787 timestamp_ms_ += kFrameIntervalMs;
8788 video_source_.IncomingCapturedFrame(
8789 CreateFrame(timestamp_ms_, kWidth, kHeight));
8790 sink_.WaitForEncodedFrame(timestamp_ms_);
8791 }
8792
8793 void ExpectEqual(const VideoCodec& actual,
8794 const VideoStream& expected) const {
8795 EXPECT_EQ(actual.numberOfSimulcastStreams, 1);
8796 EXPECT_EQ(actual.simulcastStream[0].maxFramerate, expected.max_framerate);
8797 EXPECT_EQ(actual.simulcastStream[0].minBitrate * 1000,
8798 static_cast<unsigned int>(expected.min_bitrate_bps));
8799 EXPECT_EQ(actual.simulcastStream[0].maxBitrate * 1000,
8800 static_cast<unsigned int>(expected.max_bitrate_bps));
8801 EXPECT_EQ(actual.simulcastStream[0].width,
8802 kWidth / expected.scale_resolution_down_by);
8803 EXPECT_EQ(actual.simulcastStream[0].height,
8804 kHeight / expected.scale_resolution_down_by);
8805 EXPECT_EQ(actual.simulcastStream[0].numberOfTemporalLayers,
8806 expected.num_temporal_layers);
8807 EXPECT_EQ(actual.ScalabilityMode(), expected.scalability_mode);
8808 }
8809
8810 VideoStream DefaultConfig() const {
8811 VideoStream stream;
8812 stream.max_framerate = 25;
8813 stream.min_bitrate_bps = 35000;
8814 stream.max_bitrate_bps = 900000;
8815 stream.scale_resolution_down_by = 1.0;
8816 stream.num_temporal_layers = 1;
8817 stream.bitrate_priority = 1.0;
8818 stream.scalability_mode = "";
8819 return stream;
8820 }
8821
8822 const int kWidth = 640;
8823 const int kHeight = 360;
8824 int64_t timestamp_ms_ = 0;
8825};
8826
8827TEST_F(ReconfigureEncoderTest, NotReconfiguredIfMaxFramerateChanges) {
8828 VideoStream config1 = DefaultConfig();
8829 VideoStream config2 = config1;
8830 config2.max_framerate++;
8831
8832 RunTest({config1, config2}, /*expected_num_init_encode=*/1);
8833}
8834
8835TEST_F(ReconfigureEncoderTest, NotReconfiguredIfMinBitrateChanges) {
8836 VideoStream config1 = DefaultConfig();
8837 VideoStream config2 = config1;
8838 config2.min_bitrate_bps += 10000;
8839
8840 RunTest({config1, config2}, /*expected_num_init_encode=*/1);
8841}
8842
8843TEST_F(ReconfigureEncoderTest, NotReconfiguredIfMaxBitrateChanges) {
8844 VideoStream config1 = DefaultConfig();
8845 VideoStream config2 = config1;
8846 config2.max_bitrate_bps += 100000;
8847
8848 RunTest({config1, config2}, /*expected_num_init_encode=*/1);
8849}
8850
8851TEST_F(ReconfigureEncoderTest, NotReconfiguredIfBitratePriorityChanges) {
8852 VideoStream config1 = DefaultConfig();
8853 VideoStream config2 = config1;
8854 config2.bitrate_priority = config1.bitrate_priority.value() * 2.0;
8855
8856 RunTest({config1, config2}, /*expected_num_init_encode=*/1);
8857}
8858
8859TEST_F(ReconfigureEncoderTest, ReconfiguredIfResolutionChanges) {
8860 VideoStream config1 = DefaultConfig();
8861 VideoStream config2 = config1;
8862 config2.scale_resolution_down_by *= 2;
8863
8864 RunTest({config1, config2}, /*expected_num_init_encode=*/2);
8865}
8866
8867TEST_F(ReconfigureEncoderTest, ReconfiguredIfNumTemporalLayerChanges) {
8868 VideoStream config1 = DefaultConfig();
8869 VideoStream config2 = config1;
8870 config2.num_temporal_layers = config1.num_temporal_layers.value() + 1;
8871
8872 RunTest({config1, config2}, /*expected_num_init_encode=*/2);
8873}
8874
8875TEST_F(ReconfigureEncoderTest, ReconfiguredIfScalabilityModeChanges) {
8876 VideoStream config1 = DefaultConfig();
8877 VideoStream config2 = config1;
8878 config2.scalability_mode = "L1T2";
8879
8880 RunTest({config1, config2}, /*expected_num_init_encode=*/2);
8881}
8882
Tommi62b01db2022-01-25 23:41:22 +01008883// Simple test that just creates and then immediately destroys an encoder.
8884// The purpose of the test is to make sure that nothing bad happens if the
8885// initialization step on the encoder queue, doesn't run.
8886TEST(VideoStreamEncoderSimpleTest, CreateDestroy) {
8887 class SuperLazyTaskQueue : public webrtc::TaskQueueBase {
8888 public:
8889 SuperLazyTaskQueue() = default;
8890 ~SuperLazyTaskQueue() override = default;
8891
8892 private:
8893 void Delete() override { delete this; }
8894 void PostTask(std::unique_ptr<QueuedTask> task) override {
8895 // meh.
8896 }
8897 void PostDelayedTask(std::unique_ptr<QueuedTask> task,
8898 uint32_t milliseconds) override {
8899 ASSERT_TRUE(false);
8900 }
8901 };
8902
8903 // Lots of boiler plate.
8904 GlobalSimulatedTimeController time_controller(Timestamp::Millis(0));
8905 auto stats_proxy = std::make_unique<MockableSendStatisticsProxy>(
8906 time_controller.GetClock(), VideoSendStream::Config(nullptr),
8907 webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo);
8908 SimpleVideoStreamEncoderFactory::MockFakeEncoder mock_fake_encoder(
8909 time_controller.GetClock());
8910 test::VideoEncoderProxyFactory encoder_factory(&mock_fake_encoder);
8911 std::unique_ptr<VideoBitrateAllocatorFactory> bitrate_allocator_factory =
8912 CreateBuiltinVideoBitrateAllocatorFactory();
8913 VideoStreamEncoderSettings encoder_settings{
8914 VideoEncoder::Capabilities(/*loss_notification=*/false)};
8915 encoder_settings.encoder_factory = &encoder_factory;
8916 encoder_settings.bitrate_allocator_factory = bitrate_allocator_factory.get();
8917
8918 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
8919 EXPECT_CALL((*adapter.get()), Initialize).WillOnce(Return());
8920
8921 std::unique_ptr<webrtc::TaskQueueBase, webrtc::TaskQueueDeleter>
8922 encoder_queue(new SuperLazyTaskQueue());
8923
Jonas Orelandc7f691a2022-03-09 15:12:07 +01008924 test::ScopedKeyValueConfig field_trials;
8925
Tommi62b01db2022-01-25 23:41:22 +01008926 // Construct a VideoStreamEncoder instance and let it go out of scope without
8927 // doing anything else (including calling Stop()). This should be fine since
8928 // the posted init task will simply be deleted.
8929 auto encoder = std::make_unique<VideoStreamEncoder>(
8930 time_controller.GetClock(), 1, stats_proxy.get(), encoder_settings,
8931 std::make_unique<CpuOveruseDetectorProxy>(stats_proxy.get()),
8932 std::move(adapter), std::move(encoder_queue),
8933 VideoStreamEncoder::BitrateAllocationCallbackType::
Jonas Orelandc7f691a2022-03-09 15:12:07 +01008934 kVideoBitrateAllocation,
8935 field_trials);
Tommi62b01db2022-01-25 23:41:22 +01008936}
8937
Markus Handellb4e96d42021-11-05 12:00:55 +01008938TEST(VideoStreamEncoderFrameCadenceTest, ActivatesFrameCadenceOnContentType) {
8939 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
8940 auto* adapter_ptr = adapter.get();
8941 SimpleVideoStreamEncoderFactory factory;
Markus Handell8d87c462021-12-16 11:37:16 +01008942 FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback =
8943 nullptr;
8944 EXPECT_CALL(*adapter_ptr, Initialize)
8945 .WillOnce(Invoke([&video_stream_encoder_callback](
8946 FrameCadenceAdapterInterface::Callback* callback) {
8947 video_stream_encoder_callback = callback;
8948 }));
8949 TaskQueueBase* encoder_queue = nullptr;
8950 auto video_stream_encoder =
8951 factory.Create(std::move(adapter), &encoder_queue);
Markus Handellb4e96d42021-11-05 12:00:55 +01008952
Markus Handelle59fee82021-12-23 09:29:23 +01008953 // First a call before we know the frame size and hence cannot compute the
8954 // number of simulcast layers.
8955 EXPECT_CALL(*adapter_ptr, SetZeroHertzModeEnabled(Optional(Field(
8956 &FrameCadenceAdapterInterface::
8957 ZeroHertzModeParams::num_simulcast_layers,
8958 Eq(0)))));
Markus Handellb4e96d42021-11-05 12:00:55 +01008959 VideoEncoderConfig config;
Markus Handell8d87c462021-12-16 11:37:16 +01008960 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &config);
Markus Handellb4e96d42021-11-05 12:00:55 +01008961 config.content_type = VideoEncoderConfig::ContentType::kScreen;
8962 video_stream_encoder->ConfigureEncoder(std::move(config), 0);
Markus Handelle59fee82021-12-23 09:29:23 +01008963 factory.DepleteTaskQueues();
8964
8965 // Then a call as we've computed the number of simulcast layers after a passed
8966 // frame.
8967 EXPECT_CALL(*adapter_ptr, SetZeroHertzModeEnabled(Optional(Field(
8968 &FrameCadenceAdapterInterface::
8969 ZeroHertzModeParams::num_simulcast_layers,
8970 Gt(0)))));
Markus Handell8d87c462021-12-16 11:37:16 +01008971 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/1);
Markus Handell9a478b52021-11-18 16:07:01 +01008972 factory.DepleteTaskQueues();
Markus Handellb4e96d42021-11-05 12:00:55 +01008973 Mock::VerifyAndClearExpectations(adapter_ptr);
8974
Markus Handelle59fee82021-12-23 09:29:23 +01008975 // Expect a disabled zero-hertz mode after passing realtime video.
Markus Handell8d87c462021-12-16 11:37:16 +01008976 EXPECT_CALL(*adapter_ptr, SetZeroHertzModeEnabled(Eq(absl::nullopt)));
Markus Handellb4e96d42021-11-05 12:00:55 +01008977 VideoEncoderConfig config2;
Markus Handell8d87c462021-12-16 11:37:16 +01008978 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &config2);
Markus Handellb4e96d42021-11-05 12:00:55 +01008979 config2.content_type = VideoEncoderConfig::ContentType::kRealtimeVideo;
8980 video_stream_encoder->ConfigureEncoder(std::move(config2), 0);
Markus Handell8d87c462021-12-16 11:37:16 +01008981 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/2);
Markus Handell9a478b52021-11-18 16:07:01 +01008982 factory.DepleteTaskQueues();
Markus Handellb4e96d42021-11-05 12:00:55 +01008983}
8984
8985TEST(VideoStreamEncoderFrameCadenceTest,
8986 ForwardsFramesIntoFrameCadenceAdapter) {
8987 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
8988 auto* adapter_ptr = adapter.get();
8989 test::FrameForwarder video_source;
8990 SimpleVideoStreamEncoderFactory factory;
8991 auto video_stream_encoder = factory.Create(std::move(adapter));
8992 video_stream_encoder->SetSource(
8993 &video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8994
8995 EXPECT_CALL(*adapter_ptr, OnFrame);
8996 auto buffer = rtc::make_ref_counted<NV12Buffer>(/*width=*/16, /*height=*/16);
8997 video_source.IncomingCapturedFrame(
Markus Handell8d87c462021-12-16 11:37:16 +01008998 VideoFrame::Builder().set_video_frame_buffer(std::move(buffer)).build());
Markus Handellb4e96d42021-11-05 12:00:55 +01008999}
9000
Markus Handellee225432021-11-29 12:35:12 +01009001TEST(VideoStreamEncoderFrameCadenceTest, UsesFrameCadenceAdapterForFrameRate) {
9002 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
9003 auto* adapter_ptr = adapter.get();
9004 test::FrameForwarder video_source;
9005 SimpleVideoStreamEncoderFactory factory;
9006 FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback =
9007 nullptr;
9008 EXPECT_CALL(*adapter_ptr, Initialize)
9009 .WillOnce(Invoke([&video_stream_encoder_callback](
9010 FrameCadenceAdapterInterface::Callback* callback) {
9011 video_stream_encoder_callback = callback;
9012 }));
9013 TaskQueueBase* encoder_queue = nullptr;
9014 auto video_stream_encoder =
9015 factory.Create(std::move(adapter), &encoder_queue);
9016
9017 // This is just to make the VSE operational. We'll feed a frame directly by
9018 // the callback interface.
9019 video_stream_encoder->SetSource(
9020 &video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
9021
9022 VideoEncoderConfig video_encoder_config;
9023 test::FillEncoderConfiguration(kVideoCodecGeneric, 1, &video_encoder_config);
9024 video_stream_encoder->ConfigureEncoder(std::move(video_encoder_config),
9025 /*max_data_payload_length=*/1000);
9026
9027 EXPECT_CALL(*adapter_ptr, GetInputFrameRateFps);
9028 EXPECT_CALL(*adapter_ptr, UpdateFrameRate);
Markus Handell8d87c462021-12-16 11:37:16 +01009029 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/1);
Markus Handellee225432021-11-29 12:35:12 +01009030 factory.DepleteTaskQueues();
9031}
9032
Markus Handell8d87c462021-12-16 11:37:16 +01009033TEST(VideoStreamEncoderFrameCadenceTest,
9034 DeactivatesActivatesLayersOnBitrateChanges) {
9035 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
9036 auto* adapter_ptr = adapter.get();
9037 SimpleVideoStreamEncoderFactory factory;
9038 FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback =
9039 nullptr;
9040 EXPECT_CALL(*adapter_ptr, Initialize)
9041 .WillOnce(Invoke([&video_stream_encoder_callback](
9042 FrameCadenceAdapterInterface::Callback* callback) {
9043 video_stream_encoder_callback = callback;
9044 }));
9045 TaskQueueBase* encoder_queue = nullptr;
9046 auto video_stream_encoder =
9047 factory.Create(std::move(adapter), &encoder_queue);
9048
9049 // Configure 2 simulcast layers. FillEncoderConfiguration sets min bitrates to
9050 // {150000, 450000}.
9051 VideoEncoderConfig video_encoder_config;
9052 test::FillEncoderConfiguration(kVideoCodecVP8, 2, &video_encoder_config);
9053 video_stream_encoder->ConfigureEncoder(video_encoder_config.Copy(),
9054 kMaxPayloadLength);
9055 // Ensure an encoder is created.
9056 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/1);
9057
9058 // Both layers enabled at 1 MBit/s.
9059 video_stream_encoder->OnBitrateUpdated(
9060 DataRate::KilobitsPerSec(1000), DataRate::KilobitsPerSec(1000),
9061 DataRate::KilobitsPerSec(1000), 0, 0, 0);
9062 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(0, /*enabled=*/true));
9063 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(1, /*enabled=*/true));
9064 factory.DepleteTaskQueues();
9065 Mock::VerifyAndClearExpectations(adapter_ptr);
9066
9067 // Layer 1 disabled at 200 KBit/s.
9068 video_stream_encoder->OnBitrateUpdated(
9069 DataRate::KilobitsPerSec(200), DataRate::KilobitsPerSec(200),
9070 DataRate::KilobitsPerSec(200), 0, 0, 0);
9071 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(0, /*enabled=*/true));
9072 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(1, /*enabled=*/false));
9073 factory.DepleteTaskQueues();
9074 Mock::VerifyAndClearExpectations(adapter_ptr);
9075
9076 // All layers off at suspended video.
9077 video_stream_encoder->OnBitrateUpdated(DataRate::Zero(), DataRate::Zero(),
9078 DataRate::Zero(), 0, 0, 0);
9079 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(0, /*enabled=*/false));
9080 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(1, /*enabled=*/false));
9081 factory.DepleteTaskQueues();
9082 Mock::VerifyAndClearExpectations(adapter_ptr);
9083
9084 // Both layers enabled again back at 1 MBit/s.
9085 video_stream_encoder->OnBitrateUpdated(
9086 DataRate::KilobitsPerSec(1000), DataRate::KilobitsPerSec(1000),
9087 DataRate::KilobitsPerSec(1000), 0, 0, 0);
9088 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(0, /*enabled=*/true));
9089 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(1, /*enabled=*/true));
9090 factory.DepleteTaskQueues();
9091}
9092
9093TEST(VideoStreamEncoderFrameCadenceTest, UpdatesQualityConvergence) {
9094 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
9095 auto* adapter_ptr = adapter.get();
9096 SimpleVideoStreamEncoderFactory factory;
9097 FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback =
9098 nullptr;
9099 EXPECT_CALL(*adapter_ptr, Initialize)
9100 .WillOnce(Invoke([&video_stream_encoder_callback](
9101 FrameCadenceAdapterInterface::Callback* callback) {
9102 video_stream_encoder_callback = callback;
9103 }));
9104 TaskQueueBase* encoder_queue = nullptr;
9105 auto video_stream_encoder =
9106 factory.Create(std::move(adapter), &encoder_queue);
9107
9108 // Configure 2 simulcast layers and setup 1 MBit/s to unpause the encoder.
9109 VideoEncoderConfig video_encoder_config;
9110 test::FillEncoderConfiguration(kVideoCodecVP8, 2, &video_encoder_config);
9111 video_stream_encoder->ConfigureEncoder(video_encoder_config.Copy(),
9112 kMaxPayloadLength);
9113 video_stream_encoder->OnBitrateUpdated(
9114 DataRate::KilobitsPerSec(1000), DataRate::KilobitsPerSec(1000),
9115 DataRate::KilobitsPerSec(1000), 0, 0, 0);
9116
9117 // Pass a frame which has unconverged results.
9118 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/1);
9119 EXPECT_CALL(factory.GetMockFakeEncoder(), EncodeHook)
9120 .WillRepeatedly(Invoke([](EncodedImage& encoded_image,
9121 rtc::scoped_refptr<EncodedImageBuffer> buffer) {
9122 EXPECT_FALSE(encoded_image.IsAtTargetQuality());
9123 CodecSpecificInfo codec_specific;
9124 codec_specific.codecType = kVideoCodecGeneric;
9125 return codec_specific;
9126 }));
9127 EXPECT_CALL(*adapter_ptr, UpdateLayerQualityConvergence(0, false));
9128 EXPECT_CALL(*adapter_ptr, UpdateLayerQualityConvergence(1, false));
9129 factory.DepleteTaskQueues();
9130 Mock::VerifyAndClearExpectations(adapter_ptr);
9131 Mock::VerifyAndClearExpectations(&factory.GetMockFakeEncoder());
9132
9133 // Pass a frame which converges in layer 0 and not in layer 1.
9134 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/2);
9135 EXPECT_CALL(factory.GetMockFakeEncoder(), EncodeHook)
9136 .WillRepeatedly(Invoke([](EncodedImage& encoded_image,
9137 rtc::scoped_refptr<EncodedImageBuffer> buffer) {
9138 encoded_image.SetAtTargetQuality(encoded_image.SpatialIndex() == 0);
9139 CodecSpecificInfo codec_specific;
9140 codec_specific.codecType = kVideoCodecGeneric;
9141 return codec_specific;
9142 }));
9143 EXPECT_CALL(*adapter_ptr, UpdateLayerQualityConvergence(0, true));
9144 EXPECT_CALL(*adapter_ptr, UpdateLayerQualityConvergence(1, false));
9145 factory.DepleteTaskQueues();
9146 Mock::VerifyAndClearExpectations(adapter_ptr);
9147 Mock::VerifyAndClearExpectations(&factory.GetMockFakeEncoder());
9148}
9149
Markus Handell2e0f4f02021-12-21 19:14:58 +01009150TEST(VideoStreamEncoderFrameCadenceTest,
9151 RequestsRefreshFramesWhenCadenceAdapterInstructs) {
9152 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
9153 auto* adapter_ptr = adapter.get();
9154 MockVideoSourceInterface mock_source;
9155 SimpleVideoStreamEncoderFactory factory;
9156 FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback =
9157 nullptr;
9158 EXPECT_CALL(*adapter_ptr, Initialize)
9159 .WillOnce(Invoke([&video_stream_encoder_callback](
9160 FrameCadenceAdapterInterface::Callback* callback) {
9161 video_stream_encoder_callback = callback;
9162 }));
9163 TaskQueueBase* encoder_queue = nullptr;
9164 auto video_stream_encoder =
9165 factory.Create(std::move(adapter), &encoder_queue);
9166 video_stream_encoder->SetSource(
9167 &mock_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
9168 VideoEncoderConfig config;
9169 config.content_type = VideoEncoderConfig::ContentType::kScreen;
9170 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &config);
9171 video_stream_encoder->ConfigureEncoder(std::move(config), 0);
9172 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/2);
9173 // Ensure the encoder is set up.
9174 factory.DepleteTaskQueues();
9175
Markus Handell818e7fb2021-12-30 13:01:33 +01009176 EXPECT_CALL(*adapter_ptr, ProcessKeyFrameRequest)
9177 .WillOnce(Invoke([video_stream_encoder_callback] {
9178 video_stream_encoder_callback->RequestRefreshFrame();
9179 }));
Markus Handell2e0f4f02021-12-21 19:14:58 +01009180 EXPECT_CALL(mock_source, RequestRefreshFrame);
9181 video_stream_encoder->SendKeyFrame();
9182 factory.DepleteTaskQueues();
9183 Mock::VerifyAndClearExpectations(adapter_ptr);
9184 Mock::VerifyAndClearExpectations(&mock_source);
9185
Markus Handell818e7fb2021-12-30 13:01:33 +01009186 EXPECT_CALL(*adapter_ptr, ProcessKeyFrameRequest);
Markus Handell2e0f4f02021-12-21 19:14:58 +01009187 EXPECT_CALL(mock_source, RequestRefreshFrame).Times(0);
9188 video_stream_encoder->SendKeyFrame();
9189 factory.DepleteTaskQueues();
9190}
9191
Markus Handell818e7fb2021-12-30 13:01:33 +01009192TEST(VideoStreamEncoderFrameCadenceTest,
9193 RequestsRefreshFrameForEarlyZeroHertzKeyFrameRequest) {
9194 SimpleVideoStreamEncoderFactory factory;
9195 auto encoder_queue =
9196 factory.GetTimeController()->GetTaskQueueFactory()->CreateTaskQueue(
9197 "EncoderQueue", TaskQueueFactory::Priority::NORMAL);
9198
9199 // Enables zero-hertz mode.
Jonas Orelandc7f691a2022-03-09 15:12:07 +01009200 test::ScopedKeyValueConfig field_trials(
9201 "WebRTC-ZeroHertzScreenshare/Enabled/");
Markus Handell818e7fb2021-12-30 13:01:33 +01009202 auto adapter = FrameCadenceAdapterInterface::Create(
9203 factory.GetTimeController()->GetClock(), encoder_queue.get());
9204 FrameCadenceAdapterInterface* adapter_ptr = adapter.get();
9205
9206 MockVideoSourceInterface mock_source;
9207 auto video_stream_encoder = factory.CreateWithEncoderQueue(
Jonas Orelandc7f691a2022-03-09 15:12:07 +01009208 std::move(adapter), std::move(encoder_queue), &field_trials);
Markus Handell818e7fb2021-12-30 13:01:33 +01009209
9210 video_stream_encoder->SetSource(
9211 &mock_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
9212 VideoEncoderConfig config;
9213 config.content_type = VideoEncoderConfig::ContentType::kScreen;
9214 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &config);
9215 video_stream_encoder->ConfigureEncoder(std::move(config), 0);
9216
9217 // Eventually expect a refresh frame request when requesting a key frame
9218 // before initializing zero-hertz mode. This can happen in reality because the
9219 // threads invoking key frame requests and constraints setup aren't
9220 // synchronized.
9221 EXPECT_CALL(mock_source, RequestRefreshFrame);
9222 video_stream_encoder->SendKeyFrame();
9223 adapter_ptr->OnConstraintsChanged(VideoTrackSourceConstraints{0, 30});
9224 factory.DepleteTaskQueues();
9225}
9226
perkj26091b12016-09-01 01:17:40 -07009227} // namespace webrtc