blob: 0a8c2a6c9df2c43eca88ad130bdaa21fd07f8304 [file] [log] [blame]
perkj26091b12016-09-01 01:17:40 -07001/*
2 * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Erik Språng4529fbc2018-10-12 10:30:31 +020011#include "video/video_stream_encoder.h"
12
sprangfe627f32017-03-29 08:24:59 -070013#include <algorithm>
perkj803d97f2016-11-01 11:45:46 -070014#include <limits>
Danil Chapovalovd3ba2362019-04-10 17:01:23 +020015#include <memory>
Henrik Boström56db9ff2021-03-24 09:06:45 +010016#include <tuple>
Per512ecb32016-09-23 15:52:06 +020017#include <utility>
18
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020019#include "absl/memory/memory.h"
Markus Handell9a478b52021-11-18 16:07:01 +010020#include "api/rtp_parameters.h"
Danil Chapovalovd3ba2362019-04-10 17:01:23 +020021#include "api/task_queue/default_task_queue_factory.h"
Markus Handellb4e96d42021-11-05 12:00:55 +010022#include "api/task_queue/task_queue_factory.h"
Elad Alon45befc52019-07-02 11:20:09 +020023#include "api/test/mock_fec_controller_override.h"
philipel9b058032020-02-10 11:30:00 +010024#include "api/test/mock_video_encoder.h"
Henrik Boström56db9ff2021-03-24 09:06:45 +010025#include "api/test/mock_video_encoder_factory.h"
Markus Handell8d87c462021-12-16 11:37:16 +010026#include "api/units/data_rate.h"
Jiawei Ouc2ebe212018-11-08 10:02:56 -080027#include "api/video/builtin_video_bitrate_allocator_factory.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020028#include "api/video/i420_buffer.h"
Evan Shrubsole895556e2020-10-05 09:15:13 +020029#include "api/video/nv12_buffer.h"
Evan Shrubsolece0a11d2020-04-16 11:36:55 +020030#include "api/video/video_adaptation_reason.h"
Erik Språngf93eda12019-01-16 17:10:57 +010031#include "api/video/video_bitrate_allocation.h"
Henrik Boström56db9ff2021-03-24 09:06:45 +010032#include "api/video_codecs/sdp_video_format.h"
Elad Alon370f93a2019-06-11 14:57:57 +020033#include "api/video_codecs/video_encoder.h"
Erik Språng4529fbc2018-10-12 10:30:31 +020034#include "api/video_codecs/vp8_temporal_layers.h"
Elad Aloncde8ab22019-03-20 11:56:20 +010035#include "api/video_codecs/vp8_temporal_layers_factory.h"
Henrik Boström0f0aa9c2020-06-02 13:02:36 +020036#include "call/adaptation/test/fake_adaptation_constraint.h"
Evan Shrubsoleaa6fbc12020-02-25 16:26:01 +010037#include "call/adaptation/test/fake_resource.h"
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020038#include "common_video/h264/h264_common.h"
Noah Richards51db4212019-06-12 06:59:12 -070039#include "common_video/include/video_frame_buffer.h"
Steve Anton10542f22019-01-11 09:11:00 -080040#include "media/base/video_adapter.h"
Åsa Perssonc5a74ff2020-09-20 17:50:00 +020041#include "media/engine/webrtc_video_engine.h"
Henrik Boström56db9ff2021-03-24 09:06:45 +010042#include "modules/video_coding/codecs/av1/libaom_av1_encoder.h"
43#include "modules/video_coding/codecs/h264/include/h264.h"
44#include "modules/video_coding/codecs/multiplex/include/multiplex_encoder_adapter.h"
45#include "modules/video_coding/codecs/vp8/include/vp8.h"
46#include "modules/video_coding/codecs/vp9/include/vp9.h"
Sergey Silkin86684962018-03-28 19:32:37 +020047#include "modules/video_coding/codecs/vp9/include/vp9_globals.h"
Erik Språng7444b192021-06-02 14:02:13 +020048#include "modules/video_coding/codecs/vp9/svc_config.h"
Henrik Boström91aa7322020-04-28 12:24:33 +020049#include "modules/video_coding/utility/quality_scaler.h"
Åsa Perssonc29cb2c2019-03-25 12:06:59 +010050#include "modules/video_coding/utility/simulcast_rate_allocator.h"
Tomas Gunnarsson612445e2020-09-21 14:31:23 +020051#include "rtc_base/event.h"
Åsa Persson258e9892021-02-25 10:39:51 +010052#include "rtc_base/experiments/encoder_info_settings.h"
Henrik Boström2671dac2020-05-19 16:29:09 +020053#include "rtc_base/gunit.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020054#include "rtc_base/logging.h"
Steve Anton10542f22019-01-11 09:11:00 -080055#include "rtc_base/ref_counted_object.h"
Markus Handella3765182020-07-08 13:13:32 +020056#include "rtc_base/synchronization/mutex.h"
Erik Språng7ca375c2019-02-06 16:20:17 +010057#include "system_wrappers/include/field_trial.h"
Mirko Bonadei17f48782018-09-28 08:51:10 +020058#include "system_wrappers/include/metrics.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020059#include "test/encoder_settings.h"
60#include "test/fake_encoder.h"
Kári Tristan Helgason639602a2018-08-02 10:51:40 +020061#include "test/field_trial.h"
Artem Titov33f9d2b2019-12-05 15:59:00 +010062#include "test/frame_forwarder.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020063#include "test/gmock.h"
64#include "test/gtest.h"
Henrik Boström56db9ff2021-03-24 09:06:45 +010065#include "test/mappable_native_buffer.h"
Tomas Gunnarsson612445e2020-09-21 14:31:23 +020066#include "test/time_controller/simulated_time_controller.h"
Niels Möllercbcbc222018-09-28 09:07:24 +020067#include "test/video_encoder_proxy_factory.h"
Markus Handellb4e96d42021-11-05 12:00:55 +010068#include "video/frame_cadence_adapter.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020069#include "video/send_statistics_proxy.h"
perkj26091b12016-09-01 01:17:40 -070070
71namespace webrtc {
72
sprang57c2fff2017-01-16 06:24:02 -080073using ::testing::_;
philipeld9cc8c02019-09-16 14:53:40 +020074using ::testing::AllOf;
Per Kjellanderd0a8f512020-10-07 11:28:41 +020075using ::testing::AtLeast;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +020076using ::testing::Eq;
philipeld9cc8c02019-09-16 14:53:40 +020077using ::testing::Field;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +020078using ::testing::Ge;
79using ::testing::Gt;
Markus Handellee225432021-11-29 12:35:12 +010080using ::testing::Invoke;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +020081using ::testing::Le;
82using ::testing::Lt;
philipel9b058032020-02-10 11:30:00 +010083using ::testing::Matcher;
Markus Handellb4e96d42021-11-05 12:00:55 +010084using ::testing::Mock;
philipel9b058032020-02-10 11:30:00 +010085using ::testing::NiceMock;
Markus Handell8d87c462021-12-16 11:37:16 +010086using ::testing::Not;
Markus Handellb4e96d42021-11-05 12:00:55 +010087using ::testing::Optional;
philipel9b058032020-02-10 11:30:00 +010088using ::testing::Return;
Per Kjellander4190ce92020-12-15 17:24:55 +010089using ::testing::SizeIs;
philipeld9cc8c02019-09-16 14:53:40 +020090using ::testing::StrictMock;
kthelgason876222f2016-11-29 01:44:11 -080091
perkj803d97f2016-11-01 11:45:46 -070092namespace {
Åsa Persson8c1bf952018-09-13 10:42:19 +020093const int kMinPixelsPerFrame = 320 * 180;
Åsa Perssone644a032019-11-08 15:56:00 +010094const int kQpLow = 1;
95const int kQpHigh = 2;
Åsa Persson8c1bf952018-09-13 10:42:19 +020096const int kMinFramerateFps = 2;
97const int kMinBalancedFramerateFps = 7;
98const int64_t kFrameTimeoutMs = 100;
asapersson5f7226f2016-11-25 04:37:00 -080099const size_t kMaxPayloadLength = 1440;
Asa Persson606d3cb2021-10-04 10:07:11 +0200100const DataRate kTargetBitrate = DataRate::KilobitsPerSec(1000);
101const DataRate kLowTargetBitrate = DataRate::KilobitsPerSec(100);
102const DataRate kStartBitrate = DataRate::KilobitsPerSec(600);
103const DataRate kSimulcastTargetBitrate = DataRate::KilobitsPerSec(3150);
kthelgason2bc68642017-02-07 07:02:22 -0800104const int kMaxInitialFramedrop = 4;
sprangfda496a2017-06-15 04:21:07 -0700105const int kDefaultFramerate = 30;
Åsa Persson8c1bf952018-09-13 10:42:19 +0200106const int64_t kFrameIntervalMs = rtc::kNumMillisecsPerSec / kDefaultFramerate;
Niels Möllerfe407b72019-09-10 10:48:48 +0200107const int64_t kProcessIntervalMs = 1000;
Sergey Silkin41c650b2019-10-14 13:12:19 +0200108const VideoEncoder::ResolutionBitrateLimits
109 kEncoderBitrateLimits540p(960 * 540, 100 * 1000, 100 * 1000, 2000 * 1000);
110const VideoEncoder::ResolutionBitrateLimits
111 kEncoderBitrateLimits720p(1280 * 720, 200 * 1000, 200 * 1000, 4000 * 1000);
asapersson5f7226f2016-11-25 04:37:00 -0800112
Asa Persson606d3cb2021-10-04 10:07:11 +0200113uint8_t kOptimalSps[] = {0, 0, 0, 1, H264::NaluType::kSps,
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +0200114 0x00, 0x00, 0x03, 0x03, 0xF4,
115 0x05, 0x03, 0xC7, 0xE0, 0x1B,
116 0x41, 0x10, 0x8D, 0x00};
117
Sergey Silkin0e42cf72021-03-15 10:12:57 +0100118const uint8_t kCodedFrameVp8Qp25[] = {
119 0x10, 0x02, 0x00, 0x9d, 0x01, 0x2a, 0x10, 0x00, 0x10, 0x00,
120 0x02, 0x47, 0x08, 0x85, 0x85, 0x88, 0x85, 0x84, 0x88, 0x0c,
121 0x82, 0x00, 0x0c, 0x0d, 0x60, 0x00, 0xfe, 0xfc, 0x5c, 0xd0};
122
Markus Handell8d87c462021-12-16 11:37:16 +0100123void PassAFrame(
124 TaskQueueBase* encoder_queue,
125 FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback,
126 int64_t ntp_time_ms) {
127 encoder_queue->PostTask(
128 ToQueuedTask([video_stream_encoder_callback, ntp_time_ms] {
129 video_stream_encoder_callback->OnFrame(
130 Timestamp::Millis(ntp_time_ms), 1,
131 VideoFrame::Builder()
132 .set_video_frame_buffer(rtc::make_ref_counted<NV12Buffer>(
133 /*width=*/16, /*height=*/16))
134 .build());
135 }));
136}
137
perkj803d97f2016-11-01 11:45:46 -0700138class TestBuffer : public webrtc::I420Buffer {
139 public:
140 TestBuffer(rtc::Event* event, int width, int height)
141 : I420Buffer(width, height), event_(event) {}
142
143 private:
144 friend class rtc::RefCountedObject<TestBuffer>;
145 ~TestBuffer() override {
146 if (event_)
147 event_->Set();
148 }
149 rtc::Event* const event_;
150};
151
Henrik Boström56db9ff2021-03-24 09:06:45 +0100152// A fake native buffer that can't be converted to I420. Upon scaling, it
153// produces another FakeNativeBuffer.
Noah Richards51db4212019-06-12 06:59:12 -0700154class FakeNativeBuffer : public webrtc::VideoFrameBuffer {
155 public:
156 FakeNativeBuffer(rtc::Event* event, int width, int height)
157 : event_(event), width_(width), height_(height) {}
158 webrtc::VideoFrameBuffer::Type type() const override { return Type::kNative; }
159 int width() const override { return width_; }
160 int height() const override { return height_; }
161 rtc::scoped_refptr<webrtc::I420BufferInterface> ToI420() override {
162 return nullptr;
163 }
Henrik Boström56db9ff2021-03-24 09:06:45 +0100164 rtc::scoped_refptr<VideoFrameBuffer> CropAndScale(
165 int offset_x,
166 int offset_y,
167 int crop_width,
168 int crop_height,
169 int scaled_width,
170 int scaled_height) override {
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200171 return rtc::make_ref_counted<FakeNativeBuffer>(nullptr, scaled_width,
172 scaled_height);
Henrik Boström56db9ff2021-03-24 09:06:45 +0100173 }
Noah Richards51db4212019-06-12 06:59:12 -0700174
175 private:
176 friend class rtc::RefCountedObject<FakeNativeBuffer>;
177 ~FakeNativeBuffer() override {
178 if (event_)
179 event_->Set();
180 }
181 rtc::Event* const event_;
182 const int width_;
183 const int height_;
184};
185
Evan Shrubsole895556e2020-10-05 09:15:13 +0200186// A fake native buffer that is backed by an NV12 buffer.
187class FakeNV12NativeBuffer : public webrtc::VideoFrameBuffer {
188 public:
189 FakeNV12NativeBuffer(rtc::Event* event, int width, int height)
190 : nv12_buffer_(NV12Buffer::Create(width, height)), event_(event) {}
191
192 webrtc::VideoFrameBuffer::Type type() const override { return Type::kNative; }
193 int width() const override { return nv12_buffer_->width(); }
194 int height() const override { return nv12_buffer_->height(); }
195 rtc::scoped_refptr<webrtc::I420BufferInterface> ToI420() override {
196 return nv12_buffer_->ToI420();
197 }
Evan Shrubsoleb556b082020-10-08 14:56:45 +0200198 rtc::scoped_refptr<VideoFrameBuffer> GetMappedFrameBuffer(
199 rtc::ArrayView<VideoFrameBuffer::Type> types) override {
200 if (absl::c_find(types, Type::kNV12) != types.end()) {
201 return nv12_buffer_;
202 }
203 return nullptr;
204 }
Evan Shrubsole895556e2020-10-05 09:15:13 +0200205 const NV12BufferInterface* GetNV12() const { return nv12_buffer_; }
206
207 private:
208 friend class rtc::RefCountedObject<FakeNV12NativeBuffer>;
209 ~FakeNV12NativeBuffer() override {
210 if (event_)
211 event_->Set();
212 }
213 rtc::scoped_refptr<NV12Buffer> nv12_buffer_;
214 rtc::Event* const event_;
215};
216
Niels Möller7dc26b72017-12-06 10:27:48 +0100217class CpuOveruseDetectorProxy : public OveruseFrameDetector {
218 public:
Niels Möllerd1f7eb62018-03-28 16:40:58 +0200219 explicit CpuOveruseDetectorProxy(CpuOveruseMetricsObserver* metrics_observer)
220 : OveruseFrameDetector(metrics_observer),
Henrik Boström381d1092020-05-12 18:49:07 +0200221 last_target_framerate_fps_(-1),
222 framerate_updated_event_(true /* manual_reset */,
223 false /* initially_signaled */) {}
Niels Möller7dc26b72017-12-06 10:27:48 +0100224 virtual ~CpuOveruseDetectorProxy() {}
225
226 void OnTargetFramerateUpdated(int framerate_fps) override {
Markus Handella3765182020-07-08 13:13:32 +0200227 MutexLock lock(&lock_);
Niels Möller7dc26b72017-12-06 10:27:48 +0100228 last_target_framerate_fps_ = framerate_fps;
229 OveruseFrameDetector::OnTargetFramerateUpdated(framerate_fps);
Henrik Boström381d1092020-05-12 18:49:07 +0200230 framerate_updated_event_.Set();
Niels Möller7dc26b72017-12-06 10:27:48 +0100231 }
232
233 int GetLastTargetFramerate() {
Markus Handella3765182020-07-08 13:13:32 +0200234 MutexLock lock(&lock_);
Niels Möller7dc26b72017-12-06 10:27:48 +0100235 return last_target_framerate_fps_;
236 }
237
Niels Möller4db138e2018-04-19 09:04:13 +0200238 CpuOveruseOptions GetOptions() { return options_; }
239
Henrik Boström381d1092020-05-12 18:49:07 +0200240 rtc::Event* framerate_updated_event() { return &framerate_updated_event_; }
241
Niels Möller7dc26b72017-12-06 10:27:48 +0100242 private:
Markus Handella3765182020-07-08 13:13:32 +0200243 Mutex lock_;
Niels Möller7dc26b72017-12-06 10:27:48 +0100244 int last_target_framerate_fps_ RTC_GUARDED_BY(lock_);
Henrik Boström381d1092020-05-12 18:49:07 +0200245 rtc::Event framerate_updated_event_;
Niels Möller7dc26b72017-12-06 10:27:48 +0100246};
247
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200248class FakeVideoSourceRestrictionsListener
249 : public VideoSourceRestrictionsListener {
Henrik Boström381d1092020-05-12 18:49:07 +0200250 public:
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200251 FakeVideoSourceRestrictionsListener()
Henrik Boström381d1092020-05-12 18:49:07 +0200252 : was_restrictions_updated_(false), restrictions_updated_event_() {}
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200253 ~FakeVideoSourceRestrictionsListener() override {
Henrik Boström381d1092020-05-12 18:49:07 +0200254 RTC_DCHECK(was_restrictions_updated_);
255 }
256
257 rtc::Event* restrictions_updated_event() {
258 return &restrictions_updated_event_;
259 }
260
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200261 // VideoSourceRestrictionsListener implementation.
Henrik Boström381d1092020-05-12 18:49:07 +0200262 void OnVideoSourceRestrictionsUpdated(
263 VideoSourceRestrictions restrictions,
264 const VideoAdaptationCounters& adaptation_counters,
Evan Shrubsoleec0af262020-07-01 11:47:46 +0200265 rtc::scoped_refptr<Resource> reason,
266 const VideoSourceRestrictions& unfiltered_restrictions) override {
Henrik Boström381d1092020-05-12 18:49:07 +0200267 was_restrictions_updated_ = true;
268 restrictions_updated_event_.Set();
269 }
270
271 private:
272 bool was_restrictions_updated_;
273 rtc::Event restrictions_updated_event_;
274};
275
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200276auto WantsFps(Matcher<int> fps_matcher) {
277 return Field("max_framerate_fps", &rtc::VideoSinkWants::max_framerate_fps,
278 fps_matcher);
279}
280
281auto WantsMaxPixels(Matcher<int> max_pixel_matcher) {
282 return Field("max_pixel_count", &rtc::VideoSinkWants::max_pixel_count,
283 AllOf(max_pixel_matcher, Gt(0)));
284}
285
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200286auto ResolutionMax() {
287 return AllOf(
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200288 WantsMaxPixels(Eq(std::numeric_limits<int>::max())),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200289 Field("target_pixel_count", &rtc::VideoSinkWants::target_pixel_count,
290 Eq(absl::nullopt)));
291}
292
293auto FpsMax() {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200294 return WantsFps(Eq(kDefaultFramerate));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200295}
296
297auto FpsUnlimited() {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200298 return WantsFps(Eq(std::numeric_limits<int>::max()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200299}
300
301auto FpsMatchesResolutionMax(Matcher<int> fps_matcher) {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200302 return AllOf(WantsFps(fps_matcher), ResolutionMax());
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200303}
304
305auto FpsMaxResolutionMatches(Matcher<int> pixel_matcher) {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200306 return AllOf(FpsMax(), WantsMaxPixels(pixel_matcher));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200307}
308
309auto FpsMaxResolutionMax() {
310 return AllOf(FpsMax(), ResolutionMax());
311}
312
313auto UnlimitedSinkWants() {
314 return AllOf(FpsUnlimited(), ResolutionMax());
315}
316
317auto FpsInRangeForPixelsInBalanced(int last_frame_pixels) {
318 Matcher<int> fps_range_matcher;
319
320 if (last_frame_pixels <= 320 * 240) {
321 fps_range_matcher = AllOf(Ge(7), Le(10));
Åsa Perssonc5a74ff2020-09-20 17:50:00 +0200322 } else if (last_frame_pixels <= 480 * 360) {
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200323 fps_range_matcher = AllOf(Ge(10), Le(15));
324 } else if (last_frame_pixels <= 640 * 480) {
325 fps_range_matcher = Ge(15);
326 } else {
327 fps_range_matcher = Eq(kDefaultFramerate);
328 }
329 return Field("max_framerate_fps", &rtc::VideoSinkWants::max_framerate_fps,
330 fps_range_matcher);
331}
332
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200333auto FpsEqResolutionEqTo(const rtc::VideoSinkWants& other_wants) {
334 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
335 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
336}
337
338auto FpsMaxResolutionLt(const rtc::VideoSinkWants& other_wants) {
339 return AllOf(FpsMax(), WantsMaxPixels(Lt(other_wants.max_pixel_count)));
340}
341
342auto FpsMaxResolutionGt(const rtc::VideoSinkWants& other_wants) {
343 return AllOf(FpsMax(), WantsMaxPixels(Gt(other_wants.max_pixel_count)));
344}
345
346auto FpsLtResolutionEq(const rtc::VideoSinkWants& other_wants) {
347 return AllOf(WantsFps(Lt(other_wants.max_framerate_fps)),
348 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
349}
350
351auto FpsGtResolutionEq(const rtc::VideoSinkWants& other_wants) {
352 return AllOf(WantsFps(Gt(other_wants.max_framerate_fps)),
353 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
354}
355
356auto FpsEqResolutionLt(const rtc::VideoSinkWants& other_wants) {
357 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
358 WantsMaxPixels(Lt(other_wants.max_pixel_count)));
359}
360
361auto FpsEqResolutionGt(const rtc::VideoSinkWants& other_wants) {
362 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
363 WantsMaxPixels(Gt(other_wants.max_pixel_count)));
364}
365
mflodmancc3d4422017-08-03 08:27:51 -0700366class VideoStreamEncoderUnderTest : public VideoStreamEncoder {
perkj803d97f2016-11-01 11:45:46 -0700367 public:
Markus Handell9a478b52021-11-18 16:07:01 +0100368 VideoStreamEncoderUnderTest(
369 TimeController* time_controller,
370 std::unique_ptr<FrameCadenceAdapterInterface> cadence_adapter,
371 std::unique_ptr<webrtc::TaskQueueBase, webrtc::TaskQueueDeleter>
372 encoder_queue,
373 SendStatisticsProxy* stats_proxy,
374 const VideoStreamEncoderSettings& settings,
375 VideoStreamEncoder::BitrateAllocationCallbackType
376 allocation_callback_type)
377 : VideoStreamEncoder(time_controller->GetClock(),
378 1 /* number_of_cores */,
379 stats_proxy,
380 settings,
381 std::unique_ptr<OveruseFrameDetector>(
382 overuse_detector_proxy_ =
383 new CpuOveruseDetectorProxy(stats_proxy)),
384 std::move(cadence_adapter),
385 std::move(encoder_queue),
386 allocation_callback_type),
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200387 time_controller_(time_controller),
Henrik Boström5cc28b02020-06-01 17:59:05 +0200388 fake_cpu_resource_(FakeResource::Create("FakeResource[CPU]")),
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200389 fake_quality_resource_(FakeResource::Create("FakeResource[QP]")),
Evan Shrubsoledc4d4222020-07-09 11:47:10 +0200390 fake_adaptation_constraint_("FakeAdaptationConstraint") {
Henrik Boströmc55516d2020-05-11 16:29:22 +0200391 InjectAdaptationResource(fake_quality_resource_,
Evan Shrubsolece0a11d2020-04-16 11:36:55 +0200392 VideoAdaptationReason::kQuality);
Henrik Boströmc55516d2020-05-11 16:29:22 +0200393 InjectAdaptationResource(fake_cpu_resource_, VideoAdaptationReason::kCpu);
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200394 InjectAdaptationConstraint(&fake_adaptation_constraint_);
Evan Shrubsoleaa6fbc12020-02-25 16:26:01 +0100395 }
perkj803d97f2016-11-01 11:45:46 -0700396
Henrik Boström381d1092020-05-12 18:49:07 +0200397 void SetSourceAndWaitForRestrictionsUpdated(
398 rtc::VideoSourceInterface<VideoFrame>* source,
399 const DegradationPreference& degradation_preference) {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200400 FakeVideoSourceRestrictionsListener listener;
401 AddRestrictionsListenerForTesting(&listener);
Henrik Boström381d1092020-05-12 18:49:07 +0200402 SetSource(source, degradation_preference);
403 listener.restrictions_updated_event()->Wait(5000);
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200404 RemoveRestrictionsListenerForTesting(&listener);
Henrik Boström381d1092020-05-12 18:49:07 +0200405 }
406
407 void SetSourceAndWaitForFramerateUpdated(
408 rtc::VideoSourceInterface<VideoFrame>* source,
409 const DegradationPreference& degradation_preference) {
410 overuse_detector_proxy_->framerate_updated_event()->Reset();
411 SetSource(source, degradation_preference);
412 overuse_detector_proxy_->framerate_updated_event()->Wait(5000);
413 }
414
415 void OnBitrateUpdatedAndWaitForManagedResources(
416 DataRate target_bitrate,
417 DataRate stable_target_bitrate,
418 DataRate link_allocation,
419 uint8_t fraction_lost,
420 int64_t round_trip_time_ms,
421 double cwnd_reduce_ratio) {
422 OnBitrateUpdated(target_bitrate, stable_target_bitrate, link_allocation,
423 fraction_lost, round_trip_time_ms, cwnd_reduce_ratio);
424 // Bitrate is updated on the encoder queue.
425 WaitUntilTaskQueueIsIdle();
Henrik Boström381d1092020-05-12 18:49:07 +0200426 }
427
kthelgason2fc52542017-03-03 00:24:41 -0800428 // This is used as a synchronisation mechanism, to make sure that the
429 // encoder queue is not blocked before we start sending it frames.
430 void WaitUntilTaskQueueIsIdle() {
Markus Handell28c71802021-11-08 10:11:55 +0100431 time_controller_->AdvanceTime(TimeDelta::Zero());
kthelgason2fc52542017-03-03 00:24:41 -0800432 }
433
Henrik Boström91aa7322020-04-28 12:24:33 +0200434 // Triggers resource usage measurements on the fake CPU resource.
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200435 void TriggerCpuOveruse() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200436 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200437 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200438 fake_cpu_resource_->SetUsageState(ResourceUsageState::kOveruse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200439 event.Set();
440 });
441 ASSERT_TRUE(event.Wait(5000));
Markus Handell28c71802021-11-08 10:11:55 +0100442 time_controller_->AdvanceTime(TimeDelta::Zero());
Henrik Boström91aa7322020-04-28 12:24:33 +0200443 }
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200444
Henrik Boström91aa7322020-04-28 12:24:33 +0200445 void TriggerCpuUnderuse() {
446 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200447 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200448 fake_cpu_resource_->SetUsageState(ResourceUsageState::kUnderuse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200449 event.Set();
450 });
451 ASSERT_TRUE(event.Wait(5000));
Markus Handell28c71802021-11-08 10:11:55 +0100452 time_controller_->AdvanceTime(TimeDelta::Zero());
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200453 }
kthelgason876222f2016-11-29 01:44:11 -0800454
Henrik Boström91aa7322020-04-28 12:24:33 +0200455 // Triggers resource usage measurements on the fake quality resource.
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200456 void TriggerQualityLow() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200457 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200458 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200459 fake_quality_resource_->SetUsageState(ResourceUsageState::kOveruse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200460 event.Set();
461 });
462 ASSERT_TRUE(event.Wait(5000));
Markus Handell28c71802021-11-08 10:11:55 +0100463 time_controller_->AdvanceTime(TimeDelta::Zero());
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200464 }
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200465 void TriggerQualityHigh() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200466 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200467 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200468 fake_quality_resource_->SetUsageState(ResourceUsageState::kUnderuse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200469 event.Set();
470 });
471 ASSERT_TRUE(event.Wait(5000));
Markus Handell28c71802021-11-08 10:11:55 +0100472 time_controller_->AdvanceTime(TimeDelta::Zero());
Henrik Boström91aa7322020-04-28 12:24:33 +0200473 }
474
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200475 TimeController* const time_controller_;
Niels Möller7dc26b72017-12-06 10:27:48 +0100476 CpuOveruseDetectorProxy* overuse_detector_proxy_;
Henrik Boströmc55516d2020-05-11 16:29:22 +0200477 rtc::scoped_refptr<FakeResource> fake_cpu_resource_;
478 rtc::scoped_refptr<FakeResource> fake_quality_resource_;
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200479 FakeAdaptationConstraint fake_adaptation_constraint_;
perkj803d97f2016-11-01 11:45:46 -0700480};
481
Noah Richards51db4212019-06-12 06:59:12 -0700482// Simulates simulcast behavior and makes highest stream resolutions divisible
483// by 4.
484class CroppingVideoStreamFactory
485 : public VideoEncoderConfig::VideoStreamFactoryInterface {
486 public:
Åsa Persson17b29b92020-10-17 12:57:58 +0200487 CroppingVideoStreamFactory() {}
Noah Richards51db4212019-06-12 06:59:12 -0700488
489 private:
490 std::vector<VideoStream> CreateEncoderStreams(
491 int width,
492 int height,
493 const VideoEncoderConfig& encoder_config) override {
494 std::vector<VideoStream> streams = test::CreateVideoStreams(
495 width - width % 4, height - height % 4, encoder_config);
Noah Richards51db4212019-06-12 06:59:12 -0700496 return streams;
497 }
Noah Richards51db4212019-06-12 06:59:12 -0700498};
499
sprangb1ca0732017-02-01 08:38:12 -0800500class AdaptingFrameForwarder : public test::FrameForwarder {
501 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200502 explicit AdaptingFrameForwarder(TimeController* time_controller)
503 : time_controller_(time_controller), adaptation_enabled_(false) {}
asaperssonfab67072017-04-04 05:51:49 -0700504 ~AdaptingFrameForwarder() override {}
sprangb1ca0732017-02-01 08:38:12 -0800505
506 void set_adaptation_enabled(bool enabled) {
Markus Handella3765182020-07-08 13:13:32 +0200507 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -0800508 adaptation_enabled_ = enabled;
509 }
510
asaperssonfab67072017-04-04 05:51:49 -0700511 bool adaption_enabled() const {
Markus Handella3765182020-07-08 13:13:32 +0200512 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -0800513 return adaptation_enabled_;
514 }
515
Henrik Boström1124ed12021-02-25 10:30:39 +0100516 // The "last wants" is a snapshot of the previous rtc::VideoSinkWants where
517 // the resolution or frame rate was different than it is currently. If
518 // something else is modified, such as encoder resolutions, but the resolution
519 // and frame rate stays the same, last wants is not updated.
asapersson09f05612017-05-15 23:40:18 -0700520 rtc::VideoSinkWants last_wants() const {
Markus Handella3765182020-07-08 13:13:32 +0200521 MutexLock lock(&mutex_);
asapersson09f05612017-05-15 23:40:18 -0700522 return last_wants_;
523 }
524
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200525 absl::optional<int> last_sent_width() const { return last_width_; }
526 absl::optional<int> last_sent_height() const { return last_height_; }
Jonathan Yubc771b72017-12-08 17:04:29 -0800527
sprangb1ca0732017-02-01 08:38:12 -0800528 void IncomingCapturedFrame(const VideoFrame& video_frame) override {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200529 RTC_DCHECK(time_controller_->GetMainThread()->IsCurrent());
Markus Handell28c71802021-11-08 10:11:55 +0100530 time_controller_->AdvanceTime(TimeDelta::Zero());
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200531
sprangb1ca0732017-02-01 08:38:12 -0800532 int cropped_width = 0;
533 int cropped_height = 0;
534 int out_width = 0;
535 int out_height = 0;
sprangc5d62e22017-04-02 23:53:04 -0700536 if (adaption_enabled()) {
Harald Alvestrand97597c02021-11-04 12:01:23 +0000537 RTC_DLOG(LS_INFO) << "IncomingCapturedFrame: AdaptFrameResolution()"
538 << "w=" << video_frame.width()
539 << "h=" << video_frame.height();
sprangc5d62e22017-04-02 23:53:04 -0700540 if (adapter_.AdaptFrameResolution(
541 video_frame.width(), video_frame.height(),
542 video_frame.timestamp_us() * 1000, &cropped_width,
543 &cropped_height, &out_width, &out_height)) {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100544 VideoFrame adapted_frame =
545 VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200546 .set_video_frame_buffer(rtc::make_ref_counted<TestBuffer>(
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100547 nullptr, out_width, out_height))
Åsa Persson90719572021-04-08 19:05:30 +0200548 .set_ntp_time_ms(video_frame.ntp_time_ms())
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100549 .set_timestamp_ms(99)
550 .set_rotation(kVideoRotation_0)
551 .build();
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +0100552 if (video_frame.has_update_rect()) {
553 adapted_frame.set_update_rect(
554 video_frame.update_rect().ScaleWithFrame(
555 video_frame.width(), video_frame.height(), 0, 0,
556 video_frame.width(), video_frame.height(), out_width,
557 out_height));
558 }
sprangc5d62e22017-04-02 23:53:04 -0700559 test::FrameForwarder::IncomingCapturedFrame(adapted_frame);
Jonathan Yubc771b72017-12-08 17:04:29 -0800560 last_width_.emplace(adapted_frame.width());
561 last_height_.emplace(adapted_frame.height());
562 } else {
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200563 last_width_ = absl::nullopt;
564 last_height_ = absl::nullopt;
sprangc5d62e22017-04-02 23:53:04 -0700565 }
sprangb1ca0732017-02-01 08:38:12 -0800566 } else {
Harald Alvestrand97597c02021-11-04 12:01:23 +0000567 RTC_DLOG(LS_INFO) << "IncomingCapturedFrame: adaptation not enabled";
sprangb1ca0732017-02-01 08:38:12 -0800568 test::FrameForwarder::IncomingCapturedFrame(video_frame);
Jonathan Yubc771b72017-12-08 17:04:29 -0800569 last_width_.emplace(video_frame.width());
570 last_height_.emplace(video_frame.height());
sprangb1ca0732017-02-01 08:38:12 -0800571 }
572 }
573
Åsa Perssonf5f7e8e2021-06-09 22:55:38 +0200574 void OnOutputFormatRequest(int width, int height) {
575 absl::optional<std::pair<int, int>> target_aspect_ratio =
576 std::make_pair(width, height);
577 absl::optional<int> max_pixel_count = width * height;
578 absl::optional<int> max_fps;
579 adapter_.OnOutputFormatRequest(target_aspect_ratio, max_pixel_count,
580 max_fps);
581 }
582
sprangb1ca0732017-02-01 08:38:12 -0800583 void AddOrUpdateSink(rtc::VideoSinkInterface<VideoFrame>* sink,
584 const rtc::VideoSinkWants& wants) override {
Markus Handella3765182020-07-08 13:13:32 +0200585 MutexLock lock(&mutex_);
Henrik Boström1124ed12021-02-25 10:30:39 +0100586 rtc::VideoSinkWants prev_wants = sink_wants_locked();
587 bool did_adapt =
588 prev_wants.max_pixel_count != wants.max_pixel_count ||
589 prev_wants.target_pixel_count != wants.target_pixel_count ||
590 prev_wants.max_framerate_fps != wants.max_framerate_fps;
591 if (did_adapt) {
592 last_wants_ = prev_wants;
593 }
Rasmus Brandt287e4642019-11-15 16:56:01 +0100594 adapter_.OnSinkWants(wants);
Markus Handell16038ab2020-05-28 08:37:30 +0200595 test::FrameForwarder::AddOrUpdateSinkLocked(sink, wants);
sprangb1ca0732017-02-01 08:38:12 -0800596 }
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200597
598 TimeController* const time_controller_;
sprangb1ca0732017-02-01 08:38:12 -0800599 cricket::VideoAdapter adapter_;
Markus Handella3765182020-07-08 13:13:32 +0200600 bool adaptation_enabled_ RTC_GUARDED_BY(mutex_);
601 rtc::VideoSinkWants last_wants_ RTC_GUARDED_BY(mutex_);
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200602 absl::optional<int> last_width_;
603 absl::optional<int> last_height_;
sprangb1ca0732017-02-01 08:38:12 -0800604};
sprangc5d62e22017-04-02 23:53:04 -0700605
Niels Möller213618e2018-07-24 09:29:58 +0200606// TODO(nisse): Mock only VideoStreamEncoderObserver.
sprangc5d62e22017-04-02 23:53:04 -0700607class MockableSendStatisticsProxy : public SendStatisticsProxy {
608 public:
609 MockableSendStatisticsProxy(Clock* clock,
610 const VideoSendStream::Config& config,
611 VideoEncoderConfig::ContentType content_type)
612 : SendStatisticsProxy(clock, config, content_type) {}
613
614 VideoSendStream::Stats GetStats() override {
Markus Handella3765182020-07-08 13:13:32 +0200615 MutexLock lock(&lock_);
sprangc5d62e22017-04-02 23:53:04 -0700616 if (mock_stats_)
617 return *mock_stats_;
618 return SendStatisticsProxy::GetStats();
619 }
620
Niels Möller213618e2018-07-24 09:29:58 +0200621 int GetInputFrameRate() const override {
Markus Handella3765182020-07-08 13:13:32 +0200622 MutexLock lock(&lock_);
Niels Möller213618e2018-07-24 09:29:58 +0200623 if (mock_stats_)
624 return mock_stats_->input_frame_rate;
625 return SendStatisticsProxy::GetInputFrameRate();
626 }
sprangc5d62e22017-04-02 23:53:04 -0700627 void SetMockStats(const VideoSendStream::Stats& stats) {
Markus Handella3765182020-07-08 13:13:32 +0200628 MutexLock lock(&lock_);
sprangc5d62e22017-04-02 23:53:04 -0700629 mock_stats_.emplace(stats);
630 }
631
632 void ResetMockStats() {
Markus Handella3765182020-07-08 13:13:32 +0200633 MutexLock lock(&lock_);
sprangc5d62e22017-04-02 23:53:04 -0700634 mock_stats_.reset();
635 }
636
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200637 void SetDroppedFrameCallback(std::function<void(DropReason)> callback) {
638 on_frame_dropped_ = std::move(callback);
639 }
640
sprangc5d62e22017-04-02 23:53:04 -0700641 private:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200642 void OnFrameDropped(DropReason reason) override {
643 SendStatisticsProxy::OnFrameDropped(reason);
644 if (on_frame_dropped_)
645 on_frame_dropped_(reason);
646 }
647
Markus Handella3765182020-07-08 13:13:32 +0200648 mutable Mutex lock_;
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200649 absl::optional<VideoSendStream::Stats> mock_stats_ RTC_GUARDED_BY(lock_);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200650 std::function<void(DropReason)> on_frame_dropped_;
sprangc5d62e22017-04-02 23:53:04 -0700651};
652
Markus Handellb4e96d42021-11-05 12:00:55 +0100653class SimpleVideoStreamEncoderFactory {
654 public:
655 class AdaptedVideoStreamEncoder : public VideoStreamEncoder {
656 public:
657 using VideoStreamEncoder::VideoStreamEncoder;
658 ~AdaptedVideoStreamEncoder() { Stop(); }
659 };
660
Markus Handell8d87c462021-12-16 11:37:16 +0100661 class MockFakeEncoder : public test::FakeEncoder {
662 public:
663 using FakeEncoder::FakeEncoder;
664 MOCK_METHOD(CodecSpecificInfo,
665 EncodeHook,
666 (EncodedImage & encoded_image,
667 rtc::scoped_refptr<EncodedImageBuffer> buffer),
668 (override));
669 };
670
Markus Handellee225432021-11-29 12:35:12 +0100671 SimpleVideoStreamEncoderFactory() {
Markus Handellb4e96d42021-11-05 12:00:55 +0100672 encoder_settings_.encoder_factory = &encoder_factory_;
Markus Handellee225432021-11-29 12:35:12 +0100673 encoder_settings_.bitrate_allocator_factory =
674 bitrate_allocator_factory_.get();
Markus Handellb4e96d42021-11-05 12:00:55 +0100675 }
676
677 std::unique_ptr<AdaptedVideoStreamEncoder> Create(
Markus Handellee225432021-11-29 12:35:12 +0100678 std::unique_ptr<FrameCadenceAdapterInterface> zero_hertz_adapter,
679 TaskQueueBase** encoder_queue_ptr = nullptr) {
680 auto encoder_queue =
681 time_controller_.GetTaskQueueFactory()->CreateTaskQueue(
682 "EncoderQueue", TaskQueueFactory::Priority::NORMAL);
683 if (encoder_queue_ptr)
684 *encoder_queue_ptr = encoder_queue.get();
Markus Handellb4e96d42021-11-05 12:00:55 +0100685 auto result = std::make_unique<AdaptedVideoStreamEncoder>(
686 time_controller_.GetClock(),
687 /*number_of_cores=*/1,
688 /*stats_proxy=*/stats_proxy_.get(), encoder_settings_,
689 std::make_unique<CpuOveruseDetectorProxy>(/*stats_proxy=*/nullptr),
Markus Handellee225432021-11-29 12:35:12 +0100690 std::move(zero_hertz_adapter), std::move(encoder_queue),
Markus Handellb4e96d42021-11-05 12:00:55 +0100691 VideoStreamEncoder::BitrateAllocationCallbackType::
692 kVideoBitrateAllocation);
693 result->SetSink(&sink_, /*rotation_applied=*/false);
694 return result;
695 }
696
Markus Handell9a478b52021-11-18 16:07:01 +0100697 void DepleteTaskQueues() { time_controller_.AdvanceTime(TimeDelta::Zero()); }
Markus Handell8d87c462021-12-16 11:37:16 +0100698 MockFakeEncoder& GetMockFakeEncoder() { return mock_fake_encoder_; }
Markus Handell9a478b52021-11-18 16:07:01 +0100699
Markus Handellb4e96d42021-11-05 12:00:55 +0100700 private:
701 class NullEncoderSink : public VideoStreamEncoderInterface::EncoderSink {
702 public:
703 ~NullEncoderSink() override = default;
704 void OnEncoderConfigurationChanged(
705 std::vector<VideoStream> streams,
706 bool is_svc,
707 VideoEncoderConfig::ContentType content_type,
708 int min_transmit_bitrate_bps) override {}
709 void OnBitrateAllocationUpdated(
710 const VideoBitrateAllocation& allocation) override {}
711 void OnVideoLayersAllocationUpdated(
712 VideoLayersAllocation allocation) override {}
713 Result OnEncodedImage(
714 const EncodedImage& encoded_image,
715 const CodecSpecificInfo* codec_specific_info) override {
716 return Result(EncodedImageCallback::Result::OK);
717 }
718 };
719
Markus Handellee225432021-11-29 12:35:12 +0100720 GlobalSimulatedTimeController time_controller_{Timestamp::Millis(0)};
721 std::unique_ptr<TaskQueueFactory> task_queue_factory_{
722 time_controller_.CreateTaskQueueFactory()};
723 std::unique_ptr<MockableSendStatisticsProxy> stats_proxy_ =
724 std::make_unique<MockableSendStatisticsProxy>(
725 time_controller_.GetClock(),
726 VideoSendStream::Config(nullptr),
727 webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo);
728 std::unique_ptr<VideoBitrateAllocatorFactory> bitrate_allocator_factory_ =
729 CreateBuiltinVideoBitrateAllocatorFactory();
730 VideoStreamEncoderSettings encoder_settings_{
731 VideoEncoder::Capabilities(/*loss_notification=*/false)};
Markus Handell8d87c462021-12-16 11:37:16 +0100732 MockFakeEncoder mock_fake_encoder_{time_controller_.GetClock()};
733 test::VideoEncoderProxyFactory encoder_factory_{&mock_fake_encoder_};
Markus Handellb4e96d42021-11-05 12:00:55 +0100734 NullEncoderSink sink_;
735};
736
737class MockFrameCadenceAdapter : public FrameCadenceAdapterInterface {
738 public:
739 MOCK_METHOD(void, Initialize, (Callback * callback), (override));
Markus Handell8d87c462021-12-16 11:37:16 +0100740 MOCK_METHOD(void,
741 SetZeroHertzModeEnabled,
742 (absl::optional<ZeroHertzModeParams>),
743 (override));
Markus Handellb4e96d42021-11-05 12:00:55 +0100744 MOCK_METHOD(void, OnFrame, (const VideoFrame&), (override));
Markus Handellee225432021-11-29 12:35:12 +0100745 MOCK_METHOD(absl::optional<uint32_t>, GetInputFrameRateFps, (), (override));
746 MOCK_METHOD(void, UpdateFrameRate, (), (override));
Markus Handell8d87c462021-12-16 11:37:16 +0100747 MOCK_METHOD(void,
748 UpdateLayerQualityConvergence,
749 (int spatial_index, bool converged),
750 (override));
751 MOCK_METHOD(void,
752 UpdateLayerStatus,
753 (int spatial_index, bool enabled),
754 (override));
Markus Handellb4e96d42021-11-05 12:00:55 +0100755};
756
philipel9b058032020-02-10 11:30:00 +0100757class MockEncoderSelector
758 : public VideoEncoderFactory::EncoderSelectorInterface {
759 public:
Danil Chapovalov91fdc602020-05-14 19:17:51 +0200760 MOCK_METHOD(void,
761 OnCurrentEncoder,
762 (const SdpVideoFormat& format),
763 (override));
764 MOCK_METHOD(absl::optional<SdpVideoFormat>,
765 OnAvailableBitrate,
766 (const DataRate& rate),
767 (override));
768 MOCK_METHOD(absl::optional<SdpVideoFormat>, OnEncoderBroken, (), (override));
philipel9b058032020-02-10 11:30:00 +0100769};
770
perkj803d97f2016-11-01 11:45:46 -0700771} // namespace
772
mflodmancc3d4422017-08-03 08:27:51 -0700773class VideoStreamEncoderTest : public ::testing::Test {
perkj26091b12016-09-01 01:17:40 -0700774 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200775 static const int kDefaultTimeoutMs = 1000;
perkj26091b12016-09-01 01:17:40 -0700776
mflodmancc3d4422017-08-03 08:27:51 -0700777 VideoStreamEncoderTest()
perkj26091b12016-09-01 01:17:40 -0700778 : video_send_config_(VideoSendStream::Config(nullptr)),
perkjfa10b552016-10-02 23:45:26 -0700779 codec_width_(320),
780 codec_height_(240),
Åsa Persson8c1bf952018-09-13 10:42:19 +0200781 max_framerate_(kDefaultFramerate),
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200782 fake_encoder_(&time_controller_),
Niels Möller4db138e2018-04-19 09:04:13 +0200783 encoder_factory_(&fake_encoder_),
sprangc5d62e22017-04-02 23:53:04 -0700784 stats_proxy_(new MockableSendStatisticsProxy(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200785 time_controller_.GetClock(),
perkj803d97f2016-11-01 11:45:46 -0700786 video_send_config_,
787 webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo)),
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200788 sink_(&time_controller_, &fake_encoder_) {}
perkj26091b12016-09-01 01:17:40 -0700789
790 void SetUp() override {
perkj803d97f2016-11-01 11:45:46 -0700791 metrics::Reset();
perkj26091b12016-09-01 01:17:40 -0700792 video_send_config_ = VideoSendStream::Config(nullptr);
Niels Möller4db138e2018-04-19 09:04:13 +0200793 video_send_config_.encoder_settings.encoder_factory = &encoder_factory_;
Jiawei Ouc2ebe212018-11-08 10:02:56 -0800794 video_send_config_.encoder_settings.bitrate_allocator_factory =
Sergey Silkin5ee69672019-07-02 14:18:34 +0200795 &bitrate_allocator_factory_;
Niels Möller259a4972018-04-05 15:36:51 +0200796 video_send_config_.rtp.payload_name = "FAKE";
797 video_send_config_.rtp.payload_type = 125;
perkj26091b12016-09-01 01:17:40 -0700798
Per512ecb32016-09-23 15:52:06 +0200799 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +0200800 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Åsa Persson17107062020-10-08 08:57:51 +0200801 EXPECT_EQ(1u, video_encoder_config.simulcast_layers.size());
802 video_encoder_config.simulcast_layers[0].num_temporal_layers = 1;
803 video_encoder_config.simulcast_layers[0].max_framerate = max_framerate_;
Erik Språng08127a92016-11-16 16:41:30 +0100804 video_encoder_config_ = video_encoder_config.Copy();
sprang4847ae62017-06-27 07:06:52 -0700805
Niels Möllerf1338562018-04-26 09:51:47 +0200806 ConfigureEncoder(std::move(video_encoder_config));
asapersson5f7226f2016-11-25 04:37:00 -0800807 }
808
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100809 void ConfigureEncoder(
810 VideoEncoderConfig video_encoder_config,
811 VideoStreamEncoder::BitrateAllocationCallbackType
812 allocation_callback_type =
813 VideoStreamEncoder::BitrateAllocationCallbackType::
814 kVideoBitrateAllocationWhenScreenSharing) {
mflodmancc3d4422017-08-03 08:27:51 -0700815 if (video_stream_encoder_)
816 video_stream_encoder_->Stop();
Markus Handell9a478b52021-11-18 16:07:01 +0100817
818 auto encoder_queue = GetTaskQueueFactory()->CreateTaskQueue(
819 "EncoderQueue", TaskQueueFactory::Priority::NORMAL);
820 TaskQueueBase* encoder_queue_ptr = encoder_queue.get();
821 std::unique_ptr<FrameCadenceAdapterInterface> cadence_adapter =
822 FrameCadenceAdapterInterface::Create(time_controller_.GetClock(),
823 encoder_queue_ptr);
824 video_stream_encoder_ = std::make_unique<VideoStreamEncoderUnderTest>(
825 &time_controller_, std::move(cadence_adapter), std::move(encoder_queue),
826 stats_proxy_.get(), video_send_config_.encoder_settings,
827 allocation_callback_type);
Asa Persson606d3cb2021-10-04 10:07:11 +0200828 video_stream_encoder_->SetSink(&sink_, /*rotation_applied=*/false);
mflodmancc3d4422017-08-03 08:27:51 -0700829 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -0700830 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Asa Persson606d3cb2021-10-04 10:07:11 +0200831 video_stream_encoder_->SetStartBitrate(kTargetBitrate.bps());
mflodmancc3d4422017-08-03 08:27:51 -0700832 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +0200833 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -0700834 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
asapersson5f7226f2016-11-25 04:37:00 -0800835 }
836
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100837 void ResetEncoder(const std::string& payload_name,
838 size_t num_streams,
839 size_t num_temporal_layers,
840 unsigned char num_spatial_layers,
841 bool screenshare,
842 VideoStreamEncoder::BitrateAllocationCallbackType
843 allocation_callback_type =
844 VideoStreamEncoder::BitrateAllocationCallbackType::
845 kVideoBitrateAllocationWhenScreenSharing) {
Niels Möller259a4972018-04-05 15:36:51 +0200846 video_send_config_.rtp.payload_name = payload_name;
asapersson5f7226f2016-11-25 04:37:00 -0800847
848 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +0200849 test::FillEncoderConfiguration(PayloadStringToCodecType(payload_name),
850 num_streams, &video_encoder_config);
851 for (auto& layer : video_encoder_config.simulcast_layers) {
852 layer.num_temporal_layers = num_temporal_layers;
853 layer.max_framerate = kDefaultFramerate;
854 }
Erik Språngd7329ca2019-02-21 21:19:53 +0100855 video_encoder_config.max_bitrate_bps =
Asa Persson606d3cb2021-10-04 10:07:11 +0200856 num_streams == 1 ? kTargetBitrate.bps() : kSimulcastTargetBitrate.bps();
sprang4847ae62017-06-27 07:06:52 -0700857 video_encoder_config.content_type =
858 screenshare ? VideoEncoderConfig::ContentType::kScreen
859 : VideoEncoderConfig::ContentType::kRealtimeVideo;
emircanbbcc3562017-08-18 00:28:40 -0700860 if (payload_name == "VP9") {
861 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
862 vp9_settings.numberOfSpatialLayers = num_spatial_layers;
Mirta Dvornicic97910da2020-07-14 15:29:23 +0200863 vp9_settings.automaticResizeOn = num_spatial_layers <= 1;
emircanbbcc3562017-08-18 00:28:40 -0700864 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200865 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
866 vp9_settings);
emircanbbcc3562017-08-18 00:28:40 -0700867 }
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100868 ConfigureEncoder(std::move(video_encoder_config), allocation_callback_type);
perkj26091b12016-09-01 01:17:40 -0700869 }
870
sprang57c2fff2017-01-16 06:24:02 -0800871 VideoFrame CreateFrame(int64_t ntp_time_ms,
872 rtc::Event* destruction_event) const {
Åsa Persson90719572021-04-08 19:05:30 +0200873 return VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200874 .set_video_frame_buffer(rtc::make_ref_counted<TestBuffer>(
Åsa Persson90719572021-04-08 19:05:30 +0200875 destruction_event, codec_width_, codec_height_))
876 .set_ntp_time_ms(ntp_time_ms)
877 .set_timestamp_ms(99)
878 .set_rotation(kVideoRotation_0)
879 .build();
perkj26091b12016-09-01 01:17:40 -0700880 }
881
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100882 VideoFrame CreateFrameWithUpdatedPixel(int64_t ntp_time_ms,
883 rtc::Event* destruction_event,
884 int offset_x) const {
Åsa Persson90719572021-04-08 19:05:30 +0200885 return VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200886 .set_video_frame_buffer(rtc::make_ref_counted<TestBuffer>(
Åsa Persson90719572021-04-08 19:05:30 +0200887 destruction_event, codec_width_, codec_height_))
888 .set_ntp_time_ms(ntp_time_ms)
889 .set_timestamp_ms(99)
890 .set_rotation(kVideoRotation_0)
891 .set_update_rect(VideoFrame::UpdateRect{offset_x, 0, 1, 1})
892 .build();
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100893 }
894
sprang57c2fff2017-01-16 06:24:02 -0800895 VideoFrame CreateFrame(int64_t ntp_time_ms, int width, int height) const {
Erik Språng7444b192021-06-02 14:02:13 +0200896 auto buffer = rtc::make_ref_counted<TestBuffer>(nullptr, width, height);
897 I420Buffer::SetBlack(buffer.get());
Åsa Persson90719572021-04-08 19:05:30 +0200898 return VideoFrame::Builder()
Erik Språng7444b192021-06-02 14:02:13 +0200899 .set_video_frame_buffer(std::move(buffer))
Åsa Persson90719572021-04-08 19:05:30 +0200900 .set_ntp_time_ms(ntp_time_ms)
901 .set_timestamp_ms(ntp_time_ms)
902 .set_rotation(kVideoRotation_0)
903 .build();
perkj803d97f2016-11-01 11:45:46 -0700904 }
905
Evan Shrubsole895556e2020-10-05 09:15:13 +0200906 VideoFrame CreateNV12Frame(int64_t ntp_time_ms, int width, int height) const {
Åsa Persson90719572021-04-08 19:05:30 +0200907 return VideoFrame::Builder()
908 .set_video_frame_buffer(NV12Buffer::Create(width, height))
909 .set_ntp_time_ms(ntp_time_ms)
910 .set_timestamp_ms(ntp_time_ms)
911 .set_rotation(kVideoRotation_0)
912 .build();
Evan Shrubsole895556e2020-10-05 09:15:13 +0200913 }
914
Noah Richards51db4212019-06-12 06:59:12 -0700915 VideoFrame CreateFakeNativeFrame(int64_t ntp_time_ms,
916 rtc::Event* destruction_event,
917 int width,
918 int height) const {
Åsa Persson90719572021-04-08 19:05:30 +0200919 return VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200920 .set_video_frame_buffer(rtc::make_ref_counted<FakeNativeBuffer>(
Åsa Persson90719572021-04-08 19:05:30 +0200921 destruction_event, width, height))
922 .set_ntp_time_ms(ntp_time_ms)
923 .set_timestamp_ms(99)
924 .set_rotation(kVideoRotation_0)
925 .build();
Noah Richards51db4212019-06-12 06:59:12 -0700926 }
927
Evan Shrubsole895556e2020-10-05 09:15:13 +0200928 VideoFrame CreateFakeNV12NativeFrame(int64_t ntp_time_ms,
929 rtc::Event* destruction_event,
930 int width,
931 int height) const {
Åsa Persson90719572021-04-08 19:05:30 +0200932 return VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200933 .set_video_frame_buffer(rtc::make_ref_counted<FakeNV12NativeBuffer>(
Åsa Persson90719572021-04-08 19:05:30 +0200934 destruction_event, width, height))
935 .set_ntp_time_ms(ntp_time_ms)
936 .set_timestamp_ms(99)
937 .set_rotation(kVideoRotation_0)
938 .build();
Evan Shrubsole895556e2020-10-05 09:15:13 +0200939 }
940
Noah Richards51db4212019-06-12 06:59:12 -0700941 VideoFrame CreateFakeNativeFrame(int64_t ntp_time_ms,
942 rtc::Event* destruction_event) const {
943 return CreateFakeNativeFrame(ntp_time_ms, destruction_event, codec_width_,
944 codec_height_);
945 }
946
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100947 void VerifyAllocatedBitrate(const VideoBitrateAllocation& expected_bitrate) {
Henrik Boström381d1092020-05-12 18:49:07 +0200948 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +0200949 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100950
951 video_source_.IncomingCapturedFrame(
952 CreateFrame(1, codec_width_, codec_height_));
953 WaitForEncodedFrame(1);
Per Kjellanderdcef6412020-10-07 15:09:05 +0200954 EXPECT_EQ(expected_bitrate, sink_.GetLastVideoBitrateAllocation());
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100955 }
956
sprang4847ae62017-06-27 07:06:52 -0700957 void WaitForEncodedFrame(int64_t expected_ntp_time) {
958 sink_.WaitForEncodedFrame(expected_ntp_time);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200959 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700960 }
961
962 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time, int64_t timeout_ms) {
963 bool ok = sink_.TimedWaitForEncodedFrame(expected_ntp_time, timeout_ms);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200964 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700965 return ok;
966 }
967
968 void WaitForEncodedFrame(uint32_t expected_width, uint32_t expected_height) {
969 sink_.WaitForEncodedFrame(expected_width, expected_height);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200970 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700971 }
972
973 void ExpectDroppedFrame() {
974 sink_.ExpectDroppedFrame();
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200975 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700976 }
977
978 bool WaitForFrame(int64_t timeout_ms) {
979 bool ok = sink_.WaitForFrame(timeout_ms);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200980 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700981 return ok;
982 }
983
perkj26091b12016-09-01 01:17:40 -0700984 class TestEncoder : public test::FakeEncoder {
985 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200986 explicit TestEncoder(TimeController* time_controller)
987 : FakeEncoder(time_controller->GetClock()),
988 time_controller_(time_controller) {
989 RTC_DCHECK(time_controller_);
990 }
perkj26091b12016-09-01 01:17:40 -0700991
Erik Språngaed30702018-11-05 12:57:17 +0100992 VideoEncoder::EncoderInfo GetEncoderInfo() const override {
Markus Handella3765182020-07-08 13:13:32 +0200993 MutexLock lock(&local_mutex_);
Erik Språng9d69cbe2020-10-22 17:44:42 +0200994 EncoderInfo info = FakeEncoder::GetEncoderInfo();
Erik Språngb7cb7b52019-02-26 15:52:33 +0100995 if (initialized_ == EncoderState::kInitialized) {
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100996 if (quality_scaling_) {
Åsa Perssone644a032019-11-08 15:56:00 +0100997 info.scaling_settings = VideoEncoder::ScalingSettings(
998 kQpLow, kQpHigh, kMinPixelsPerFrame);
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100999 }
1000 info.is_hardware_accelerated = is_hardware_accelerated_;
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001001 for (int i = 0; i < kMaxSpatialLayers; ++i) {
1002 if (temporal_layers_supported_[i]) {
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01001003 info.fps_allocation[i].clear();
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001004 int num_layers = temporal_layers_supported_[i].value() ? 2 : 1;
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01001005 for (int tid = 0; tid < num_layers; ++tid)
1006 info.fps_allocation[i].push_back(255 / (num_layers - tid));
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001007 }
1008 }
Erik Språngaed30702018-11-05 12:57:17 +01001009 }
Sergey Silkin6456e352019-07-08 17:56:40 +02001010
1011 info.resolution_bitrate_limits = resolution_bitrate_limits_;
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01001012 info.requested_resolution_alignment = requested_resolution_alignment_;
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02001013 info.apply_alignment_to_all_simulcast_layers =
1014 apply_alignment_to_all_simulcast_layers_;
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001015 info.preferred_pixel_formats = preferred_pixel_formats_;
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08001016 if (is_qp_trusted_.has_value()) {
1017 info.is_qp_trusted = is_qp_trusted_;
1018 }
Erik Språngaed30702018-11-05 12:57:17 +01001019 return info;
kthelgason876222f2016-11-29 01:44:11 -08001020 }
1021
Erik Språngb7cb7b52019-02-26 15:52:33 +01001022 int32_t RegisterEncodeCompleteCallback(
1023 EncodedImageCallback* callback) override {
Markus Handella3765182020-07-08 13:13:32 +02001024 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001025 encoded_image_callback_ = callback;
1026 return FakeEncoder::RegisterEncodeCompleteCallback(callback);
1027 }
1028
perkjfa10b552016-10-02 23:45:26 -07001029 void ContinueEncode() { continue_encode_event_.Set(); }
1030
1031 void CheckLastTimeStampsMatch(int64_t ntp_time_ms,
1032 uint32_t timestamp) const {
Markus Handella3765182020-07-08 13:13:32 +02001033 MutexLock lock(&local_mutex_);
perkjfa10b552016-10-02 23:45:26 -07001034 EXPECT_EQ(timestamp_, timestamp);
1035 EXPECT_EQ(ntp_time_ms_, ntp_time_ms);
1036 }
1037
kthelgason2fc52542017-03-03 00:24:41 -08001038 void SetQualityScaling(bool b) {
Markus Handella3765182020-07-08 13:13:32 +02001039 MutexLock lock(&local_mutex_);
kthelgason2fc52542017-03-03 00:24:41 -08001040 quality_scaling_ = b;
1041 }
kthelgasonad9010c2017-02-14 00:46:51 -08001042
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01001043 void SetRequestedResolutionAlignment(int requested_resolution_alignment) {
Markus Handella3765182020-07-08 13:13:32 +02001044 MutexLock lock(&local_mutex_);
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01001045 requested_resolution_alignment_ = requested_resolution_alignment;
1046 }
1047
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02001048 void SetApplyAlignmentToAllSimulcastLayers(bool b) {
1049 MutexLock lock(&local_mutex_);
1050 apply_alignment_to_all_simulcast_layers_ = b;
1051 }
1052
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01001053 void SetIsHardwareAccelerated(bool is_hardware_accelerated) {
Markus Handella3765182020-07-08 13:13:32 +02001054 MutexLock lock(&local_mutex_);
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01001055 is_hardware_accelerated_ = is_hardware_accelerated;
1056 }
1057
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001058 void SetTemporalLayersSupported(size_t spatial_idx, bool supported) {
1059 RTC_DCHECK_LT(spatial_idx, kMaxSpatialLayers);
Markus Handella3765182020-07-08 13:13:32 +02001060 MutexLock lock(&local_mutex_);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001061 temporal_layers_supported_[spatial_idx] = supported;
1062 }
1063
Sergey Silkin6456e352019-07-08 17:56:40 +02001064 void SetResolutionBitrateLimits(
1065 std::vector<ResolutionBitrateLimits> thresholds) {
Markus Handella3765182020-07-08 13:13:32 +02001066 MutexLock lock(&local_mutex_);
Sergey Silkin6456e352019-07-08 17:56:40 +02001067 resolution_bitrate_limits_ = thresholds;
1068 }
1069
sprangfe627f32017-03-29 08:24:59 -07001070 void ForceInitEncodeFailure(bool force_failure) {
Markus Handella3765182020-07-08 13:13:32 +02001071 MutexLock lock(&local_mutex_);
sprangfe627f32017-03-29 08:24:59 -07001072 force_init_encode_failed_ = force_failure;
1073 }
1074
Niels Möller6bb5ab92019-01-11 11:11:10 +01001075 void SimulateOvershoot(double rate_factor) {
Markus Handella3765182020-07-08 13:13:32 +02001076 MutexLock lock(&local_mutex_);
Niels Möller6bb5ab92019-01-11 11:11:10 +01001077 rate_factor_ = rate_factor;
1078 }
1079
Erik Språngd7329ca2019-02-21 21:19:53 +01001080 uint32_t GetLastFramerate() const {
Markus Handella3765182020-07-08 13:13:32 +02001081 MutexLock lock(&local_mutex_);
Niels Möller6bb5ab92019-01-11 11:11:10 +01001082 return last_framerate_;
1083 }
1084
Erik Språngd7329ca2019-02-21 21:19:53 +01001085 VideoFrame::UpdateRect GetLastUpdateRect() const {
Markus Handella3765182020-07-08 13:13:32 +02001086 MutexLock lock(&local_mutex_);
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01001087 return last_update_rect_;
1088 }
1089
Niels Möller87e2d782019-03-07 10:18:23 +01001090 const std::vector<VideoFrameType>& LastFrameTypes() const {
Markus Handella3765182020-07-08 13:13:32 +02001091 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001092 return last_frame_types_;
1093 }
1094
1095 void InjectFrame(const VideoFrame& input_image, bool keyframe) {
Niels Möller87e2d782019-03-07 10:18:23 +01001096 const std::vector<VideoFrameType> frame_type = {
Niels Möller8f7ce222019-03-21 15:43:58 +01001097 keyframe ? VideoFrameType::kVideoFrameKey
1098 : VideoFrameType::kVideoFrameDelta};
Erik Språngd7329ca2019-02-21 21:19:53 +01001099 {
Markus Handella3765182020-07-08 13:13:32 +02001100 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001101 last_frame_types_ = frame_type;
1102 }
Niels Möllerb859b322019-03-07 12:40:01 +01001103 FakeEncoder::Encode(input_image, &frame_type);
Erik Språngd7329ca2019-02-21 21:19:53 +01001104 }
1105
Sergey Silkin0e42cf72021-03-15 10:12:57 +01001106 void InjectEncodedImage(const EncodedImage& image,
1107 const CodecSpecificInfo* codec_specific_info) {
Markus Handella3765182020-07-08 13:13:32 +02001108 MutexLock lock(&local_mutex_);
Sergey Silkin0e42cf72021-03-15 10:12:57 +01001109 encoded_image_callback_->OnEncodedImage(image, codec_specific_info);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001110 }
1111
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001112 void SetEncodedImageData(
1113 rtc::scoped_refptr<EncodedImageBufferInterface> encoded_image_data) {
Markus Handella3765182020-07-08 13:13:32 +02001114 MutexLock lock(&local_mutex_);
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001115 encoded_image_data_ = encoded_image_data;
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001116 }
1117
Erik Språngd7329ca2019-02-21 21:19:53 +01001118 void ExpectNullFrame() {
Markus Handella3765182020-07-08 13:13:32 +02001119 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001120 expect_null_frame_ = true;
1121 }
1122
Erik Språng5056af02019-09-02 15:53:11 +02001123 absl::optional<VideoEncoder::RateControlParameters>
1124 GetAndResetLastRateControlSettings() {
1125 auto settings = last_rate_control_settings_;
1126 last_rate_control_settings_.reset();
1127 return settings;
Erik Språngd7329ca2019-02-21 21:19:53 +01001128 }
1129
Henrik Boström56db9ff2021-03-24 09:06:45 +01001130 int GetLastInputWidth() const {
1131 MutexLock lock(&local_mutex_);
1132 return last_input_width_;
1133 }
1134
1135 int GetLastInputHeight() const {
1136 MutexLock lock(&local_mutex_);
1137 return last_input_height_;
1138 }
1139
Evan Shrubsole895556e2020-10-05 09:15:13 +02001140 absl::optional<VideoFrameBuffer::Type> GetLastInputPixelFormat() {
1141 MutexLock lock(&local_mutex_);
1142 return last_input_pixel_format_;
1143 }
1144
Evan Shrubsole7c079f62019-09-26 09:55:03 +02001145 int GetNumSetRates() const {
Markus Handella3765182020-07-08 13:13:32 +02001146 MutexLock lock(&local_mutex_);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02001147 return num_set_rates_;
1148 }
1149
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001150 void SetPreferredPixelFormats(
1151 absl::InlinedVector<VideoFrameBuffer::Type, kMaxPreferredPixelFormats>
1152 pixel_formats) {
1153 MutexLock lock(&local_mutex_);
1154 preferred_pixel_formats_ = std::move(pixel_formats);
1155 }
1156
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08001157 void SetIsQpTrusted(absl::optional<bool> trusted) {
1158 MutexLock lock(&local_mutex_);
1159 is_qp_trusted_ = trusted;
1160 }
1161
perkjfa10b552016-10-02 23:45:26 -07001162 private:
perkj26091b12016-09-01 01:17:40 -07001163 int32_t Encode(const VideoFrame& input_image,
Niels Möller87e2d782019-03-07 10:18:23 +01001164 const std::vector<VideoFrameType>* frame_types) override {
perkj26091b12016-09-01 01:17:40 -07001165 {
Markus Handella3765182020-07-08 13:13:32 +02001166 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001167 if (expect_null_frame_) {
1168 EXPECT_EQ(input_image.timestamp(), 0u);
1169 EXPECT_EQ(input_image.width(), 1);
1170 last_frame_types_ = *frame_types;
1171 expect_null_frame_ = false;
1172 } else {
1173 EXPECT_GT(input_image.timestamp(), timestamp_);
1174 EXPECT_GT(input_image.ntp_time_ms(), ntp_time_ms_);
1175 EXPECT_EQ(input_image.timestamp(), input_image.ntp_time_ms() * 90);
1176 }
perkj26091b12016-09-01 01:17:40 -07001177
1178 timestamp_ = input_image.timestamp();
1179 ntp_time_ms_ = input_image.ntp_time_ms();
perkj803d97f2016-11-01 11:45:46 -07001180 last_input_width_ = input_image.width();
1181 last_input_height_ = input_image.height();
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01001182 last_update_rect_ = input_image.update_rect();
Erik Språngd7329ca2019-02-21 21:19:53 +01001183 last_frame_types_ = *frame_types;
Evan Shrubsole895556e2020-10-05 09:15:13 +02001184 last_input_pixel_format_ = input_image.video_frame_buffer()->type();
perkj26091b12016-09-01 01:17:40 -07001185 }
Niels Möllerb859b322019-03-07 12:40:01 +01001186 int32_t result = FakeEncoder::Encode(input_image, frame_types);
perkj26091b12016-09-01 01:17:40 -07001187 return result;
1188 }
1189
Niels Möller08ae7ce2020-09-23 15:58:12 +02001190 CodecSpecificInfo EncodeHook(
1191 EncodedImage& encoded_image,
1192 rtc::scoped_refptr<EncodedImageBuffer> buffer) override {
Danil Chapovalov2549f172020-08-12 17:30:36 +02001193 CodecSpecificInfo codec_specific;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001194 {
1195 MutexLock lock(&mutex_);
Danil Chapovalov2549f172020-08-12 17:30:36 +02001196 codec_specific.codecType = config_.codecType;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001197 }
1198 MutexLock lock(&local_mutex_);
1199 if (encoded_image_data_) {
Danil Chapovalov2549f172020-08-12 17:30:36 +02001200 encoded_image.SetEncodedData(encoded_image_data_);
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001201 }
Danil Chapovalov2549f172020-08-12 17:30:36 +02001202 return codec_specific;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001203 }
1204
sprangfe627f32017-03-29 08:24:59 -07001205 int32_t InitEncode(const VideoCodec* config,
Elad Alon370f93a2019-06-11 14:57:57 +02001206 const Settings& settings) override {
1207 int res = FakeEncoder::InitEncode(config, settings);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001208
Markus Handella3765182020-07-08 13:13:32 +02001209 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001210 EXPECT_EQ(initialized_, EncoderState::kUninitialized);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001211
Erik Språng82fad3d2018-03-21 09:57:23 +01001212 if (config->codecType == kVideoCodecVP8) {
sprangfe627f32017-03-29 08:24:59 -07001213 // Simulate setting up temporal layers, in order to validate the life
1214 // cycle of these objects.
Elad Aloncde8ab22019-03-20 11:56:20 +01001215 Vp8TemporalLayersFactory factory;
Elad Alon45befc52019-07-02 11:20:09 +02001216 frame_buffer_controller_ =
1217 factory.Create(*config, settings, &fec_controller_override_);
sprangfe627f32017-03-29 08:24:59 -07001218 }
Erik Språngb7cb7b52019-02-26 15:52:33 +01001219 if (force_init_encode_failed_) {
1220 initialized_ = EncoderState::kInitializationFailed;
sprangfe627f32017-03-29 08:24:59 -07001221 return -1;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001222 }
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01001223
Erik Språngb7cb7b52019-02-26 15:52:33 +01001224 initialized_ = EncoderState::kInitialized;
sprangfe627f32017-03-29 08:24:59 -07001225 return res;
1226 }
1227
Erik Språngb7cb7b52019-02-26 15:52:33 +01001228 int32_t Release() override {
Markus Handella3765182020-07-08 13:13:32 +02001229 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001230 EXPECT_NE(initialized_, EncoderState::kUninitialized);
1231 initialized_ = EncoderState::kUninitialized;
1232 return FakeEncoder::Release();
1233 }
1234
Erik Språng16cb8f52019-04-12 13:59:09 +02001235 void SetRates(const RateControlParameters& parameters) {
Markus Handella3765182020-07-08 13:13:32 +02001236 MutexLock lock(&local_mutex_);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02001237 num_set_rates_++;
Niels Möller6bb5ab92019-01-11 11:11:10 +01001238 VideoBitrateAllocation adjusted_rate_allocation;
1239 for (size_t si = 0; si < kMaxSpatialLayers; ++si) {
1240 for (size_t ti = 0; ti < kMaxTemporalStreams; ++ti) {
Erik Språng16cb8f52019-04-12 13:59:09 +02001241 if (parameters.bitrate.HasBitrate(si, ti)) {
Niels Möller6bb5ab92019-01-11 11:11:10 +01001242 adjusted_rate_allocation.SetBitrate(
1243 si, ti,
Erik Språng16cb8f52019-04-12 13:59:09 +02001244 static_cast<uint32_t>(parameters.bitrate.GetBitrate(si, ti) *
Niels Möller6bb5ab92019-01-11 11:11:10 +01001245 rate_factor_));
1246 }
1247 }
1248 }
Erik Språng16cb8f52019-04-12 13:59:09 +02001249 last_framerate_ = static_cast<uint32_t>(parameters.framerate_fps + 0.5);
Erik Språng5056af02019-09-02 15:53:11 +02001250 last_rate_control_settings_ = parameters;
Erik Språng16cb8f52019-04-12 13:59:09 +02001251 RateControlParameters adjusted_paramters = parameters;
1252 adjusted_paramters.bitrate = adjusted_rate_allocation;
1253 FakeEncoder::SetRates(adjusted_paramters);
Niels Möller6bb5ab92019-01-11 11:11:10 +01001254 }
1255
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001256 TimeController* const time_controller_;
Markus Handella3765182020-07-08 13:13:32 +02001257 mutable Mutex local_mutex_;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001258 enum class EncoderState {
1259 kUninitialized,
1260 kInitializationFailed,
1261 kInitialized
Markus Handella3765182020-07-08 13:13:32 +02001262 } initialized_ RTC_GUARDED_BY(local_mutex_) = EncoderState::kUninitialized;
perkj26091b12016-09-01 01:17:40 -07001263 rtc::Event continue_encode_event_;
Markus Handella3765182020-07-08 13:13:32 +02001264 uint32_t timestamp_ RTC_GUARDED_BY(local_mutex_) = 0;
1265 int64_t ntp_time_ms_ RTC_GUARDED_BY(local_mutex_) = 0;
1266 int last_input_width_ RTC_GUARDED_BY(local_mutex_) = 0;
1267 int last_input_height_ RTC_GUARDED_BY(local_mutex_) = 0;
1268 bool quality_scaling_ RTC_GUARDED_BY(local_mutex_) = true;
1269 int requested_resolution_alignment_ RTC_GUARDED_BY(local_mutex_) = 1;
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02001270 bool apply_alignment_to_all_simulcast_layers_ RTC_GUARDED_BY(local_mutex_) =
1271 false;
Markus Handella3765182020-07-08 13:13:32 +02001272 bool is_hardware_accelerated_ RTC_GUARDED_BY(local_mutex_) = false;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001273 rtc::scoped_refptr<EncodedImageBufferInterface> encoded_image_data_
1274 RTC_GUARDED_BY(local_mutex_);
Elad Aloncde8ab22019-03-20 11:56:20 +01001275 std::unique_ptr<Vp8FrameBufferController> frame_buffer_controller_
Markus Handella3765182020-07-08 13:13:32 +02001276 RTC_GUARDED_BY(local_mutex_);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001277 absl::optional<bool>
1278 temporal_layers_supported_[kMaxSpatialLayers] RTC_GUARDED_BY(
Markus Handella3765182020-07-08 13:13:32 +02001279 local_mutex_);
1280 bool force_init_encode_failed_ RTC_GUARDED_BY(local_mutex_) = false;
1281 double rate_factor_ RTC_GUARDED_BY(local_mutex_) = 1.0;
1282 uint32_t last_framerate_ RTC_GUARDED_BY(local_mutex_) = 0;
Erik Språng5056af02019-09-02 15:53:11 +02001283 absl::optional<VideoEncoder::RateControlParameters>
1284 last_rate_control_settings_;
Markus Handella3765182020-07-08 13:13:32 +02001285 VideoFrame::UpdateRect last_update_rect_ RTC_GUARDED_BY(local_mutex_) = {
1286 0, 0, 0, 0};
Niels Möller87e2d782019-03-07 10:18:23 +01001287 std::vector<VideoFrameType> last_frame_types_;
Erik Språngd7329ca2019-02-21 21:19:53 +01001288 bool expect_null_frame_ = false;
Markus Handella3765182020-07-08 13:13:32 +02001289 EncodedImageCallback* encoded_image_callback_ RTC_GUARDED_BY(local_mutex_) =
1290 nullptr;
Evan Shrubsolefb862742020-03-16 16:18:36 +01001291 NiceMock<MockFecControllerOverride> fec_controller_override_;
Sergey Silkin6456e352019-07-08 17:56:40 +02001292 std::vector<ResolutionBitrateLimits> resolution_bitrate_limits_
Markus Handella3765182020-07-08 13:13:32 +02001293 RTC_GUARDED_BY(local_mutex_);
1294 int num_set_rates_ RTC_GUARDED_BY(local_mutex_) = 0;
Evan Shrubsole895556e2020-10-05 09:15:13 +02001295 absl::optional<VideoFrameBuffer::Type> last_input_pixel_format_
1296 RTC_GUARDED_BY(local_mutex_);
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001297 absl::InlinedVector<VideoFrameBuffer::Type, kMaxPreferredPixelFormats>
1298 preferred_pixel_formats_ RTC_GUARDED_BY(local_mutex_);
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08001299 absl::optional<bool> is_qp_trusted_ RTC_GUARDED_BY(local_mutex_);
perkj26091b12016-09-01 01:17:40 -07001300 };
1301
mflodmancc3d4422017-08-03 08:27:51 -07001302 class TestSink : public VideoStreamEncoder::EncoderSink {
perkj26091b12016-09-01 01:17:40 -07001303 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001304 TestSink(TimeController* time_controller, TestEncoder* test_encoder)
1305 : time_controller_(time_controller), test_encoder_(test_encoder) {
1306 RTC_DCHECK(time_controller_);
1307 }
perkj26091b12016-09-01 01:17:40 -07001308
perkj26091b12016-09-01 01:17:40 -07001309 void WaitForEncodedFrame(int64_t expected_ntp_time) {
sprang4847ae62017-06-27 07:06:52 -07001310 EXPECT_TRUE(
1311 TimedWaitForEncodedFrame(expected_ntp_time, kDefaultTimeoutMs));
1312 }
1313
1314 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time,
1315 int64_t timeout_ms) {
perkj26091b12016-09-01 01:17:40 -07001316 uint32_t timestamp = 0;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001317 if (!WaitForFrame(timeout_ms))
sprang4847ae62017-06-27 07:06:52 -07001318 return false;
perkj26091b12016-09-01 01:17:40 -07001319 {
Markus Handella3765182020-07-08 13:13:32 +02001320 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -08001321 timestamp = last_timestamp_;
perkj26091b12016-09-01 01:17:40 -07001322 }
1323 test_encoder_->CheckLastTimeStampsMatch(expected_ntp_time, timestamp);
sprang4847ae62017-06-27 07:06:52 -07001324 return true;
perkj26091b12016-09-01 01:17:40 -07001325 }
1326
sprangb1ca0732017-02-01 08:38:12 -08001327 void WaitForEncodedFrame(uint32_t expected_width,
1328 uint32_t expected_height) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001329 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Åsa Perssonc74d8da2017-12-04 14:13:56 +01001330 CheckLastFrameSizeMatches(expected_width, expected_height);
sprangc5d62e22017-04-02 23:53:04 -07001331 }
1332
Åsa Perssonc74d8da2017-12-04 14:13:56 +01001333 void CheckLastFrameSizeMatches(uint32_t expected_width,
sprangc5d62e22017-04-02 23:53:04 -07001334 uint32_t expected_height) {
sprangb1ca0732017-02-01 08:38:12 -08001335 uint32_t width = 0;
1336 uint32_t height = 0;
sprangb1ca0732017-02-01 08:38:12 -08001337 {
Markus Handella3765182020-07-08 13:13:32 +02001338 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -08001339 width = last_width_;
1340 height = last_height_;
1341 }
1342 EXPECT_EQ(expected_height, height);
1343 EXPECT_EQ(expected_width, width);
1344 }
1345
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001346 void CheckLastFrameRotationMatches(VideoRotation expected_rotation) {
1347 VideoRotation rotation;
1348 {
Markus Handella3765182020-07-08 13:13:32 +02001349 MutexLock lock(&mutex_);
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001350 rotation = last_rotation_;
1351 }
1352 EXPECT_EQ(expected_rotation, rotation);
1353 }
1354
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001355 void ExpectDroppedFrame() { EXPECT_FALSE(WaitForFrame(100)); }
kthelgason2bc68642017-02-07 07:02:22 -08001356
sprangc5d62e22017-04-02 23:53:04 -07001357 bool WaitForFrame(int64_t timeout_ms) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001358 RTC_DCHECK(time_controller_->GetMainThread()->IsCurrent());
Markus Handell28c71802021-11-08 10:11:55 +01001359 time_controller_->AdvanceTime(TimeDelta::Zero());
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001360 bool ret = encoded_frame_event_.Wait(timeout_ms);
Markus Handell28c71802021-11-08 10:11:55 +01001361 time_controller_->AdvanceTime(TimeDelta::Zero());
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001362 return ret;
sprangc5d62e22017-04-02 23:53:04 -07001363 }
1364
perkj26091b12016-09-01 01:17:40 -07001365 void SetExpectNoFrames() {
Markus Handella3765182020-07-08 13:13:32 +02001366 MutexLock lock(&mutex_);
perkj26091b12016-09-01 01:17:40 -07001367 expect_frames_ = false;
1368 }
1369
asaperssonfab67072017-04-04 05:51:49 -07001370 int number_of_reconfigurations() const {
Markus Handella3765182020-07-08 13:13:32 +02001371 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001372 return number_of_reconfigurations_;
1373 }
1374
asaperssonfab67072017-04-04 05:51:49 -07001375 int last_min_transmit_bitrate() const {
Markus Handella3765182020-07-08 13:13:32 +02001376 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001377 return min_transmit_bitrate_bps_;
1378 }
1379
Erik Språngd7329ca2019-02-21 21:19:53 +01001380 void SetNumExpectedLayers(size_t num_layers) {
Markus Handella3765182020-07-08 13:13:32 +02001381 MutexLock lock(&mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001382 num_expected_layers_ = num_layers;
1383 }
1384
Erik Språngb7cb7b52019-02-26 15:52:33 +01001385 int64_t GetLastCaptureTimeMs() const {
Markus Handella3765182020-07-08 13:13:32 +02001386 MutexLock lock(&mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001387 return last_capture_time_ms_;
1388 }
1389
Sergey Silkin0e42cf72021-03-15 10:12:57 +01001390 const EncodedImage& GetLastEncodedImage() {
1391 MutexLock lock(&mutex_);
1392 return last_encoded_image_;
1393 }
1394
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001395 std::vector<uint8_t> GetLastEncodedImageData() {
Markus Handella3765182020-07-08 13:13:32 +02001396 MutexLock lock(&mutex_);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001397 return std::move(last_encoded_image_data_);
1398 }
1399
Per Kjellanderdcef6412020-10-07 15:09:05 +02001400 VideoBitrateAllocation GetLastVideoBitrateAllocation() {
1401 MutexLock lock(&mutex_);
1402 return last_bitrate_allocation_;
1403 }
1404
1405 int number_of_bitrate_allocations() const {
1406 MutexLock lock(&mutex_);
1407 return number_of_bitrate_allocations_;
1408 }
1409
Per Kjellandera9434842020-10-15 17:53:22 +02001410 VideoLayersAllocation GetLastVideoLayersAllocation() {
1411 MutexLock lock(&mutex_);
1412 return last_layers_allocation_;
1413 }
1414
1415 int number_of_layers_allocations() const {
1416 MutexLock lock(&mutex_);
1417 return number_of_layers_allocations_;
1418 }
1419
perkj26091b12016-09-01 01:17:40 -07001420 private:
sergeyu2cb155a2016-11-04 11:39:29 -07001421 Result OnEncodedImage(
1422 const EncodedImage& encoded_image,
Danil Chapovalov2549f172020-08-12 17:30:36 +02001423 const CodecSpecificInfo* codec_specific_info) override {
Markus Handella3765182020-07-08 13:13:32 +02001424 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001425 EXPECT_TRUE(expect_frames_);
Sergey Silkin0e42cf72021-03-15 10:12:57 +01001426 last_encoded_image_ = EncodedImage(encoded_image);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001427 last_encoded_image_data_ = std::vector<uint8_t>(
1428 encoded_image.data(), encoded_image.data() + encoded_image.size());
Erik Språngd7329ca2019-02-21 21:19:53 +01001429 uint32_t timestamp = encoded_image.Timestamp();
1430 if (last_timestamp_ != timestamp) {
1431 num_received_layers_ = 1;
Erik Språng7444b192021-06-02 14:02:13 +02001432 last_width_ = encoded_image._encodedWidth;
1433 last_height_ = encoded_image._encodedHeight;
Erik Språngd7329ca2019-02-21 21:19:53 +01001434 } else {
1435 ++num_received_layers_;
Erik Språng7444b192021-06-02 14:02:13 +02001436 last_width_ = std::max(encoded_image._encodedWidth, last_width_);
1437 last_height_ = std::max(encoded_image._encodedHeight, last_height_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001438 }
1439 last_timestamp_ = timestamp;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001440 last_capture_time_ms_ = encoded_image.capture_time_ms_;
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001441 last_rotation_ = encoded_image.rotation_;
Erik Språngd7329ca2019-02-21 21:19:53 +01001442 if (num_received_layers_ == num_expected_layers_) {
1443 encoded_frame_event_.Set();
1444 }
sprangb1ca0732017-02-01 08:38:12 -08001445 return Result(Result::OK, last_timestamp_);
Per512ecb32016-09-23 15:52:06 +02001446 }
1447
Rasmus Brandtc402dbe2019-02-04 11:09:46 +01001448 void OnEncoderConfigurationChanged(
1449 std::vector<VideoStream> streams,
Ilya Nikolaevskiy93be66c2020-04-02 14:10:27 +02001450 bool is_svc,
Rasmus Brandtc402dbe2019-02-04 11:09:46 +01001451 VideoEncoderConfig::ContentType content_type,
1452 int min_transmit_bitrate_bps) override {
Markus Handella3765182020-07-08 13:13:32 +02001453 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001454 ++number_of_reconfigurations_;
1455 min_transmit_bitrate_bps_ = min_transmit_bitrate_bps;
1456 }
1457
Per Kjellanderdcef6412020-10-07 15:09:05 +02001458 void OnBitrateAllocationUpdated(
1459 const VideoBitrateAllocation& allocation) override {
1460 MutexLock lock(&mutex_);
1461 ++number_of_bitrate_allocations_;
1462 last_bitrate_allocation_ = allocation;
1463 }
1464
Per Kjellandera9434842020-10-15 17:53:22 +02001465 void OnVideoLayersAllocationUpdated(
1466 VideoLayersAllocation allocation) override {
1467 MutexLock lock(&mutex_);
1468 ++number_of_layers_allocations_;
1469 last_layers_allocation_ = allocation;
1470 rtc::StringBuilder log;
1471 for (const auto& layer : allocation.active_spatial_layers) {
1472 log << layer.width << "x" << layer.height << "@" << layer.frame_rate_fps
1473 << "[";
1474 for (const auto target_bitrate :
1475 layer.target_bitrate_per_temporal_layer) {
1476 log << target_bitrate.kbps() << ",";
1477 }
1478 log << "]";
1479 }
Harald Alvestrand97597c02021-11-04 12:01:23 +00001480 RTC_DLOG(LS_INFO) << "OnVideoLayersAllocationUpdated " << log.str();
Per Kjellandera9434842020-10-15 17:53:22 +02001481 }
1482
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001483 TimeController* const time_controller_;
Markus Handella3765182020-07-08 13:13:32 +02001484 mutable Mutex mutex_;
perkj26091b12016-09-01 01:17:40 -07001485 TestEncoder* test_encoder_;
1486 rtc::Event encoded_frame_event_;
Sergey Silkin0e42cf72021-03-15 10:12:57 +01001487 EncodedImage last_encoded_image_;
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001488 std::vector<uint8_t> last_encoded_image_data_;
sprangb1ca0732017-02-01 08:38:12 -08001489 uint32_t last_timestamp_ = 0;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001490 int64_t last_capture_time_ms_ = 0;
sprangb1ca0732017-02-01 08:38:12 -08001491 uint32_t last_height_ = 0;
1492 uint32_t last_width_ = 0;
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001493 VideoRotation last_rotation_ = kVideoRotation_0;
Erik Språngd7329ca2019-02-21 21:19:53 +01001494 size_t num_expected_layers_ = 1;
1495 size_t num_received_layers_ = 0;
perkj26091b12016-09-01 01:17:40 -07001496 bool expect_frames_ = true;
Per512ecb32016-09-23 15:52:06 +02001497 int number_of_reconfigurations_ = 0;
1498 int min_transmit_bitrate_bps_ = 0;
Per Kjellanderdcef6412020-10-07 15:09:05 +02001499 VideoBitrateAllocation last_bitrate_allocation_ RTC_GUARDED_BY(&mutex_);
1500 int number_of_bitrate_allocations_ RTC_GUARDED_BY(&mutex_) = 0;
Per Kjellandera9434842020-10-15 17:53:22 +02001501 VideoLayersAllocation last_layers_allocation_ RTC_GUARDED_BY(&mutex_);
1502 int number_of_layers_allocations_ RTC_GUARDED_BY(&mutex_) = 0;
perkj26091b12016-09-01 01:17:40 -07001503 };
1504
Sergey Silkin5ee69672019-07-02 14:18:34 +02001505 class VideoBitrateAllocatorProxyFactory
1506 : public VideoBitrateAllocatorFactory {
1507 public:
1508 VideoBitrateAllocatorProxyFactory()
1509 : bitrate_allocator_factory_(
1510 CreateBuiltinVideoBitrateAllocatorFactory()) {}
1511
1512 std::unique_ptr<VideoBitrateAllocator> CreateVideoBitrateAllocator(
1513 const VideoCodec& codec) override {
Markus Handella3765182020-07-08 13:13:32 +02001514 MutexLock lock(&mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001515 codec_config_ = codec;
1516 return bitrate_allocator_factory_->CreateVideoBitrateAllocator(codec);
1517 }
1518
1519 VideoCodec codec_config() const {
Markus Handella3765182020-07-08 13:13:32 +02001520 MutexLock lock(&mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001521 return codec_config_;
1522 }
1523
1524 private:
1525 std::unique_ptr<VideoBitrateAllocatorFactory> bitrate_allocator_factory_;
1526
Markus Handella3765182020-07-08 13:13:32 +02001527 mutable Mutex mutex_;
1528 VideoCodec codec_config_ RTC_GUARDED_BY(mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001529 };
1530
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001531 Clock* clock() { return time_controller_.GetClock(); }
1532 void AdvanceTime(TimeDelta duration) {
1533 time_controller_.AdvanceTime(duration);
1534 }
1535
1536 int64_t CurrentTimeMs() { return clock()->CurrentTime().ms(); }
1537
1538 protected:
1539 virtual TaskQueueFactory* GetTaskQueueFactory() {
1540 return time_controller_.GetTaskQueueFactory();
1541 }
1542
1543 GlobalSimulatedTimeController time_controller_{Timestamp::Micros(1234)};
perkj26091b12016-09-01 01:17:40 -07001544 VideoSendStream::Config video_send_config_;
Erik Språng08127a92016-11-16 16:41:30 +01001545 VideoEncoderConfig video_encoder_config_;
Per512ecb32016-09-23 15:52:06 +02001546 int codec_width_;
1547 int codec_height_;
sprang4847ae62017-06-27 07:06:52 -07001548 int max_framerate_;
perkj26091b12016-09-01 01:17:40 -07001549 TestEncoder fake_encoder_;
Niels Möllercbcbc222018-09-28 09:07:24 +02001550 test::VideoEncoderProxyFactory encoder_factory_;
Sergey Silkin5ee69672019-07-02 14:18:34 +02001551 VideoBitrateAllocatorProxyFactory bitrate_allocator_factory_;
sprangc5d62e22017-04-02 23:53:04 -07001552 std::unique_ptr<MockableSendStatisticsProxy> stats_proxy_;
perkj26091b12016-09-01 01:17:40 -07001553 TestSink sink_;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001554 AdaptingFrameForwarder video_source_{&time_controller_};
mflodmancc3d4422017-08-03 08:27:51 -07001555 std::unique_ptr<VideoStreamEncoderUnderTest> video_stream_encoder_;
perkj26091b12016-09-01 01:17:40 -07001556};
1557
mflodmancc3d4422017-08-03 08:27:51 -07001558TEST_F(VideoStreamEncoderTest, EncodeOneFrame) {
Henrik Boström381d1092020-05-12 18:49:07 +02001559 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001560 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möllerc572ff32018-11-07 08:43:50 +01001561 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 07:53:41 -07001562 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
sprang4847ae62017-06-27 07:06:52 -07001563 WaitForEncodedFrame(1);
perkja49cbd32016-09-16 07:53:41 -07001564 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
mflodmancc3d4422017-08-03 08:27:51 -07001565 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001566}
1567
mflodmancc3d4422017-08-03 08:27:51 -07001568TEST_F(VideoStreamEncoderTest, DropsFramesBeforeFirstOnBitrateUpdated) {
perkj26091b12016-09-01 01:17:40 -07001569 // Dropped since no target bitrate has been set.
Niels Möllerc572ff32018-11-07 08:43:50 +01001570 rtc::Event frame_destroyed_event;
Sebastian Janssona3177052018-04-10 13:05:49 +02001571 // The encoder will cache up to one frame for a short duration. Adding two
1572 // frames means that the first frame will be dropped and the second frame will
1573 // be sent when the encoder is enabled.
perkja49cbd32016-09-16 07:53:41 -07001574 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001575 AdvanceTime(TimeDelta::Millis(10));
Sebastian Janssona3177052018-04-10 13:05:49 +02001576 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Markus Handell28c71802021-11-08 10:11:55 +01001577 AdvanceTime(TimeDelta::Zero());
perkja49cbd32016-09-16 07:53:41 -07001578 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -07001579
Henrik Boström381d1092020-05-12 18:49:07 +02001580 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001581 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001582
Sebastian Janssona3177052018-04-10 13:05:49 +02001583 // The pending frame should be received.
sprang4847ae62017-06-27 07:06:52 -07001584 WaitForEncodedFrame(2);
Sebastian Janssona3177052018-04-10 13:05:49 +02001585 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
1586
1587 WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 08:27:51 -07001588 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001589}
1590
mflodmancc3d4422017-08-03 08:27:51 -07001591TEST_F(VideoStreamEncoderTest, DropsFramesWhenRateSetToZero) {
Henrik Boström381d1092020-05-12 18:49:07 +02001592 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001593 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkja49cbd32016-09-16 07:53:41 -07001594 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001595 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001596
Henrik Boström381d1092020-05-12 18:49:07 +02001597 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001598 DataRate::Zero(), DataRate::Zero(), DataRate::Zero(), 0, 0, 0);
1599
Sebastian Janssona3177052018-04-10 13:05:49 +02001600 // The encoder will cache up to one frame for a short duration. Adding two
1601 // frames means that the first frame will be dropped and the second frame will
1602 // be sent when the encoder is resumed.
perkja49cbd32016-09-16 07:53:41 -07001603 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Sebastian Janssona3177052018-04-10 13:05:49 +02001604 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
perkj26091b12016-09-01 01:17:40 -07001605
Henrik Boström381d1092020-05-12 18:49:07 +02001606 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001607 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07001608 WaitForEncodedFrame(3);
Sebastian Janssona3177052018-04-10 13:05:49 +02001609 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
1610 WaitForEncodedFrame(4);
mflodmancc3d4422017-08-03 08:27:51 -07001611 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001612}
1613
mflodmancc3d4422017-08-03 08:27:51 -07001614TEST_F(VideoStreamEncoderTest, DropsFramesWithSameOrOldNtpTimestamp) {
Henrik Boström381d1092020-05-12 18:49:07 +02001615 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001616 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkja49cbd32016-09-16 07:53:41 -07001617 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001618 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001619
1620 // This frame will be dropped since it has the same ntp timestamp.
perkja49cbd32016-09-16 07:53:41 -07001621 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
perkj26091b12016-09-01 01:17:40 -07001622
perkja49cbd32016-09-16 07:53:41 -07001623 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001624 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07001625 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001626}
1627
mflodmancc3d4422017-08-03 08:27:51 -07001628TEST_F(VideoStreamEncoderTest, DropsFrameAfterStop) {
Henrik Boström381d1092020-05-12 18:49:07 +02001629 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001630 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001631
perkja49cbd32016-09-16 07:53:41 -07001632 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001633 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001634
mflodmancc3d4422017-08-03 08:27:51 -07001635 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001636 sink_.SetExpectNoFrames();
Niels Möllerc572ff32018-11-07 08:43:50 +01001637 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 07:53:41 -07001638 video_source_.IncomingCapturedFrame(CreateFrame(2, &frame_destroyed_event));
1639 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -07001640}
1641
Markus Handell9a478b52021-11-18 16:07:01 +01001642TEST_F(VideoStreamEncoderTest, DropsPendingFramesOnSlowEncode) {
1643 test::FrameForwarder source;
1644 video_stream_encoder_->SetSource(&source,
1645 DegradationPreference::MAINTAIN_FRAMERATE);
Henrik Boström381d1092020-05-12 18:49:07 +02001646 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001647 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001648
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001649 int dropped_count = 0;
1650 stats_proxy_->SetDroppedFrameCallback(
1651 [&dropped_count](VideoStreamEncoderObserver::DropReason) {
1652 ++dropped_count;
1653 });
1654
Markus Handell9a478b52021-11-18 16:07:01 +01001655 source.IncomingCapturedFrame(CreateFrame(1, nullptr));
1656 source.IncomingCapturedFrame(CreateFrame(2, nullptr));
1657 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07001658 video_stream_encoder_->Stop();
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001659 EXPECT_EQ(1, dropped_count);
perkj26091b12016-09-01 01:17:40 -07001660}
1661
Henrik Boström56db9ff2021-03-24 09:06:45 +01001662TEST_F(VideoStreamEncoderTest, NativeFrameWithoutI420SupportGetsDelivered) {
Henrik Boström381d1092020-05-12 18:49:07 +02001663 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001664 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Noah Richards51db4212019-06-12 06:59:12 -07001665
1666 rtc::Event frame_destroyed_event;
1667 video_source_.IncomingCapturedFrame(
1668 CreateFakeNativeFrame(1, &frame_destroyed_event));
Henrik Boström56db9ff2021-03-24 09:06:45 +01001669 WaitForEncodedFrame(1);
1670 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
1671 fake_encoder_.GetLastInputPixelFormat());
Asa Persson606d3cb2021-10-04 10:07:11 +02001672 EXPECT_EQ(fake_encoder_.config().width, fake_encoder_.GetLastInputWidth());
1673 EXPECT_EQ(fake_encoder_.config().height, fake_encoder_.GetLastInputHeight());
Noah Richards51db4212019-06-12 06:59:12 -07001674 video_stream_encoder_->Stop();
1675}
1676
Henrik Boström56db9ff2021-03-24 09:06:45 +01001677TEST_F(VideoStreamEncoderTest,
1678 NativeFrameWithoutI420SupportGetsCroppedIfNecessary) {
Noah Richards51db4212019-06-12 06:59:12 -07001679 // Use the cropping factory.
1680 video_encoder_config_.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02001681 rtc::make_ref_counted<CroppingVideoStreamFactory>();
Noah Richards51db4212019-06-12 06:59:12 -07001682 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config_),
1683 kMaxPayloadLength);
1684 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
1685
1686 // Capture a frame at codec_width_/codec_height_.
Henrik Boström381d1092020-05-12 18:49:07 +02001687 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001688 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Noah Richards51db4212019-06-12 06:59:12 -07001689 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1690 WaitForEncodedFrame(1);
1691 // The encoder will have been configured once.
1692 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Asa Persson606d3cb2021-10-04 10:07:11 +02001693 EXPECT_EQ(codec_width_, fake_encoder_.config().width);
1694 EXPECT_EQ(codec_height_, fake_encoder_.config().height);
Noah Richards51db4212019-06-12 06:59:12 -07001695
1696 // Now send in a fake frame that needs to be cropped as the width/height
1697 // aren't divisible by 4 (see CreateEncoderStreams above).
1698 rtc::Event frame_destroyed_event;
1699 video_source_.IncomingCapturedFrame(CreateFakeNativeFrame(
1700 2, &frame_destroyed_event, codec_width_ + 1, codec_height_ + 1));
Henrik Boström56db9ff2021-03-24 09:06:45 +01001701 WaitForEncodedFrame(2);
1702 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
1703 fake_encoder_.GetLastInputPixelFormat());
Asa Persson606d3cb2021-10-04 10:07:11 +02001704 EXPECT_EQ(fake_encoder_.config().width, fake_encoder_.GetLastInputWidth());
1705 EXPECT_EQ(fake_encoder_.config().height, fake_encoder_.GetLastInputHeight());
Noah Richards51db4212019-06-12 06:59:12 -07001706 video_stream_encoder_->Stop();
1707}
1708
Evan Shrubsole895556e2020-10-05 09:15:13 +02001709TEST_F(VideoStreamEncoderTest, NonI420FramesShouldNotBeConvertedToI420) {
1710 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001711 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsole895556e2020-10-05 09:15:13 +02001712
1713 video_source_.IncomingCapturedFrame(
1714 CreateNV12Frame(1, codec_width_, codec_height_));
1715 WaitForEncodedFrame(1);
1716 EXPECT_EQ(VideoFrameBuffer::Type::kNV12,
1717 fake_encoder_.GetLastInputPixelFormat());
1718 video_stream_encoder_->Stop();
1719}
1720
Henrik Boström56db9ff2021-03-24 09:06:45 +01001721TEST_F(VideoStreamEncoderTest, NativeFrameGetsDelivered_NoFrameTypePreference) {
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001722 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001723 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001724
1725 fake_encoder_.SetPreferredPixelFormats({});
1726
1727 rtc::Event frame_destroyed_event;
1728 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1729 1, &frame_destroyed_event, codec_width_, codec_height_));
1730 WaitForEncodedFrame(1);
Henrik Boström56db9ff2021-03-24 09:06:45 +01001731 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001732 fake_encoder_.GetLastInputPixelFormat());
1733 video_stream_encoder_->Stop();
1734}
1735
Henrik Boström56db9ff2021-03-24 09:06:45 +01001736TEST_F(VideoStreamEncoderTest,
1737 NativeFrameGetsDelivered_PixelFormatPreferenceMatches) {
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001738 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001739 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001740
1741 fake_encoder_.SetPreferredPixelFormats({VideoFrameBuffer::Type::kNV12});
1742
1743 rtc::Event frame_destroyed_event;
1744 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1745 1, &frame_destroyed_event, codec_width_, codec_height_));
1746 WaitForEncodedFrame(1);
Henrik Boström56db9ff2021-03-24 09:06:45 +01001747 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001748 fake_encoder_.GetLastInputPixelFormat());
1749 video_stream_encoder_->Stop();
1750}
1751
Henrik Boström56db9ff2021-03-24 09:06:45 +01001752TEST_F(VideoStreamEncoderTest, NativeFrameGetsDelivered_MappingIsNotFeasible) {
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001753 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001754 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001755
1756 // Fake NV12 native frame does not allow mapping to I444.
1757 fake_encoder_.SetPreferredPixelFormats({VideoFrameBuffer::Type::kI444});
1758
1759 rtc::Event frame_destroyed_event;
1760 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1761 1, &frame_destroyed_event, codec_width_, codec_height_));
1762 WaitForEncodedFrame(1);
Henrik Boström56db9ff2021-03-24 09:06:45 +01001763 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001764 fake_encoder_.GetLastInputPixelFormat());
1765 video_stream_encoder_->Stop();
1766}
1767
Henrik Boström56db9ff2021-03-24 09:06:45 +01001768TEST_F(VideoStreamEncoderTest, NativeFrameGetsDelivered_BackedByNV12) {
Evan Shrubsole895556e2020-10-05 09:15:13 +02001769 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001770 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsole895556e2020-10-05 09:15:13 +02001771
1772 rtc::Event frame_destroyed_event;
1773 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1774 1, &frame_destroyed_event, codec_width_, codec_height_));
1775 WaitForEncodedFrame(1);
Henrik Boström56db9ff2021-03-24 09:06:45 +01001776 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
Evan Shrubsole895556e2020-10-05 09:15:13 +02001777 fake_encoder_.GetLastInputPixelFormat());
1778 video_stream_encoder_->Stop();
1779}
1780
Ying Wang9b881ab2020-02-07 14:29:32 +01001781TEST_F(VideoStreamEncoderTest, DropsFramesWhenCongestionWindowPushbackSet) {
Henrik Boström381d1092020-05-12 18:49:07 +02001782 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001783 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ying Wang9b881ab2020-02-07 14:29:32 +01001784 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1785 WaitForEncodedFrame(1);
1786
Henrik Boström381d1092020-05-12 18:49:07 +02001787 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001788 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0.5);
Ying Wang9b881ab2020-02-07 14:29:32 +01001789 // The congestion window pushback is set to 0.5, which will drop 1/2 of
1790 // frames. Adding two frames means that the first frame will be dropped and
1791 // the second frame will be sent to the encoder.
1792 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1793 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
1794 WaitForEncodedFrame(3);
1795 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
1796 video_source_.IncomingCapturedFrame(CreateFrame(5, nullptr));
1797 WaitForEncodedFrame(5);
1798 EXPECT_EQ(2u, stats_proxy_->GetStats().frames_dropped_by_congestion_window);
1799 video_stream_encoder_->Stop();
1800}
1801
mflodmancc3d4422017-08-03 08:27:51 -07001802TEST_F(VideoStreamEncoderTest,
1803 ConfigureEncoderTriggersOnEncoderConfigurationChanged) {
Henrik Boström381d1092020-05-12 18:49:07 +02001804 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001805 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per21d45d22016-10-30 21:37:57 +01001806 EXPECT_EQ(0, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 15:52:06 +02001807
1808 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 16:24:55 +02001809 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001810 WaitForEncodedFrame(1);
Per21d45d22016-10-30 21:37:57 +01001811 // The encoder will have been configured once when the first frame is
1812 // received.
1813 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 15:52:06 +02001814
1815 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02001816 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Per512ecb32016-09-23 15:52:06 +02001817 video_encoder_config.min_transmit_bitrate_bps = 9999;
mflodmancc3d4422017-08-03 08:27:51 -07001818 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02001819 kMaxPayloadLength);
Per512ecb32016-09-23 15:52:06 +02001820
1821 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 16:24:55 +02001822 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001823 WaitForEncodedFrame(2);
Per21d45d22016-10-30 21:37:57 +01001824 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkj3b703ed2016-09-29 23:25:40 -07001825 EXPECT_EQ(9999, sink_.last_min_transmit_bitrate());
perkj26105b42016-09-29 22:39:10 -07001826
mflodmancc3d4422017-08-03 08:27:51 -07001827 video_stream_encoder_->Stop();
perkj26105b42016-09-29 22:39:10 -07001828}
1829
mflodmancc3d4422017-08-03 08:27:51 -07001830TEST_F(VideoStreamEncoderTest, FrameResolutionChangeReconfigureEncoder) {
Henrik Boström381d1092020-05-12 18:49:07 +02001831 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001832 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkjfa10b552016-10-02 23:45:26 -07001833
1834 // Capture a frame and wait for it to synchronize with the encoder thread.
1835 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001836 WaitForEncodedFrame(1);
Per21d45d22016-10-30 21:37:57 +01001837 // The encoder will have been configured once.
1838 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Asa Persson606d3cb2021-10-04 10:07:11 +02001839 EXPECT_EQ(codec_width_, fake_encoder_.config().width);
1840 EXPECT_EQ(codec_height_, fake_encoder_.config().height);
perkjfa10b552016-10-02 23:45:26 -07001841
1842 codec_width_ *= 2;
1843 codec_height_ *= 2;
1844 // Capture a frame with a higher resolution and wait for it to synchronize
1845 // with the encoder thread.
1846 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001847 WaitForEncodedFrame(2);
Asa Persson606d3cb2021-10-04 10:07:11 +02001848 EXPECT_EQ(codec_width_, fake_encoder_.config().width);
1849 EXPECT_EQ(codec_height_, fake_encoder_.config().height);
Per21d45d22016-10-30 21:37:57 +01001850 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkjfa10b552016-10-02 23:45:26 -07001851
mflodmancc3d4422017-08-03 08:27:51 -07001852 video_stream_encoder_->Stop();
perkjfa10b552016-10-02 23:45:26 -07001853}
1854
Sergey Silkin443b7ee2019-06-28 12:53:07 +02001855TEST_F(VideoStreamEncoderTest,
1856 EncoderInstanceDestroyedBeforeAnotherInstanceCreated) {
Henrik Boström381d1092020-05-12 18:49:07 +02001857 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001858 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkin443b7ee2019-06-28 12:53:07 +02001859
1860 // Capture a frame and wait for it to synchronize with the encoder thread.
1861 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1862 WaitForEncodedFrame(1);
1863
1864 VideoEncoderConfig video_encoder_config;
1865 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1866 // Changing the max payload data length recreates encoder.
1867 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1868 kMaxPayloadLength / 2);
1869
1870 // Capture a frame and wait for it to synchronize with the encoder thread.
1871 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1872 WaitForEncodedFrame(2);
1873 EXPECT_EQ(1, encoder_factory_.GetMaxNumberOfSimultaneousEncoderInstances());
1874
1875 video_stream_encoder_->Stop();
1876}
1877
Sergey Silkin5ee69672019-07-02 14:18:34 +02001878TEST_F(VideoStreamEncoderTest, BitrateLimitsChangeReconfigureRateAllocator) {
Henrik Boström381d1092020-05-12 18:49:07 +02001879 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001880 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001881
1882 VideoEncoderConfig video_encoder_config;
1883 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02001884 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
1885 video_stream_encoder_->SetStartBitrate(kStartBitrate.bps());
Sergey Silkin5ee69672019-07-02 14:18:34 +02001886 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1887 kMaxPayloadLength);
1888
1889 // Capture a frame and wait for it to synchronize with the encoder thread.
1890 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1891 WaitForEncodedFrame(1);
1892 // The encoder will have been configured once when the first frame is
1893 // received.
1894 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Asa Persson606d3cb2021-10-04 10:07:11 +02001895 EXPECT_EQ(kTargetBitrate.bps(),
Sergey Silkin5ee69672019-07-02 14:18:34 +02001896 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
Asa Persson606d3cb2021-10-04 10:07:11 +02001897 EXPECT_EQ(kStartBitrate.bps(),
Sergey Silkin5ee69672019-07-02 14:18:34 +02001898 bitrate_allocator_factory_.codec_config().startBitrate * 1000);
1899
Sergey Silkin6456e352019-07-08 17:56:40 +02001900 test::FillEncoderConfiguration(kVideoCodecVP8, 1,
1901 &video_encoder_config); //???
Asa Persson606d3cb2021-10-04 10:07:11 +02001902 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps() * 2;
1903 video_stream_encoder_->SetStartBitrate(kStartBitrate.bps() * 2);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001904 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1905 kMaxPayloadLength);
1906
1907 // Capture a frame and wait for it to synchronize with the encoder thread.
1908 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1909 WaitForEncodedFrame(2);
1910 EXPECT_EQ(2, sink_.number_of_reconfigurations());
1911 // Bitrate limits have changed - rate allocator should be reconfigured,
1912 // encoder should not be reconfigured.
Asa Persson606d3cb2021-10-04 10:07:11 +02001913 EXPECT_EQ(kTargetBitrate.bps() * 2,
Sergey Silkin5ee69672019-07-02 14:18:34 +02001914 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
Asa Persson606d3cb2021-10-04 10:07:11 +02001915 EXPECT_EQ(kStartBitrate.bps() * 2,
Sergey Silkin5ee69672019-07-02 14:18:34 +02001916 bitrate_allocator_factory_.codec_config().startBitrate * 1000);
Asa Persson606d3cb2021-10-04 10:07:11 +02001917 EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
Sergey Silkin5ee69672019-07-02 14:18:34 +02001918
1919 video_stream_encoder_->Stop();
1920}
1921
Sergey Silkin6456e352019-07-08 17:56:40 +02001922TEST_F(VideoStreamEncoderTest,
Sergey Silkincd02eba2020-01-20 14:48:40 +01001923 IntersectionOfEncoderAndAppBitrateLimitsUsedWhenBothProvided) {
Henrik Boström381d1092020-05-12 18:49:07 +02001924 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001925 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkin6456e352019-07-08 17:56:40 +02001926
Sergey Silkincd02eba2020-01-20 14:48:40 +01001927 const uint32_t kMinEncBitrateKbps = 100;
1928 const uint32_t kMaxEncBitrateKbps = 1000;
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001929 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
Sergey Silkincd02eba2020-01-20 14:48:40 +01001930 /*frame_size_pixels=*/codec_width_ * codec_height_,
1931 /*min_start_bitrate_bps=*/0,
1932 /*min_bitrate_bps=*/kMinEncBitrateKbps * 1000,
1933 /*max_bitrate_bps=*/kMaxEncBitrateKbps * 1000);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001934 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
1935
Sergey Silkincd02eba2020-01-20 14:48:40 +01001936 VideoEncoderConfig video_encoder_config;
1937 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1938 video_encoder_config.max_bitrate_bps = (kMaxEncBitrateKbps + 1) * 1000;
1939 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1940 (kMinEncBitrateKbps + 1) * 1000;
1941 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1942 kMaxPayloadLength);
1943
1944 // When both encoder and app provide bitrate limits, the intersection of
1945 // provided sets should be used.
1946 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1947 WaitForEncodedFrame(1);
1948 EXPECT_EQ(kMaxEncBitrateKbps,
1949 bitrate_allocator_factory_.codec_config().maxBitrate);
1950 EXPECT_EQ(kMinEncBitrateKbps + 1,
1951 bitrate_allocator_factory_.codec_config().minBitrate);
1952
1953 video_encoder_config.max_bitrate_bps = (kMaxEncBitrateKbps - 1) * 1000;
1954 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1955 (kMinEncBitrateKbps - 1) * 1000;
1956 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1957 kMaxPayloadLength);
1958 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001959 WaitForEncodedFrame(2);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001960 EXPECT_EQ(kMaxEncBitrateKbps - 1,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001961 bitrate_allocator_factory_.codec_config().maxBitrate);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001962 EXPECT_EQ(kMinEncBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001963 bitrate_allocator_factory_.codec_config().minBitrate);
1964
Sergey Silkincd02eba2020-01-20 14:48:40 +01001965 video_stream_encoder_->Stop();
1966}
1967
1968TEST_F(VideoStreamEncoderTest,
1969 EncoderAndAppLimitsDontIntersectEncoderLimitsIgnored) {
Henrik Boström381d1092020-05-12 18:49:07 +02001970 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001971 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001972
1973 const uint32_t kMinAppBitrateKbps = 100;
1974 const uint32_t kMaxAppBitrateKbps = 200;
1975 const uint32_t kMinEncBitrateKbps = kMaxAppBitrateKbps + 1;
1976 const uint32_t kMaxEncBitrateKbps = kMaxAppBitrateKbps * 2;
1977 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
1978 /*frame_size_pixels=*/codec_width_ * codec_height_,
1979 /*min_start_bitrate_bps=*/0,
1980 /*min_bitrate_bps=*/kMinEncBitrateKbps * 1000,
1981 /*max_bitrate_bps=*/kMaxEncBitrateKbps * 1000);
1982 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
1983
1984 VideoEncoderConfig video_encoder_config;
1985 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1986 video_encoder_config.max_bitrate_bps = kMaxAppBitrateKbps * 1000;
1987 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1988 kMinAppBitrateKbps * 1000;
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001989 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1990 kMaxPayloadLength);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001991
Sergey Silkincd02eba2020-01-20 14:48:40 +01001992 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1993 WaitForEncodedFrame(1);
1994 EXPECT_EQ(kMaxAppBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001995 bitrate_allocator_factory_.codec_config().maxBitrate);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001996 EXPECT_EQ(kMinAppBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001997 bitrate_allocator_factory_.codec_config().minBitrate);
Sergey Silkin6456e352019-07-08 17:56:40 +02001998
1999 video_stream_encoder_->Stop();
2000}
2001
2002TEST_F(VideoStreamEncoderTest,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002003 EncoderRecommendedMaxAndMinBitratesUsedForGivenResolution) {
Henrik Boström381d1092020-05-12 18:49:07 +02002004 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002005 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkin6456e352019-07-08 17:56:40 +02002006
2007 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits_270p(
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002008 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002009 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits_360p(
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002010 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002011 fake_encoder_.SetResolutionBitrateLimits(
2012 {encoder_bitrate_limits_270p, encoder_bitrate_limits_360p});
2013
2014 VideoEncoderConfig video_encoder_config;
2015 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
2016 video_encoder_config.max_bitrate_bps = 0;
2017 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
2018 kMaxPayloadLength);
2019
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002020 // 270p. The bitrate limits recommended by encoder for 270p should be used.
Sergey Silkin6456e352019-07-08 17:56:40 +02002021 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
2022 WaitForEncodedFrame(1);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002023 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
2024 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002025 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
2026 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
2027
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002028 // 360p. The bitrate limits recommended by encoder for 360p should be used.
Sergey Silkin6456e352019-07-08 17:56:40 +02002029 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2030 WaitForEncodedFrame(2);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002031 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
2032 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002033 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
2034 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
2035
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002036 // Resolution between 270p and 360p. The bitrate limits recommended by
Sergey Silkin6456e352019-07-08 17:56:40 +02002037 // encoder for 360p should be used.
2038 video_source_.IncomingCapturedFrame(
2039 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
2040 WaitForEncodedFrame(3);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002041 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
2042 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002043 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
2044 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
2045
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002046 // Resolution higher than 360p. The caps recommended by encoder should be
Sergey Silkin6456e352019-07-08 17:56:40 +02002047 // ignored.
2048 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
2049 WaitForEncodedFrame(4);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002050 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
2051 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002052 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
2053 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002054 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
2055 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002056 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
2057 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
2058
2059 // Resolution lower than 270p. The max bitrate limit recommended by encoder
2060 // for 270p should be used.
2061 video_source_.IncomingCapturedFrame(CreateFrame(5, 320, 180));
2062 WaitForEncodedFrame(5);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002063 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
2064 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002065 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
2066 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
2067
2068 video_stream_encoder_->Stop();
2069}
2070
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002071TEST_F(VideoStreamEncoderTest, EncoderRecommendedMaxBitrateCapsTargetBitrate) {
Henrik Boström381d1092020-05-12 18:49:07 +02002072 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002073 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002074
2075 VideoEncoderConfig video_encoder_config;
2076 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
2077 video_encoder_config.max_bitrate_bps = 0;
2078 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
2079 kMaxPayloadLength);
2080
2081 // Encode 720p frame to get the default encoder target bitrate.
2082 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
2083 WaitForEncodedFrame(1);
2084 const uint32_t kDefaultTargetBitrateFor720pKbps =
2085 bitrate_allocator_factory_.codec_config()
2086 .simulcastStream[0]
2087 .targetBitrate;
2088
2089 // Set the max recommended encoder bitrate to something lower than the default
2090 // target bitrate.
2091 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
2092 1280 * 720, 10 * 1000, 10 * 1000,
2093 kDefaultTargetBitrateFor720pKbps / 2 * 1000);
2094 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
2095
2096 // Change resolution to trigger encoder reinitialization.
2097 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2098 WaitForEncodedFrame(2);
2099 video_source_.IncomingCapturedFrame(CreateFrame(3, 1280, 720));
2100 WaitForEncodedFrame(3);
2101
2102 // Ensure the target bitrate is capped by the max bitrate.
2103 EXPECT_EQ(bitrate_allocator_factory_.codec_config().maxBitrate * 1000,
2104 static_cast<uint32_t>(encoder_bitrate_limits.max_bitrate_bps));
2105 EXPECT_EQ(bitrate_allocator_factory_.codec_config()
2106 .simulcastStream[0]
2107 .targetBitrate *
2108 1000,
2109 static_cast<uint32_t>(encoder_bitrate_limits.max_bitrate_bps));
2110
2111 video_stream_encoder_->Stop();
2112}
2113
Åsa Perssona7e34d32021-01-20 15:36:13 +01002114TEST_F(VideoStreamEncoderTest,
2115 EncoderMaxAndMinBitratesUsedForTwoStreamsHighestActive) {
2116 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2117 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2118 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2119 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2120 fake_encoder_.SetResolutionBitrateLimits(
2121 {kEncoderLimits270p, kEncoderLimits360p});
2122
2123 // Two streams, highest stream active.
2124 VideoEncoderConfig config;
2125 const int kNumStreams = 2;
2126 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2127 config.max_bitrate_bps = 0;
2128 config.simulcast_layers[0].active = false;
2129 config.simulcast_layers[1].active = true;
2130 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002131 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssona7e34d32021-01-20 15:36:13 +01002132 "VP8", /*max qp*/ 56, /*screencast*/ false,
2133 /*screenshare enabled*/ false);
2134 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2135
2136 // The encoder bitrate limits for 270p should be used.
2137 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
2138 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02002139 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002140 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002141 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002142 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002143 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002144
2145 // The encoder bitrate limits for 360p should be used.
2146 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2147 EXPECT_FALSE(WaitForFrame(1000));
2148 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002149 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002150 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002151 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002152
2153 // Resolution b/w 270p and 360p. The encoder limits for 360p should be used.
2154 video_source_.IncomingCapturedFrame(
2155 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
2156 EXPECT_FALSE(WaitForFrame(1000));
2157 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002158 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002159 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002160 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002161
2162 // Resolution higher than 360p. Encoder limits should be ignored.
2163 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
2164 EXPECT_FALSE(WaitForFrame(1000));
2165 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002166 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002167 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002168 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002169 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002170 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002171 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002172 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002173
2174 // Resolution lower than 270p. The encoder limits for 270p should be used.
2175 video_source_.IncomingCapturedFrame(CreateFrame(5, 320, 180));
2176 EXPECT_FALSE(WaitForFrame(1000));
2177 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002178 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002179 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002180 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002181
2182 video_stream_encoder_->Stop();
2183}
2184
2185TEST_F(VideoStreamEncoderTest,
Åsa Persson258e9892021-02-25 10:39:51 +01002186 DefaultEncoderMaxAndMinBitratesUsedForTwoStreamsHighestActive) {
2187 // Two streams, highest stream active.
2188 VideoEncoderConfig config;
2189 const int kNumStreams = 2;
2190 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2191 config.max_bitrate_bps = 0;
2192 config.simulcast_layers[0].active = false;
2193 config.simulcast_layers[1].active = true;
2194 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002195 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Persson258e9892021-02-25 10:39:51 +01002196 "VP8", /*max qp*/ 56, /*screencast*/ false,
2197 /*screenshare enabled*/ false);
2198 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2199
2200 // Default bitrate limits for 270p should be used.
2201 const absl::optional<VideoEncoder::ResolutionBitrateLimits>
2202 kDefaultLimits270p =
2203 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
Sergey Silkina86b29b2021-03-05 13:29:19 +01002204 kVideoCodecVP8, 480 * 270);
Åsa Persson258e9892021-02-25 10:39:51 +01002205 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
2206 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02002207 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
Åsa Persson258e9892021-02-25 10:39:51 +01002208 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits270p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002209 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002210 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits270p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002211 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002212
2213 // Default bitrate limits for 360p should be used.
2214 const absl::optional<VideoEncoder::ResolutionBitrateLimits>
2215 kDefaultLimits360p =
2216 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
Sergey Silkina86b29b2021-03-05 13:29:19 +01002217 kVideoCodecVP8, 640 * 360);
Åsa Persson258e9892021-02-25 10:39:51 +01002218 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2219 EXPECT_FALSE(WaitForFrame(1000));
2220 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002221 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002222 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002223 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002224
2225 // Resolution b/w 270p and 360p. The default limits for 360p should be used.
2226 video_source_.IncomingCapturedFrame(
2227 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
2228 EXPECT_FALSE(WaitForFrame(1000));
2229 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002230 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002231 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002232 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002233
2234 // Default bitrate limits for 540p should be used.
2235 const absl::optional<VideoEncoder::ResolutionBitrateLimits>
2236 kDefaultLimits540p =
2237 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
Sergey Silkina86b29b2021-03-05 13:29:19 +01002238 kVideoCodecVP8, 960 * 540);
Åsa Persson258e9892021-02-25 10:39:51 +01002239 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
2240 EXPECT_FALSE(WaitForFrame(1000));
2241 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits540p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002242 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002243 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits540p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002244 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002245
2246 video_stream_encoder_->Stop();
2247}
2248
2249TEST_F(VideoStreamEncoderTest,
Åsa Perssona7e34d32021-01-20 15:36:13 +01002250 EncoderMaxAndMinBitratesUsedForThreeStreamsMiddleActive) {
2251 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2252 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2253 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2254 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2255 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
2256 1280 * 720, 54 * 1000, 31 * 1000, 3456 * 1000);
2257 fake_encoder_.SetResolutionBitrateLimits(
2258 {kEncoderLimits270p, kEncoderLimits360p, kEncoderLimits720p});
2259
2260 // Three streams, middle stream active.
2261 VideoEncoderConfig config;
2262 const int kNumStreams = 3;
2263 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2264 config.simulcast_layers[0].active = false;
2265 config.simulcast_layers[1].active = true;
2266 config.simulcast_layers[2].active = false;
2267 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002268 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssona7e34d32021-01-20 15:36:13 +01002269 "VP8", /*max qp*/ 56, /*screencast*/ false,
2270 /*screenshare enabled*/ false);
2271 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2272
2273 // The encoder bitrate limits for 360p should be used.
2274 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
2275 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02002276 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002277 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002278 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002279 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002280 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002281
2282 // The encoder bitrate limits for 270p should be used.
2283 video_source_.IncomingCapturedFrame(CreateFrame(2, 960, 540));
2284 EXPECT_FALSE(WaitForFrame(1000));
2285 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002286 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002287 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002288 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002289
2290 video_stream_encoder_->Stop();
2291}
2292
2293TEST_F(VideoStreamEncoderTest,
2294 EncoderMaxAndMinBitratesNotUsedForThreeStreamsLowestActive) {
2295 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2296 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2297 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2298 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2299 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
2300 1280 * 720, 54 * 1000, 31 * 1000, 3456 * 1000);
2301 fake_encoder_.SetResolutionBitrateLimits(
2302 {kEncoderLimits270p, kEncoderLimits360p, kEncoderLimits720p});
2303
2304 // Three streams, lowest stream active.
2305 VideoEncoderConfig config;
2306 const int kNumStreams = 3;
2307 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2308 config.simulcast_layers[0].active = true;
2309 config.simulcast_layers[1].active = false;
2310 config.simulcast_layers[2].active = false;
2311 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002312 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssona7e34d32021-01-20 15:36:13 +01002313 "VP8", /*max qp*/ 56, /*screencast*/ false,
2314 /*screenshare enabled*/ false);
2315 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2316
2317 // Resolution on lowest stream lower than 270p. The encoder limits not applied
2318 // on lowest stream, limits for 270p should not be used
2319 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
2320 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02002321 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002322 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002323 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002324 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002325 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002326
2327 video_stream_encoder_->Stop();
2328}
2329
2330TEST_F(VideoStreamEncoderTest,
2331 EncoderMaxBitrateCappedByConfigForTwoStreamsHighestActive) {
2332 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2333 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2334 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2335 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2336 fake_encoder_.SetResolutionBitrateLimits(
2337 {kEncoderLimits270p, kEncoderLimits360p});
2338 const int kMaxBitrateBps = kEncoderLimits360p.max_bitrate_bps - 100 * 1000;
2339
2340 // Two streams, highest stream active.
2341 VideoEncoderConfig config;
2342 const int kNumStreams = 2;
2343 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2344 config.simulcast_layers[0].active = false;
2345 config.simulcast_layers[1].active = true;
2346 config.simulcast_layers[1].max_bitrate_bps = kMaxBitrateBps;
2347 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002348 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssona7e34d32021-01-20 15:36:13 +01002349 "VP8", /*max qp*/ 56, /*screencast*/ false,
2350 /*screenshare enabled*/ false);
2351 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2352
2353 // The encoder bitrate limits for 270p should be used.
2354 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
2355 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02002356 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002357 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002358 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002359 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002360 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002361
2362 // The max configured bitrate is less than the encoder limit for 360p.
2363 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2364 EXPECT_FALSE(WaitForFrame(1000));
2365 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002366 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002367 EXPECT_EQ(static_cast<uint32_t>(kMaxBitrateBps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002368 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002369
2370 video_stream_encoder_->Stop();
2371}
2372
mflodmancc3d4422017-08-03 08:27:51 -07002373TEST_F(VideoStreamEncoderTest, SwitchSourceDeregisterEncoderAsSink) {
perkj803d97f2016-11-01 11:45:46 -07002374 EXPECT_TRUE(video_source_.has_sinks());
2375 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002376 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002377 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 11:45:46 -07002378 EXPECT_FALSE(video_source_.has_sinks());
2379 EXPECT_TRUE(new_video_source.has_sinks());
2380
mflodmancc3d4422017-08-03 08:27:51 -07002381 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002382}
2383
mflodmancc3d4422017-08-03 08:27:51 -07002384TEST_F(VideoStreamEncoderTest, SinkWantsRotationApplied) {
perkj803d97f2016-11-01 11:45:46 -07002385 EXPECT_FALSE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 08:27:51 -07002386 video_stream_encoder_->SetSink(&sink_, true /*rotation_applied*/);
perkj803d97f2016-11-01 11:45:46 -07002387 EXPECT_TRUE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 08:27:51 -07002388 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002389}
2390
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002391class ResolutionAlignmentTest
2392 : public VideoStreamEncoderTest,
2393 public ::testing::WithParamInterface<
2394 ::testing::tuple<int, std::vector<double>>> {
2395 public:
2396 ResolutionAlignmentTest()
2397 : requested_alignment_(::testing::get<0>(GetParam())),
2398 scale_factors_(::testing::get<1>(GetParam())) {}
2399
2400 protected:
2401 const int requested_alignment_;
2402 const std::vector<double> scale_factors_;
2403};
2404
2405INSTANTIATE_TEST_SUITE_P(
2406 AlignmentAndScaleFactors,
2407 ResolutionAlignmentTest,
2408 ::testing::Combine(
2409 ::testing::Values(1, 2, 3, 4, 5, 6, 16, 22), // requested_alignment_
2410 ::testing::Values(std::vector<double>{-1.0}, // scale_factors_
2411 std::vector<double>{-1.0, -1.0},
2412 std::vector<double>{-1.0, -1.0, -1.0},
2413 std::vector<double>{4.0, 2.0, 1.0},
2414 std::vector<double>{9999.0, -1.0, 1.0},
2415 std::vector<double>{3.99, 2.01, 1.0},
2416 std::vector<double>{4.9, 1.7, 1.25},
2417 std::vector<double>{10.0, 4.0, 3.0},
2418 std::vector<double>{1.75, 3.5},
2419 std::vector<double>{1.5, 2.5},
2420 std::vector<double>{1.3, 1.0})));
2421
2422TEST_P(ResolutionAlignmentTest, SinkWantsAlignmentApplied) {
2423 // Set requested resolution alignment.
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002424 video_source_.set_adaptation_enabled(true);
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002425 fake_encoder_.SetRequestedResolutionAlignment(requested_alignment_);
2426 fake_encoder_.SetApplyAlignmentToAllSimulcastLayers(true);
2427
2428 // Fill config with the scaling factor by which to reduce encoding size.
2429 const int num_streams = scale_factors_.size();
2430 VideoEncoderConfig config;
2431 test::FillEncoderConfiguration(kVideoCodecVP8, num_streams, &config);
2432 for (int i = 0; i < num_streams; ++i) {
2433 config.simulcast_layers[i].scale_resolution_down_by = scale_factors_[i];
2434 }
2435 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002436 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002437 "VP8", /*max qp*/ 56, /*screencast*/ false,
2438 /*screenshare enabled*/ false);
2439 video_stream_encoder_->ConfigureEncoder(std::move(config), kMaxPayloadLength);
2440
Henrik Boström381d1092020-05-12 18:49:07 +02002441 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002442 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
2443 0, 0, 0);
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002444 // Wait for all layers before triggering event.
2445 sink_.SetNumExpectedLayers(num_streams);
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002446
2447 // On the 1st frame, we should have initialized the encoder and
2448 // asked for its resolution requirements.
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002449 int64_t timestamp_ms = kFrameIntervalMs;
2450 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
2451 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02002452 EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002453
2454 // On the 2nd frame, we should be receiving a correctly aligned resolution.
2455 // (It's up the to the encoder to potentially drop the previous frame,
2456 // to avoid coding back-to-back keyframes.)
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002457 timestamp_ms += kFrameIntervalMs;
2458 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
2459 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02002460 EXPECT_GE(fake_encoder_.GetNumInitializations(), 1);
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002461
Asa Persson606d3cb2021-10-04 10:07:11 +02002462 VideoCodec codec = fake_encoder_.config();
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002463 EXPECT_EQ(codec.numberOfSimulcastStreams, num_streams);
2464 // Frame size should be a multiple of the requested alignment.
2465 for (int i = 0; i < codec.numberOfSimulcastStreams; ++i) {
2466 EXPECT_EQ(codec.simulcastStream[i].width % requested_alignment_, 0);
2467 EXPECT_EQ(codec.simulcastStream[i].height % requested_alignment_, 0);
2468 // Aspect ratio should match.
2469 EXPECT_EQ(codec.width * codec.simulcastStream[i].height,
2470 codec.height * codec.simulcastStream[i].width);
2471 }
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002472
2473 video_stream_encoder_->Stop();
2474}
2475
Jonathan Yubc771b72017-12-08 17:04:29 -08002476TEST_F(VideoStreamEncoderTest, TestCpuDowngrades_BalancedMode) {
2477 const int kFramerateFps = 30;
asaperssonf7e294d2017-06-13 23:25:22 -07002478 const int kWidth = 1280;
2479 const int kHeight = 720;
Jonathan Yubc771b72017-12-08 17:04:29 -08002480
2481 // We rely on the automatic resolution adaptation, but we handle framerate
2482 // adaptation manually by mocking the stats proxy.
2483 video_source_.set_adaptation_enabled(true);
asaperssonf7e294d2017-06-13 23:25:22 -07002484
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002485 // Enable BALANCED preference, no initial limitation.
Henrik Boström381d1092020-05-12 18:49:07 +02002486 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002487 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002488 video_stream_encoder_->SetSource(&video_source_,
2489 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002490 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
asaperssonf7e294d2017-06-13 23:25:22 -07002491 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002492 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asaperssonf7e294d2017-06-13 23:25:22 -07002493 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2494
Jonathan Yubc771b72017-12-08 17:04:29 -08002495 // Adapt down as far as possible.
2496 rtc::VideoSinkWants last_wants;
2497 int64_t t = 1;
2498 int loop_count = 0;
2499 do {
2500 ++loop_count;
2501 last_wants = video_source_.sink_wants();
2502
2503 // Simulate the framerate we've been asked to adapt to.
2504 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
2505 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
2506 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2507 mock_stats.input_frame_rate = fps;
2508 stats_proxy_->SetMockStats(mock_stats);
2509
2510 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
2511 sink_.WaitForEncodedFrame(t);
2512 t += frame_interval_ms;
2513
mflodmancc3d4422017-08-03 08:27:51 -07002514 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002515 EXPECT_THAT(
Jonathan Yubc771b72017-12-08 17:04:29 -08002516 video_source_.sink_wants(),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002517 FpsInRangeForPixelsInBalanced(*video_source_.last_sent_width() *
2518 *video_source_.last_sent_height()));
Jonathan Yubc771b72017-12-08 17:04:29 -08002519 } while (video_source_.sink_wants().max_pixel_count <
2520 last_wants.max_pixel_count ||
2521 video_source_.sink_wants().max_framerate_fps <
2522 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07002523
Jonathan Yubc771b72017-12-08 17:04:29 -08002524 // Verify that we've adapted all the way down.
2525 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-13 23:25:22 -07002526 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002527 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
2528 EXPECT_EQ(loop_count - 1,
asaperssonf7e294d2017-06-13 23:25:22 -07002529 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Jonathan Yubc771b72017-12-08 17:04:29 -08002530 EXPECT_EQ(kMinPixelsPerFrame, *video_source_.last_sent_width() *
2531 *video_source_.last_sent_height());
2532 EXPECT_EQ(kMinBalancedFramerateFps,
2533 video_source_.sink_wants().max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07002534
Jonathan Yubc771b72017-12-08 17:04:29 -08002535 // Adapt back up the same number of times we adapted down.
2536 for (int i = 0; i < loop_count - 1; ++i) {
2537 last_wants = video_source_.sink_wants();
2538
2539 // Simulate the framerate we've been asked to adapt to.
2540 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
2541 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
2542 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2543 mock_stats.input_frame_rate = fps;
2544 stats_proxy_->SetMockStats(mock_stats);
2545
2546 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
2547 sink_.WaitForEncodedFrame(t);
2548 t += frame_interval_ms;
2549
Henrik Boström91aa7322020-04-28 12:24:33 +02002550 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002551 EXPECT_THAT(
Jonathan Yubc771b72017-12-08 17:04:29 -08002552 video_source_.sink_wants(),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002553 FpsInRangeForPixelsInBalanced(*video_source_.last_sent_width() *
2554 *video_source_.last_sent_height()));
Jonathan Yubc771b72017-12-08 17:04:29 -08002555 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count >
2556 last_wants.max_pixel_count ||
2557 video_source_.sink_wants().max_framerate_fps >
2558 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07002559 }
2560
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002561 EXPECT_THAT(video_source_.sink_wants(), FpsMaxResolutionMax());
Jonathan Yubc771b72017-12-08 17:04:29 -08002562 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-13 23:25:22 -07002563 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002564 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2565 EXPECT_EQ((loop_count - 1) * 2,
2566 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07002567
mflodmancc3d4422017-08-03 08:27:51 -07002568 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07002569}
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002570
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002571TEST_F(VideoStreamEncoderTest,
2572 SinkWantsNotChangedByResourceLimitedBeforeDegradationPreferenceChange) {
Asa Persson606d3cb2021-10-04 10:07:11 +02002573 video_stream_encoder_->OnBitrateUpdated(kTargetBitrate, kTargetBitrate,
2574 kTargetBitrate, 0, 0, 0);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002575 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002576
2577 const int kFrameWidth = 1280;
2578 const int kFrameHeight = 720;
2579
2580 int64_t ntp_time = kFrameIntervalMs;
2581
2582 // Force an input frame rate to be available, or the adaptation call won't
2583 // know what framerate to adapt form.
2584 const int kInputFps = 30;
2585 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2586 stats.input_frame_rate = kInputFps;
2587 stats_proxy_->SetMockStats(stats);
2588
2589 video_source_.set_adaptation_enabled(true);
2590 video_stream_encoder_->SetSource(
2591 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002592 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002593 video_source_.IncomingCapturedFrame(
2594 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2595 sink_.WaitForEncodedFrame(ntp_time);
2596 ntp_time += kFrameIntervalMs;
2597
2598 // Trigger CPU overuse.
2599 video_stream_encoder_->TriggerCpuOveruse();
2600 video_source_.IncomingCapturedFrame(
2601 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2602 sink_.WaitForEncodedFrame(ntp_time);
2603 ntp_time += kFrameIntervalMs;
2604
2605 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2606 EXPECT_EQ(std::numeric_limits<int>::max(),
2607 video_source_.sink_wants().max_pixel_count);
2608 // Some framerate constraint should be set.
2609 int restricted_fps = video_source_.sink_wants().max_framerate_fps;
2610 EXPECT_LT(restricted_fps, kInputFps);
2611 video_source_.IncomingCapturedFrame(
2612 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2613 sink_.WaitForEncodedFrame(ntp_time);
2614 ntp_time += 100;
2615
Henrik Boström2671dac2020-05-19 16:29:09 +02002616 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002617 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
2618 // Give the encoder queue time to process the change in degradation preference
2619 // by waiting for an encoded frame.
2620 video_source_.IncomingCapturedFrame(
2621 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2622 sink_.WaitForEncodedFrame(ntp_time);
2623 ntp_time += kFrameIntervalMs;
2624
2625 video_stream_encoder_->TriggerQualityLow();
2626 video_source_.IncomingCapturedFrame(
2627 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2628 sink_.WaitForEncodedFrame(ntp_time);
2629 ntp_time += kFrameIntervalMs;
2630
2631 // Some resolution constraint should be set.
2632 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2633 EXPECT_LT(video_source_.sink_wants().max_pixel_count,
2634 kFrameWidth * kFrameHeight);
2635 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, kInputFps);
2636
2637 int pixel_count = video_source_.sink_wants().max_pixel_count;
2638 // Triggering a CPU underuse should not change the sink wants since it has
2639 // not been overused for resolution since we changed degradation preference.
2640 video_stream_encoder_->TriggerCpuUnderuse();
2641 video_source_.IncomingCapturedFrame(
2642 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2643 sink_.WaitForEncodedFrame(ntp_time);
2644 ntp_time += kFrameIntervalMs;
2645 EXPECT_EQ(video_source_.sink_wants().max_pixel_count, pixel_count);
2646 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, kInputFps);
2647
Evan Shrubsole64469032020-06-11 10:45:29 +02002648 // Change the degradation preference back. CPU underuse should not adapt since
2649 // QP is most limited.
Henrik Boström2671dac2020-05-19 16:29:09 +02002650 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002651 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
2652 video_source_.IncomingCapturedFrame(
2653 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2654 sink_.WaitForEncodedFrame(ntp_time);
2655 ntp_time += 100;
2656 // Resolution adaptations is gone after changing degradation preference.
2657 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2658 EXPECT_EQ(std::numeric_limits<int>::max(),
2659 video_source_.sink_wants().max_pixel_count);
2660 // The fps adaptation from above is now back.
2661 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, restricted_fps);
2662
2663 // Trigger CPU underuse.
2664 video_stream_encoder_->TriggerCpuUnderuse();
2665 video_source_.IncomingCapturedFrame(
2666 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2667 sink_.WaitForEncodedFrame(ntp_time);
2668 ntp_time += kFrameIntervalMs;
Evan Shrubsole64469032020-06-11 10:45:29 +02002669 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, restricted_fps);
2670
2671 // Trigger QP underuse, fps should return to normal.
2672 video_stream_encoder_->TriggerQualityHigh();
2673 video_source_.IncomingCapturedFrame(
2674 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2675 sink_.WaitForEncodedFrame(ntp_time);
2676 ntp_time += kFrameIntervalMs;
2677 EXPECT_THAT(video_source_.sink_wants(), FpsMax());
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002678
2679 video_stream_encoder_->Stop();
2680}
2681
mflodmancc3d4422017-08-03 08:27:51 -07002682TEST_F(VideoStreamEncoderTest, SinkWantsStoredByDegradationPreference) {
Henrik Boström381d1092020-05-12 18:49:07 +02002683 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002684 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002685 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
perkj803d97f2016-11-01 11:45:46 -07002686
sprangc5d62e22017-04-02 23:53:04 -07002687 const int kFrameWidth = 1280;
2688 const int kFrameHeight = 720;
sprangc5d62e22017-04-02 23:53:04 -07002689
Åsa Persson8c1bf952018-09-13 10:42:19 +02002690 int64_t frame_timestamp = 1;
perkj803d97f2016-11-01 11:45:46 -07002691
kthelgason5e13d412016-12-01 03:59:51 -08002692 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002693 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002694 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002695 frame_timestamp += kFrameIntervalMs;
2696
perkj803d97f2016-11-01 11:45:46 -07002697 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002698 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07002699 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002700 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002701 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002702 frame_timestamp += kFrameIntervalMs;
sprang3ea3c772017-03-30 07:23:48 -07002703
asapersson0944a802017-04-07 00:57:58 -07002704 // Default degradation preference is maintain-framerate, so will lower max
sprangc5d62e22017-04-02 23:53:04 -07002705 // wanted resolution.
2706 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2707 EXPECT_LT(video_source_.sink_wants().max_pixel_count,
2708 kFrameWidth * kFrameHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002709 EXPECT_EQ(kDefaultFramerate, video_source_.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07002710
2711 // Set new source, switch to maintain-resolution.
perkj803d97f2016-11-01 11:45:46 -07002712 test::FrameForwarder new_video_source;
Henrik Boström381d1092020-05-12 18:49:07 +02002713 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002714 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01002715 // Give the encoder queue time to process the change in degradation preference
2716 // by waiting for an encoded frame.
2717 new_video_source.IncomingCapturedFrame(
2718 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2719 sink_.WaitForEncodedFrame(frame_timestamp);
2720 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002721 // Initially no degradation registered.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002722 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
perkj803d97f2016-11-01 11:45:46 -07002723
sprangc5d62e22017-04-02 23:53:04 -07002724 // Force an input frame rate to be available, or the adaptation call won't
2725 // know what framerate to adapt form.
asapersson02465b82017-04-10 01:12:52 -07002726 const int kInputFps = 30;
sprangc5d62e22017-04-02 23:53:04 -07002727 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson02465b82017-04-10 01:12:52 -07002728 stats.input_frame_rate = kInputFps;
sprangc5d62e22017-04-02 23:53:04 -07002729 stats_proxy_->SetMockStats(stats);
2730
mflodmancc3d4422017-08-03 08:27:51 -07002731 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07002732 new_video_source.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002733 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002734 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002735 frame_timestamp += kFrameIntervalMs;
2736
2737 // Some framerate constraint should be set.
sprang84a37592017-02-10 07:04:27 -08002738 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
sprangc5d62e22017-04-02 23:53:04 -07002739 EXPECT_EQ(std::numeric_limits<int>::max(),
2740 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 01:12:52 -07002741 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
sprangc5d62e22017-04-02 23:53:04 -07002742
asapersson02465b82017-04-10 01:12:52 -07002743 // Turn off degradation completely.
Henrik Boström381d1092020-05-12 18:49:07 +02002744 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
2745 &new_video_source, webrtc::DegradationPreference::DISABLED);
Henrik Boström07b17df2020-01-15 11:42:12 +01002746 // Give the encoder queue time to process the change in degradation preference
2747 // by waiting for an encoded frame.
2748 new_video_source.IncomingCapturedFrame(
2749 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2750 sink_.WaitForEncodedFrame(frame_timestamp);
2751 frame_timestamp += kFrameIntervalMs;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002752 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
sprangc5d62e22017-04-02 23:53:04 -07002753
mflodmancc3d4422017-08-03 08:27:51 -07002754 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07002755 new_video_source.IncomingCapturedFrame(
2756 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002757 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002758 frame_timestamp += kFrameIntervalMs;
2759
2760 // Still no degradation.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002761 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
perkj803d97f2016-11-01 11:45:46 -07002762
2763 // Calling SetSource with resolution scaling enabled apply the old SinkWants.
Henrik Boström381d1092020-05-12 18:49:07 +02002764 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002765 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Henrik Boström07b17df2020-01-15 11:42:12 +01002766 // Give the encoder queue time to process the change in degradation preference
2767 // by waiting for an encoded frame.
2768 new_video_source.IncomingCapturedFrame(
2769 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2770 sink_.WaitForEncodedFrame(frame_timestamp);
2771 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002772 EXPECT_LT(new_video_source.sink_wants().max_pixel_count,
2773 kFrameWidth * kFrameHeight);
sprang84a37592017-02-10 07:04:27 -08002774 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002775 EXPECT_EQ(kDefaultFramerate, new_video_source.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07002776
2777 // Calling SetSource with framerate scaling enabled apply the old SinkWants.
Henrik Boström381d1092020-05-12 18:49:07 +02002778 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002779 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01002780 // Give the encoder queue time to process the change in degradation preference
2781 // by waiting for an encoded frame.
2782 new_video_source.IncomingCapturedFrame(
2783 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2784 sink_.WaitForEncodedFrame(frame_timestamp);
2785 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002786 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
2787 EXPECT_EQ(std::numeric_limits<int>::max(),
2788 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 01:12:52 -07002789 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
perkj803d97f2016-11-01 11:45:46 -07002790
mflodmancc3d4422017-08-03 08:27:51 -07002791 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002792}
2793
mflodmancc3d4422017-08-03 08:27:51 -07002794TEST_F(VideoStreamEncoderTest, StatsTracksQualityAdaptationStats) {
Henrik Boström381d1092020-05-12 18:49:07 +02002795 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002796 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkj803d97f2016-11-01 11:45:46 -07002797
asaperssonfab67072017-04-04 05:51:49 -07002798 const int kWidth = 1280;
2799 const int kHeight = 720;
asaperssonfab67072017-04-04 05:51:49 -07002800 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002801 WaitForEncodedFrame(1);
asaperssonfab67072017-04-04 05:51:49 -07002802 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2803 EXPECT_FALSE(stats.bw_limited_resolution);
2804 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
2805
2806 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002807 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07002808 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002809 WaitForEncodedFrame(2);
asaperssonfab67072017-04-04 05:51:49 -07002810
2811 stats = stats_proxy_->GetStats();
2812 EXPECT_TRUE(stats.bw_limited_resolution);
2813 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
2814
2815 // Trigger adapt up.
mflodmancc3d4422017-08-03 08:27:51 -07002816 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 05:51:49 -07002817 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002818 WaitForEncodedFrame(3);
asaperssonfab67072017-04-04 05:51:49 -07002819
2820 stats = stats_proxy_->GetStats();
2821 EXPECT_FALSE(stats.bw_limited_resolution);
2822 EXPECT_EQ(2, stats.number_of_quality_adapt_changes);
2823 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2824
mflodmancc3d4422017-08-03 08:27:51 -07002825 video_stream_encoder_->Stop();
asaperssonfab67072017-04-04 05:51:49 -07002826}
2827
mflodmancc3d4422017-08-03 08:27:51 -07002828TEST_F(VideoStreamEncoderTest, StatsTracksCpuAdaptationStats) {
Henrik Boström381d1092020-05-12 18:49:07 +02002829 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002830 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07002831
2832 const int kWidth = 1280;
2833 const int kHeight = 720;
asaperssonfab67072017-04-04 05:51:49 -07002834 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002835 WaitForEncodedFrame(1);
perkj803d97f2016-11-01 11:45:46 -07002836 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2837 EXPECT_FALSE(stats.cpu_limited_resolution);
2838 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2839
2840 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002841 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07002842 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002843 WaitForEncodedFrame(2);
perkj803d97f2016-11-01 11:45:46 -07002844
2845 stats = stats_proxy_->GetStats();
2846 EXPECT_TRUE(stats.cpu_limited_resolution);
2847 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2848
2849 // Trigger CPU normal use.
Henrik Boström91aa7322020-04-28 12:24:33 +02002850 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07002851 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002852 WaitForEncodedFrame(3);
perkj803d97f2016-11-01 11:45:46 -07002853
2854 stats = stats_proxy_->GetStats();
2855 EXPECT_FALSE(stats.cpu_limited_resolution);
2856 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asaperssonfab67072017-04-04 05:51:49 -07002857 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07002858
mflodmancc3d4422017-08-03 08:27:51 -07002859 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002860}
2861
mflodmancc3d4422017-08-03 08:27:51 -07002862TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsCpuAdaptation) {
Henrik Boström381d1092020-05-12 18:49:07 +02002863 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002864 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08002865
asaperssonfab67072017-04-04 05:51:49 -07002866 const int kWidth = 1280;
2867 const int kHeight = 720;
2868 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002869 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08002870 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002871 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002872 EXPECT_FALSE(stats.cpu_limited_resolution);
2873 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2874
asaperssonfab67072017-04-04 05:51:49 -07002875 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002876 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07002877 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002878 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 01:44:11 -08002879 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002880 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002881 EXPECT_TRUE(stats.cpu_limited_resolution);
2882 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2883
2884 // Set new source with adaptation still enabled.
2885 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002886 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002887 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 01:44:11 -08002888
asaperssonfab67072017-04-04 05:51:49 -07002889 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002890 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08002891 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002892 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002893 EXPECT_TRUE(stats.cpu_limited_resolution);
2894 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2895
2896 // Set adaptation disabled.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002897 video_stream_encoder_->SetSource(&new_video_source,
2898 webrtc::DegradationPreference::DISABLED);
kthelgason876222f2016-11-29 01:44:11 -08002899
asaperssonfab67072017-04-04 05:51:49 -07002900 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002901 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08002902 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002903 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002904 EXPECT_FALSE(stats.cpu_limited_resolution);
2905 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2906
2907 // Set adaptation back to enabled.
mflodmancc3d4422017-08-03 08:27:51 -07002908 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002909 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 01:44:11 -08002910
asaperssonfab67072017-04-04 05:51:49 -07002911 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002912 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 01:44:11 -08002913 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002914 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002915 EXPECT_TRUE(stats.cpu_limited_resolution);
2916 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2917
asaperssonfab67072017-04-04 05:51:49 -07002918 // Trigger CPU normal use.
Henrik Boström91aa7322020-04-28 12:24:33 +02002919 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07002920 new_video_source.IncomingCapturedFrame(CreateFrame(6, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002921 WaitForEncodedFrame(6);
kthelgason876222f2016-11-29 01:44:11 -08002922 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002923 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002924 EXPECT_FALSE(stats.cpu_limited_resolution);
2925 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 01:12:52 -07002926 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002927
mflodmancc3d4422017-08-03 08:27:51 -07002928 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08002929}
2930
mflodmancc3d4422017-08-03 08:27:51 -07002931TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsQualityAdaptation) {
Henrik Boström381d1092020-05-12 18:49:07 +02002932 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002933 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08002934
asaperssonfab67072017-04-04 05:51:49 -07002935 const int kWidth = 1280;
2936 const int kHeight = 720;
2937 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002938 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08002939 VideoSendStream::Stats stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002940 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002941 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002942 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002943
2944 // Set new source with adaptation still enabled.
2945 test::FrameForwarder new_video_source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002946 video_stream_encoder_->SetSource(&new_video_source,
2947 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 01:44:11 -08002948
asaperssonfab67072017-04-04 05:51:49 -07002949 new_video_source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002950 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 01:44:11 -08002951 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002952 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002953 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002954 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002955
asaperssonfab67072017-04-04 05:51:49 -07002956 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002957 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07002958 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002959 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08002960 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002961 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002962 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002963 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002964
asaperssonfab67072017-04-04 05:51:49 -07002965 // Set new source with adaptation still enabled.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002966 video_stream_encoder_->SetSource(&new_video_source,
2967 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 01:44:11 -08002968
asaperssonfab67072017-04-04 05:51:49 -07002969 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002970 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08002971 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002972 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002973 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002974 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002975
asapersson02465b82017-04-10 01:12:52 -07002976 // Disable resolution scaling.
mflodmancc3d4422017-08-03 08:27:51 -07002977 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002978 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 01:44:11 -08002979
asaperssonfab67072017-04-04 05:51:49 -07002980 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002981 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 01:44:11 -08002982 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002983 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002984 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002985 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
2986 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002987
mflodmancc3d4422017-08-03 08:27:51 -07002988 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08002989}
2990
mflodmancc3d4422017-08-03 08:27:51 -07002991TEST_F(VideoStreamEncoderTest,
2992 QualityAdaptationStatsAreResetWhenScalerIsDisabled) {
Henrik Boström381d1092020-05-12 18:49:07 +02002993 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002994 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson36e9eb42017-03-31 05:29:12 -07002995
2996 const int kWidth = 1280;
2997 const int kHeight = 720;
Åsa Persson8c1bf952018-09-13 10:42:19 +02002998 int64_t timestamp_ms = kFrameIntervalMs;
asapersson36e9eb42017-03-31 05:29:12 -07002999 video_source_.set_adaptation_enabled(true);
Åsa Persson8c1bf952018-09-13 10:42:19 +02003000 video_source_.IncomingCapturedFrame(
3001 CreateFrame(timestamp_ms, kWidth, kHeight));
3002 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07003003 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3004 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3005 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3006
3007 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07003008 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003009 timestamp_ms += kFrameIntervalMs;
3010 video_source_.IncomingCapturedFrame(
3011 CreateFrame(timestamp_ms, kWidth, kHeight));
3012 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07003013 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3014 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3015 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3016
3017 // Trigger overuse.
mflodmancc3d4422017-08-03 08:27:51 -07003018 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003019 timestamp_ms += kFrameIntervalMs;
3020 video_source_.IncomingCapturedFrame(
3021 CreateFrame(timestamp_ms, kWidth, kHeight));
3022 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07003023 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3024 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3025 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3026
Niels Möller4db138e2018-04-19 09:04:13 +02003027 // Leave source unchanged, but disable quality scaler.
asapersson36e9eb42017-03-31 05:29:12 -07003028 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 09:04:13 +02003029
3030 VideoEncoderConfig video_encoder_config;
3031 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
3032 // Make format different, to force recreation of encoder.
3033 video_encoder_config.video_format.parameters["foo"] = "foo";
3034 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02003035 kMaxPayloadLength);
Åsa Persson8c1bf952018-09-13 10:42:19 +02003036 timestamp_ms += kFrameIntervalMs;
3037 video_source_.IncomingCapturedFrame(
3038 CreateFrame(timestamp_ms, kWidth, kHeight));
3039 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07003040 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3041 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3042 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3043
mflodmancc3d4422017-08-03 08:27:51 -07003044 video_stream_encoder_->Stop();
asapersson36e9eb42017-03-31 05:29:12 -07003045}
3046
mflodmancc3d4422017-08-03 08:27:51 -07003047TEST_F(VideoStreamEncoderTest,
Evan Shrubsole33be9df2020-03-05 18:39:32 +01003048 StatsTracksCpuAdaptationStatsWhenSwitchingSource_Balanced) {
Henrik Boström381d1092020-05-12 18:49:07 +02003049 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003050 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsole33be9df2020-03-05 18:39:32 +01003051
3052 const int kWidth = 1280;
3053 const int kHeight = 720;
3054 int sequence = 1;
3055
3056 // Enable BALANCED preference, no initial limitation.
3057 test::FrameForwarder source;
3058 video_stream_encoder_->SetSource(&source,
3059 webrtc::DegradationPreference::BALANCED);
3060 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3061 WaitForEncodedFrame(sequence++);
3062 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3063 EXPECT_FALSE(stats.cpu_limited_resolution);
3064 EXPECT_FALSE(stats.cpu_limited_framerate);
3065 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
3066
3067 // Trigger CPU overuse, should now adapt down.
3068 video_stream_encoder_->TriggerCpuOveruse();
3069 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3070 WaitForEncodedFrame(sequence++);
3071 stats = stats_proxy_->GetStats();
3072 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3073
3074 // Set new degradation preference should clear restrictions since we changed
3075 // from BALANCED.
Evan Shrubsolede2049e2020-05-25 16:54:21 +02003076 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole33be9df2020-03-05 18:39:32 +01003077 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
3078 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3079 WaitForEncodedFrame(sequence++);
3080 stats = stats_proxy_->GetStats();
3081 EXPECT_FALSE(stats.cpu_limited_resolution);
3082 EXPECT_FALSE(stats.cpu_limited_framerate);
3083 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3084
3085 // Force an input frame rate to be available, or the adaptation call won't
3086 // know what framerate to adapt from.
3087 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
3088 mock_stats.input_frame_rate = 30;
3089 stats_proxy_->SetMockStats(mock_stats);
3090 video_stream_encoder_->TriggerCpuOveruse();
3091 stats_proxy_->ResetMockStats();
3092 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3093 WaitForEncodedFrame(sequence++);
3094
3095 // We have now adapted once.
3096 stats = stats_proxy_->GetStats();
3097 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3098
3099 // Back to BALANCED, should clear the restrictions again.
Evan Shrubsolede2049e2020-05-25 16:54:21 +02003100 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
3101 &source, webrtc::DegradationPreference::BALANCED);
Evan Shrubsole33be9df2020-03-05 18:39:32 +01003102 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3103 WaitForEncodedFrame(sequence++);
3104 stats = stats_proxy_->GetStats();
3105 EXPECT_FALSE(stats.cpu_limited_resolution);
3106 EXPECT_FALSE(stats.cpu_limited_framerate);
3107 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3108
3109 video_stream_encoder_->Stop();
3110}
3111
3112TEST_F(VideoStreamEncoderTest,
mflodmancc3d4422017-08-03 08:27:51 -07003113 StatsTracksCpuAdaptationStatsWhenSwitchingSource) {
Henrik Boström381d1092020-05-12 18:49:07 +02003114 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003115 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkj803d97f2016-11-01 11:45:46 -07003116
asapersson0944a802017-04-07 00:57:58 -07003117 const int kWidth = 1280;
3118 const int kHeight = 720;
sprang84a37592017-02-10 07:04:27 -08003119 int sequence = 1;
perkj803d97f2016-11-01 11:45:46 -07003120
asaperssonfab67072017-04-04 05:51:49 -07003121 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003122 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003123 VideoSendStream::Stats stats = stats_proxy_->GetStats();
sprang84a37592017-02-10 07:04:27 -08003124 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003125 EXPECT_FALSE(stats.cpu_limited_framerate);
sprang84a37592017-02-10 07:04:27 -08003126 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
3127
asapersson02465b82017-04-10 01:12:52 -07003128 // Trigger CPU overuse, should now adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07003129 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07003130 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003131 WaitForEncodedFrame(sequence++);
sprang84a37592017-02-10 07:04:27 -08003132 stats = stats_proxy_->GetStats();
perkj803d97f2016-11-01 11:45:46 -07003133 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003134 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07003135 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3136
3137 // Set new source with adaptation still enabled.
3138 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07003139 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003140 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 11:45:46 -07003141
3142 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003143 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003144 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003145 stats = stats_proxy_->GetStats();
3146 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson09f05612017-05-15 23:40:18 -07003147 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07003148 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3149
sprangc5d62e22017-04-02 23:53:04 -07003150 // Set cpu adaptation by frame dropping.
mflodmancc3d4422017-08-03 08:27:51 -07003151 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003152 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
perkj803d97f2016-11-01 11:45:46 -07003153 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003154 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003155 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003156 stats = stats_proxy_->GetStats();
sprangc5d62e22017-04-02 23:53:04 -07003157 // Not adapted at first.
perkj803d97f2016-11-01 11:45:46 -07003158 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson09f05612017-05-15 23:40:18 -07003159 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07003160 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3161
sprangc5d62e22017-04-02 23:53:04 -07003162 // Force an input frame rate to be available, or the adaptation call won't
asapersson09f05612017-05-15 23:40:18 -07003163 // know what framerate to adapt from.
sprangc5d62e22017-04-02 23:53:04 -07003164 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
3165 mock_stats.input_frame_rate = 30;
3166 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 08:27:51 -07003167 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07003168 stats_proxy_->ResetMockStats();
3169
3170 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003171 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003172 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003173
3174 // Framerate now adapted.
3175 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07003176 EXPECT_FALSE(stats.cpu_limited_resolution);
3177 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003178 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3179
3180 // Disable CPU adaptation.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003181 video_stream_encoder_->SetSource(&new_video_source,
3182 webrtc::DegradationPreference::DISABLED);
sprangc5d62e22017-04-02 23:53:04 -07003183 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003184 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003185 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003186
3187 stats = stats_proxy_->GetStats();
3188 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003189 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003190 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3191
3192 // Try to trigger overuse. Should not succeed.
3193 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 08:27:51 -07003194 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07003195 stats_proxy_->ResetMockStats();
3196
3197 stats = stats_proxy_->GetStats();
3198 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003199 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003200 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3201
3202 // Switch back the source with resolution adaptation enabled.
mflodmancc3d4422017-08-03 08:27:51 -07003203 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003204 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssonfab67072017-04-04 05:51:49 -07003205 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003206 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003207 stats = stats_proxy_->GetStats();
3208 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003209 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003210 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07003211
3212 // Trigger CPU normal usage.
Henrik Boström91aa7322020-04-28 12:24:33 +02003213 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07003214 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003215 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003216 stats = stats_proxy_->GetStats();
3217 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003218 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003219 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
3220
3221 // Back to the source with adaptation off, set it back to maintain-resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003222 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003223 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07003224 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003225 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003226 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003227 stats = stats_proxy_->GetStats();
asapersson13874762017-06-07 00:01:02 -07003228 // Disabled, since we previously switched the source to disabled.
sprangc5d62e22017-04-02 23:53:04 -07003229 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003230 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003231 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
3232
3233 // Trigger CPU normal usage.
Henrik Boström91aa7322020-04-28 12:24:33 +02003234 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07003235 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003236 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003237 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003238 stats = stats_proxy_->GetStats();
3239 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003240 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003241 EXPECT_EQ(4, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 01:12:52 -07003242 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07003243
mflodmancc3d4422017-08-03 08:27:51 -07003244 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07003245}
3246
mflodmancc3d4422017-08-03 08:27:51 -07003247TEST_F(VideoStreamEncoderTest,
3248 ScalingUpAndDownDoesNothingWithMaintainResolution) {
asaperssonfab67072017-04-04 05:51:49 -07003249 const int kWidth = 1280;
3250 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003251 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003252 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08003253
asaperssonfab67072017-04-04 05:51:49 -07003254 // Expect no scaling to begin with.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003255 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
kthelgason876222f2016-11-29 01:44:11 -08003256
asaperssonfab67072017-04-04 05:51:49 -07003257 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003258 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08003259
asaperssonfab67072017-04-04 05:51:49 -07003260 // Trigger scale down.
mflodmancc3d4422017-08-03 08:27:51 -07003261 video_stream_encoder_->TriggerQualityLow();
kthelgason5e13d412016-12-01 03:59:51 -08003262
asaperssonfab67072017-04-04 05:51:49 -07003263 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003264 WaitForEncodedFrame(2);
kthelgason5e13d412016-12-01 03:59:51 -08003265
kthelgason876222f2016-11-29 01:44:11 -08003266 // Expect a scale down.
3267 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count);
asaperssonfab67072017-04-04 05:51:49 -07003268 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason876222f2016-11-29 01:44:11 -08003269
asapersson02465b82017-04-10 01:12:52 -07003270 // Set resolution scaling disabled.
kthelgason876222f2016-11-29 01:44:11 -08003271 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07003272 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003273 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 01:44:11 -08003274
asaperssonfab67072017-04-04 05:51:49 -07003275 // Trigger scale down.
mflodmancc3d4422017-08-03 08:27:51 -07003276 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07003277 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003278 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08003279
asaperssonfab67072017-04-04 05:51:49 -07003280 // Expect no scaling.
sprangc5d62e22017-04-02 23:53:04 -07003281 EXPECT_EQ(std::numeric_limits<int>::max(),
3282 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 01:44:11 -08003283
asaperssonfab67072017-04-04 05:51:49 -07003284 // Trigger scale up.
mflodmancc3d4422017-08-03 08:27:51 -07003285 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 05:51:49 -07003286 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003287 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08003288
asapersson02465b82017-04-10 01:12:52 -07003289 // Expect nothing to change, still no scaling.
sprangc5d62e22017-04-02 23:53:04 -07003290 EXPECT_EQ(std::numeric_limits<int>::max(),
3291 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 01:44:11 -08003292
mflodmancc3d4422017-08-03 08:27:51 -07003293 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08003294}
3295
mflodmancc3d4422017-08-03 08:27:51 -07003296TEST_F(VideoStreamEncoderTest,
3297 SkipsSameAdaptDownRequest_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07003298 const int kWidth = 1280;
3299 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003300 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003301 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003302
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003303 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07003304 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07003305 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003306 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07003307
3308 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003309 WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003310 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003311 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3312 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3313
3314 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003315 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003316 EXPECT_THAT(source.sink_wants(),
3317 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson02465b82017-04-10 01:12:52 -07003318 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
3319 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3320 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3321
3322 // Trigger adapt down for same input resolution, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003323 video_stream_encoder_->TriggerCpuOveruse();
asapersson02465b82017-04-10 01:12:52 -07003324 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
3325 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3326 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3327
mflodmancc3d4422017-08-03 08:27:51 -07003328 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003329}
3330
mflodmancc3d4422017-08-03 08:27:51 -07003331TEST_F(VideoStreamEncoderTest, SkipsSameOrLargerAdaptDownRequest_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07003332 const int kWidth = 1280;
3333 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003334 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003335 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003336
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003337 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07003338 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003339 video_stream_encoder_->SetSource(&source,
3340 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003341 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3342 sink_.WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003343 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003344
3345 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003346 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003347 EXPECT_THAT(source.sink_wants(),
3348 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07003349 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3350 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3351 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
3352
3353 // Trigger adapt down for same input resolution, expect no change.
3354 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
3355 sink_.WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07003356 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003357 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
3358 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3359 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3360
3361 // Trigger adapt down for larger input resolution, expect no change.
3362 source.IncomingCapturedFrame(CreateFrame(3, kWidth + 1, kHeight + 1));
3363 sink_.WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 08:27:51 -07003364 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003365 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
3366 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3367 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3368
mflodmancc3d4422017-08-03 08:27:51 -07003369 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003370}
3371
mflodmancc3d4422017-08-03 08:27:51 -07003372TEST_F(VideoStreamEncoderTest,
Åsa Perssonf5f7e8e2021-06-09 22:55:38 +02003373 FpsCountReturnsToZeroForFewerAdaptationsUpThanDown) {
3374 const int kWidth = 640;
3375 const int kHeight = 360;
3376 const int64_t kFrameIntervalMs = 150;
3377 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003378 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Åsa Perssonf5f7e8e2021-06-09 22:55:38 +02003379
3380 // Enable BALANCED preference, no initial limitation.
3381 AdaptingFrameForwarder source(&time_controller_);
3382 source.set_adaptation_enabled(true);
3383 video_stream_encoder_->SetSource(&source,
3384 webrtc::DegradationPreference::BALANCED);
3385
3386 int64_t timestamp_ms = kFrameIntervalMs;
3387 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3388 sink_.WaitForEncodedFrame(kWidth, kHeight);
3389 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
3390 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3391 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3392 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3393
3394 // Trigger adapt down, expect reduced fps (640x360@15fps).
3395 video_stream_encoder_->TriggerQualityLow();
3396 timestamp_ms += kFrameIntervalMs;
3397 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3398 sink_.WaitForEncodedFrame(timestamp_ms);
3399 EXPECT_THAT(source.sink_wants(),
3400 FpsMatchesResolutionMax(Lt(kDefaultFramerate)));
3401 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3402 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3403 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3404
3405 // Source requests 270p, expect reduced resolution (480x270@15fps).
3406 source.OnOutputFormatRequest(480, 270);
3407 timestamp_ms += kFrameIntervalMs;
3408 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3409 WaitForEncodedFrame(480, 270);
3410 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3411
3412 // Trigger adapt down, expect reduced fps (480x270@10fps).
3413 video_stream_encoder_->TriggerQualityLow();
3414 timestamp_ms += kFrameIntervalMs;
3415 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3416 sink_.WaitForEncodedFrame(timestamp_ms);
3417 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
3418 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3419 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3420 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3421
3422 // Source requests QVGA, expect reduced resolution (320x180@10fps).
3423 source.OnOutputFormatRequest(320, 180);
3424 timestamp_ms += kFrameIntervalMs;
3425 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3426 WaitForEncodedFrame(320, 180);
3427 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3428
3429 // Trigger adapt down, expect reduced fps (320x180@7fps).
3430 video_stream_encoder_->TriggerQualityLow();
3431 timestamp_ms += kFrameIntervalMs;
3432 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3433 sink_.WaitForEncodedFrame(timestamp_ms);
3434 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
3435 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3436 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3437 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3438
3439 // Source requests VGA, expect increased resolution (640x360@7fps).
3440 source.OnOutputFormatRequest(640, 360);
3441 timestamp_ms += kFrameIntervalMs;
3442 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3443 WaitForEncodedFrame(timestamp_ms);
3444 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3445
3446 // Trigger adapt up, expect increased fps (640x360@(max-2)fps).
3447 video_stream_encoder_->TriggerQualityHigh();
3448 timestamp_ms += kFrameIntervalMs;
3449 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3450 WaitForEncodedFrame(timestamp_ms);
3451 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3452 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3453 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3454 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3455
3456 // Trigger adapt up, expect increased fps (640x360@(max-1)fps).
3457 video_stream_encoder_->TriggerQualityHigh();
3458 timestamp_ms += kFrameIntervalMs;
3459 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3460 WaitForEncodedFrame(timestamp_ms);
3461 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3462 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3463 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3464 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3465
3466 // Trigger adapt up, expect increased fps (640x360@maxfps).
3467 video_stream_encoder_->TriggerQualityHigh();
3468 timestamp_ms += kFrameIntervalMs;
3469 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3470 WaitForEncodedFrame(timestamp_ms);
3471 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3472 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3473 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3474 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3475
3476 video_stream_encoder_->Stop();
3477}
3478
3479TEST_F(VideoStreamEncoderTest,
3480 FpsCountReturnsToZeroForFewerAdaptationsUpThanDownWithTwoResources) {
3481 const int kWidth = 1280;
3482 const int kHeight = 720;
3483 const int64_t kFrameIntervalMs = 150;
3484 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003485 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Åsa Perssonf5f7e8e2021-06-09 22:55:38 +02003486
3487 // Enable BALANCED preference, no initial limitation.
3488 AdaptingFrameForwarder source(&time_controller_);
3489 source.set_adaptation_enabled(true);
3490 video_stream_encoder_->SetSource(&source,
3491 webrtc::DegradationPreference::BALANCED);
3492
3493 int64_t timestamp_ms = kFrameIntervalMs;
3494 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3495 sink_.WaitForEncodedFrame(kWidth, kHeight);
3496 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
3497 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3498 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3499 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3500
3501 // Trigger adapt down, expect scaled down resolution (960x540@maxfps).
3502 video_stream_encoder_->TriggerQualityLow();
3503 timestamp_ms += kFrameIntervalMs;
3504 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3505 sink_.WaitForEncodedFrame(timestamp_ms);
3506 EXPECT_THAT(source.sink_wants(),
3507 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
3508 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3509 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3510 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3511
3512 // Trigger adapt down, expect scaled down resolution (640x360@maxfps).
3513 video_stream_encoder_->TriggerQualityLow();
3514 timestamp_ms += kFrameIntervalMs;
3515 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3516 sink_.WaitForEncodedFrame(timestamp_ms);
3517 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
3518 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3519 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3520 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3521
3522 // Trigger adapt down, expect reduced fps (640x360@15fps).
3523 video_stream_encoder_->TriggerQualityLow();
3524 timestamp_ms += kFrameIntervalMs;
3525 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3526 WaitForEncodedFrame(timestamp_ms);
3527 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
3528 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3529 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3530 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3531
3532 // Source requests QVGA, expect reduced resolution (320x180@15fps).
3533 source.OnOutputFormatRequest(320, 180);
3534 timestamp_ms += kFrameIntervalMs;
3535 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3536 WaitForEncodedFrame(320, 180);
3537 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3538 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3539
3540 // Trigger adapt down, expect reduced fps (320x180@7fps).
3541 video_stream_encoder_->TriggerCpuOveruse();
3542 timestamp_ms += kFrameIntervalMs;
3543 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3544 WaitForEncodedFrame(timestamp_ms);
3545 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
3546 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3547 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3548 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3549 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
3550 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3551 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3552
3553 // Source requests HD, expect increased resolution (640x360@7fps).
3554 source.OnOutputFormatRequest(1280, 720);
3555 timestamp_ms += kFrameIntervalMs;
3556 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3557 WaitForEncodedFrame(timestamp_ms);
3558 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3559 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3560
3561 // Trigger adapt up, expect increased fps (640x360@(max-1)fps).
3562 video_stream_encoder_->TriggerCpuUnderuse();
3563 timestamp_ms += kFrameIntervalMs;
3564 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3565 WaitForEncodedFrame(timestamp_ms);
3566 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3567 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3568 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3569 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3570 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
3571 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3572 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3573
3574 // Trigger adapt up, expect increased fps (640x360@maxfps).
3575 video_stream_encoder_->TriggerQualityHigh();
3576 video_stream_encoder_->TriggerCpuUnderuse();
3577 timestamp_ms += kFrameIntervalMs;
3578 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3579 WaitForEncodedFrame(timestamp_ms);
3580 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3581 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3582 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3583 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3584 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3585 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3586 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3587
3588 // Trigger adapt up, expect increased resolution (960x570@maxfps).
3589 video_stream_encoder_->TriggerQualityHigh();
3590 video_stream_encoder_->TriggerCpuUnderuse();
3591 timestamp_ms += kFrameIntervalMs;
3592 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3593 WaitForEncodedFrame(timestamp_ms);
3594 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
3595 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3596 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3597 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3598 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3599 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3600 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3601
3602 // Trigger adapt up, expect increased resolution (1280x720@maxfps).
3603 video_stream_encoder_->TriggerQualityHigh();
3604 video_stream_encoder_->TriggerCpuUnderuse();
3605 timestamp_ms += kFrameIntervalMs;
3606 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3607 WaitForEncodedFrame(timestamp_ms);
3608 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
3609 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3610 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3611 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3612 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3613 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3614 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3615
3616 video_stream_encoder_->Stop();
3617}
3618
3619TEST_F(VideoStreamEncoderTest,
mflodmancc3d4422017-08-03 08:27:51 -07003620 NoChangeForInitialNormalUsage_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07003621 const int kWidth = 1280;
3622 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003623 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003624 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003625
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003626 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07003627 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07003628 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003629 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07003630
3631 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003632 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003633 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003634 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3635 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3636
3637 // Trigger adapt up, expect no change.
Henrik Boström91aa7322020-04-28 12:24:33 +02003638 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003639 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003640 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3641 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3642
mflodmancc3d4422017-08-03 08:27:51 -07003643 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003644}
3645
mflodmancc3d4422017-08-03 08:27:51 -07003646TEST_F(VideoStreamEncoderTest,
3647 NoChangeForInitialNormalUsage_MaintainResolutionMode) {
asapersson02465b82017-04-10 01:12:52 -07003648 const int kWidth = 1280;
3649 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003650 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003651 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003652
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003653 // Enable MAINTAIN_RESOLUTION preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07003654 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07003655 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003656 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
asapersson02465b82017-04-10 01:12:52 -07003657
3658 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003659 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003660 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003661 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 01:12:52 -07003662 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3663
3664 // Trigger adapt up, expect no change.
Henrik Boström91aa7322020-04-28 12:24:33 +02003665 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003666 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003667 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 01:12:52 -07003668 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3669
mflodmancc3d4422017-08-03 08:27:51 -07003670 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003671}
3672
mflodmancc3d4422017-08-03 08:27:51 -07003673TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07003674 const int kWidth = 1280;
3675 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003676 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003677 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003678
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003679 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07003680 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003681 video_stream_encoder_->SetSource(&source,
3682 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003683
3684 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3685 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003686 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003687 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3688 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3689 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3690
3691 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003692 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003693 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003694 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3695 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3696 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3697
mflodmancc3d4422017-08-03 08:27:51 -07003698 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003699}
3700
mflodmancc3d4422017-08-03 08:27:51 -07003701TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_DisabledMode) {
asapersson09f05612017-05-15 23:40:18 -07003702 const int kWidth = 1280;
3703 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003704 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003705 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07003706
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003707 // Enable DISABLED preference, no initial limitation.
asapersson09f05612017-05-15 23:40:18 -07003708 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003709 video_stream_encoder_->SetSource(&source,
3710 webrtc::DegradationPreference::DISABLED);
asapersson09f05612017-05-15 23:40:18 -07003711
3712 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3713 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003714 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003715 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3716 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3717 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3718
3719 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003720 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003721 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003722 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3723 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3724 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3725
mflodmancc3d4422017-08-03 08:27:51 -07003726 video_stream_encoder_->Stop();
asapersson09f05612017-05-15 23:40:18 -07003727}
3728
mflodmancc3d4422017-08-03 08:27:51 -07003729TEST_F(VideoStreamEncoderTest,
3730 AdaptsResolutionForLowQuality_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07003731 const int kWidth = 1280;
3732 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003733 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003734 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003735
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003736 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003737 AdaptingFrameForwarder source(&time_controller_);
asapersson02465b82017-04-10 01:12:52 -07003738 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07003739 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003740 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07003741
3742 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003743 WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003744 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003745 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3746 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3747
3748 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003749 video_stream_encoder_->TriggerQualityLow();
asapersson02465b82017-04-10 01:12:52 -07003750 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003751 WaitForEncodedFrame(2);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003752 EXPECT_THAT(source.sink_wants(),
3753 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson02465b82017-04-10 01:12:52 -07003754 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3755 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3756
3757 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003758 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003759 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003760 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3761 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3762 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3763
mflodmancc3d4422017-08-03 08:27:51 -07003764 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003765}
3766
mflodmancc3d4422017-08-03 08:27:51 -07003767TEST_F(VideoStreamEncoderTest,
3768 AdaptsFramerateForLowQuality_MaintainResolutionMode) {
asapersson09f05612017-05-15 23:40:18 -07003769 const int kWidth = 1280;
3770 const int kHeight = 720;
3771 const int kInputFps = 30;
Henrik Boström381d1092020-05-12 18:49:07 +02003772 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003773 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07003774
3775 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3776 stats.input_frame_rate = kInputFps;
3777 stats_proxy_->SetMockStats(stats);
3778
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003779 // Expect no scaling to begin with (preference: MAINTAIN_FRAMERATE).
asapersson09f05612017-05-15 23:40:18 -07003780 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3781 sink_.WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003782 EXPECT_THAT(video_source_.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003783
3784 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003785 video_stream_encoder_->TriggerQualityLow();
asapersson09f05612017-05-15 23:40:18 -07003786 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
3787 sink_.WaitForEncodedFrame(2);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003788 EXPECT_THAT(video_source_.sink_wants(),
3789 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson09f05612017-05-15 23:40:18 -07003790
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003791 // Enable MAINTAIN_RESOLUTION preference.
asapersson09f05612017-05-15 23:40:18 -07003792 test::FrameForwarder new_video_source;
Henrik Boström2671dac2020-05-19 16:29:09 +02003793 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003794 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01003795 // Give the encoder queue time to process the change in degradation preference
3796 // by waiting for an encoded frame.
3797 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
3798 sink_.WaitForEncodedFrame(3);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003799 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003800
3801 // Trigger adapt down, expect reduced framerate.
mflodmancc3d4422017-08-03 08:27:51 -07003802 video_stream_encoder_->TriggerQualityLow();
Henrik Boström07b17df2020-01-15 11:42:12 +01003803 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
3804 sink_.WaitForEncodedFrame(4);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003805 EXPECT_THAT(new_video_source.sink_wants(),
3806 FpsMatchesResolutionMax(Lt(kInputFps)));
asapersson09f05612017-05-15 23:40:18 -07003807
3808 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003809 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003810 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003811
mflodmancc3d4422017-08-03 08:27:51 -07003812 video_stream_encoder_->Stop();
asapersson09f05612017-05-15 23:40:18 -07003813}
3814
mflodmancc3d4422017-08-03 08:27:51 -07003815TEST_F(VideoStreamEncoderTest, DoesNotScaleBelowSetResolutionLimit) {
asaperssond0de2952017-04-21 01:47:31 -07003816 const int kWidth = 1280;
3817 const int kHeight = 720;
3818 const size_t kNumFrames = 10;
3819
Henrik Boström381d1092020-05-12 18:49:07 +02003820 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003821 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
kthelgason5e13d412016-12-01 03:59:51 -08003822
asaperssond0de2952017-04-21 01:47:31 -07003823 // Enable adapter, expected input resolutions when downscaling:
asapersson142fcc92017-08-17 08:58:54 -07003824 // 1280x720 -> 960x540 -> 640x360 -> 480x270 -> 320x180 (kMinPixelsPerFrame)
asaperssond0de2952017-04-21 01:47:31 -07003825 video_source_.set_adaptation_enabled(true);
3826
3827 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3828 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3829
3830 int downscales = 0;
3831 for (size_t i = 1; i <= kNumFrames; i++) {
Åsa Persson8c1bf952018-09-13 10:42:19 +02003832 video_source_.IncomingCapturedFrame(
3833 CreateFrame(i * kFrameIntervalMs, kWidth, kHeight));
3834 WaitForEncodedFrame(i * kFrameIntervalMs);
asaperssond0de2952017-04-21 01:47:31 -07003835
asaperssonfab67072017-04-04 05:51:49 -07003836 // Trigger scale down.
asaperssond0de2952017-04-21 01:47:31 -07003837 rtc::VideoSinkWants last_wants = video_source_.sink_wants();
mflodmancc3d4422017-08-03 08:27:51 -07003838 video_stream_encoder_->TriggerQualityLow();
sprangc5d62e22017-04-02 23:53:04 -07003839 EXPECT_GE(video_source_.sink_wants().max_pixel_count, kMinPixelsPerFrame);
asaperssond0de2952017-04-21 01:47:31 -07003840
3841 if (video_source_.sink_wants().max_pixel_count < last_wants.max_pixel_count)
3842 ++downscales;
3843
3844 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3845 EXPECT_EQ(downscales,
3846 stats_proxy_->GetStats().number_of_quality_adapt_changes);
3847 EXPECT_GT(downscales, 0);
kthelgason5e13d412016-12-01 03:59:51 -08003848 }
mflodmancc3d4422017-08-03 08:27:51 -07003849 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07003850}
3851
mflodmancc3d4422017-08-03 08:27:51 -07003852TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07003853 AdaptsResolutionUpAndDownTwiceOnOveruse_MaintainFramerateMode) {
3854 const int kWidth = 1280;
3855 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003856 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003857 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07003858
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003859 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003860 AdaptingFrameForwarder source(&time_controller_);
asaperssond0de2952017-04-21 01:47:31 -07003861 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07003862 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003863 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07003864
Åsa Persson8c1bf952018-09-13 10:42:19 +02003865 int64_t timestamp_ms = kFrameIntervalMs;
3866 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003867 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003868 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003869 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3870 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3871
3872 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003873 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003874 timestamp_ms += kFrameIntervalMs;
3875 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3876 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003877 EXPECT_THAT(source.sink_wants(),
3878 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07003879 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3880 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3881
3882 // Trigger adapt up, expect no restriction.
Henrik Boström91aa7322020-04-28 12:24:33 +02003883 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003884 timestamp_ms += kFrameIntervalMs;
3885 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003886 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003887 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003888 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3889 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3890
3891 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003892 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003893 timestamp_ms += kFrameIntervalMs;
3894 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3895 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003896 EXPECT_THAT(source.sink_wants(),
3897 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07003898 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3899 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3900
3901 // Trigger adapt up, expect no restriction.
Henrik Boström91aa7322020-04-28 12:24:33 +02003902 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003903 timestamp_ms += kFrameIntervalMs;
3904 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asapersson09f05612017-05-15 23:40:18 -07003905 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003906 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003907 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3908 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3909
mflodmancc3d4422017-08-03 08:27:51 -07003910 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07003911}
3912
mflodmancc3d4422017-08-03 08:27:51 -07003913TEST_F(VideoStreamEncoderTest,
asaperssonf7e294d2017-06-13 23:25:22 -07003914 AdaptsResolutionUpAndDownTwiceForLowQuality_BalancedMode_NoFpsLimit) {
3915 const int kWidth = 1280;
3916 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003917 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003918 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003919
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003920 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003921 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07003922 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003923 video_stream_encoder_->SetSource(&source,
3924 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003925
Åsa Persson8c1bf952018-09-13 10:42:19 +02003926 int64_t timestamp_ms = kFrameIntervalMs;
3927 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07003928 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003929 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003930 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3931 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3932
3933 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003934 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003935 timestamp_ms += kFrameIntervalMs;
3936 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3937 sink_.WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003938 EXPECT_THAT(source.sink_wants(),
3939 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07003940 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3941 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3942
3943 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003944 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003945 timestamp_ms += kFrameIntervalMs;
3946 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07003947 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003948 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003949 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3950 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3951
3952 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003953 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003954 timestamp_ms += kFrameIntervalMs;
3955 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3956 sink_.WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003957 EXPECT_THAT(source.sink_wants(),
3958 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07003959 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3960 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3961
3962 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003963 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003964 timestamp_ms += kFrameIntervalMs;
3965 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07003966 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003967 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003968 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3969 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3970
mflodmancc3d4422017-08-03 08:27:51 -07003971 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003972}
3973
Sergey Silkin41c650b2019-10-14 13:12:19 +02003974TEST_F(VideoStreamEncoderTest, AdaptUpIfBwEstimateIsHigherThanMinBitrate) {
3975 fake_encoder_.SetResolutionBitrateLimits(
3976 {kEncoderBitrateLimits540p, kEncoderBitrateLimits720p});
3977
Henrik Boström381d1092020-05-12 18:49:07 +02003978 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003979 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
3980 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
3981 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps), 0,
3982 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003983
3984 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003985 AdaptingFrameForwarder source(&time_controller_);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003986 source.set_adaptation_enabled(true);
3987 video_stream_encoder_->SetSource(
3988 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
3989
3990 // Insert 720p frame.
3991 int64_t timestamp_ms = kFrameIntervalMs;
3992 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
3993 WaitForEncodedFrame(1280, 720);
3994
3995 // Reduce bitrate and trigger adapt down.
Henrik Boström381d1092020-05-12 18:49:07 +02003996 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003997 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
3998 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
3999 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps), 0,
4000 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004001 video_stream_encoder_->TriggerQualityLow();
4002
4003 // Insert 720p frame. It should be downscaled and encoded.
4004 timestamp_ms += kFrameIntervalMs;
4005 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
4006 WaitForEncodedFrame(960, 540);
4007
4008 // Trigger adapt up. Higher resolution should not be requested duo to lack
4009 // of bitrate.
4010 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004011 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMatches(Lt(1280 * 720)));
Sergey Silkin41c650b2019-10-14 13:12:19 +02004012
4013 // Increase bitrate.
Henrik Boström381d1092020-05-12 18:49:07 +02004014 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004015 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
4016 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
4017 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps), 0,
4018 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004019
4020 // Trigger adapt up. Higher resolution should be requested.
4021 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004022 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
Sergey Silkin41c650b2019-10-14 13:12:19 +02004023
4024 video_stream_encoder_->Stop();
4025}
4026
4027TEST_F(VideoStreamEncoderTest, DropFirstFramesIfBwEstimateIsTooLow) {
4028 fake_encoder_.SetResolutionBitrateLimits(
4029 {kEncoderBitrateLimits540p, kEncoderBitrateLimits720p});
4030
4031 // Set bitrate equal to min bitrate of 540p.
Henrik Boström381d1092020-05-12 18:49:07 +02004032 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004033 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
4034 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
4035 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps), 0,
4036 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004037
4038 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004039 AdaptingFrameForwarder source(&time_controller_);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004040 source.set_adaptation_enabled(true);
4041 video_stream_encoder_->SetSource(
4042 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
4043
4044 // Insert 720p frame. It should be dropped and lower resolution should be
4045 // requested.
4046 int64_t timestamp_ms = kFrameIntervalMs;
4047 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
4048 ExpectDroppedFrame();
Henrik Boström2671dac2020-05-19 16:29:09 +02004049 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < 1280 * 720, 5000);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004050
4051 // Insert 720p frame. It should be downscaled and encoded.
4052 timestamp_ms += kFrameIntervalMs;
4053 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
4054 WaitForEncodedFrame(960, 540);
4055
4056 video_stream_encoder_->Stop();
4057}
4058
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004059class BalancedDegradationTest : public VideoStreamEncoderTest {
4060 protected:
4061 void SetupTest() {
4062 // Reset encoder for field trials to take effect.
4063 ConfigureEncoder(video_encoder_config_.Copy());
Asa Persson606d3cb2021-10-04 10:07:11 +02004064 OnBitrateUpdated(kTargetBitrate);
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004065
4066 // Enable BALANCED preference.
4067 source_.set_adaptation_enabled(true);
Åsa Perssonccfb3402019-09-25 15:13:04 +02004068 video_stream_encoder_->SetSource(&source_, DegradationPreference::BALANCED);
4069 }
4070
Asa Persson606d3cb2021-10-04 10:07:11 +02004071 void OnBitrateUpdated(DataRate bitrate) {
Henrik Boström381d1092020-05-12 18:49:07 +02004072 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004073 bitrate, bitrate, bitrate, 0, 0, 0);
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004074 }
4075
Åsa Persson45b176f2019-09-30 11:19:05 +02004076 void InsertFrame() {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004077 timestamp_ms_ += kFrameIntervalMs;
4078 source_.IncomingCapturedFrame(CreateFrame(timestamp_ms_, kWidth, kHeight));
Åsa Persson45b176f2019-09-30 11:19:05 +02004079 }
4080
4081 void InsertFrameAndWaitForEncoded() {
4082 InsertFrame();
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004083 sink_.WaitForEncodedFrame(timestamp_ms_);
4084 }
4085
4086 const int kWidth = 640; // pixels:640x360=230400
4087 const int kHeight = 360;
4088 const int64_t kFrameIntervalMs = 150; // Use low fps to not drop any frame.
4089 int64_t timestamp_ms_ = 0;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004090 AdaptingFrameForwarder source_{&time_controller_};
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004091};
4092
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02004093TEST_F(BalancedDegradationTest, AdaptDownTwiceIfMinFpsDiffLtThreshold) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004094 test::ScopedFieldTrials field_trials(
4095 "WebRTC-Video-BalancedDegradationSettings/"
4096 "pixels:57600|129600|230400,fps:7|10|24,fps_diff:1|1|1/");
4097 SetupTest();
4098
4099 // Force input frame rate.
4100 const int kInputFps = 24;
4101 VideoSendStream::Stats stats = stats_proxy_->GetStats();
4102 stats.input_frame_rate = kInputFps;
4103 stats_proxy_->SetMockStats(stats);
4104
Åsa Persson45b176f2019-09-30 11:19:05 +02004105 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004106 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004107
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02004108 // Trigger adapt down, expect scaled down framerate and resolution,
4109 // since Fps diff (input-requested:0) < threshold.
4110 video_stream_encoder_->TriggerQualityLow();
4111 EXPECT_THAT(source_.sink_wants(),
4112 AllOf(WantsFps(Eq(24)), WantsMaxPixels(Le(230400))));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004113
4114 video_stream_encoder_->Stop();
4115}
4116
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02004117TEST_F(BalancedDegradationTest, AdaptDownOnceIfFpsDiffGeThreshold) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004118 test::ScopedFieldTrials field_trials(
4119 "WebRTC-Video-BalancedDegradationSettings/"
4120 "pixels:57600|129600|230400,fps:7|10|24,fps_diff:1|1|1/");
4121 SetupTest();
4122
4123 // Force input frame rate.
4124 const int kInputFps = 25;
4125 VideoSendStream::Stats stats = stats_proxy_->GetStats();
4126 stats.input_frame_rate = kInputFps;
4127 stats_proxy_->SetMockStats(stats);
4128
Åsa Persson45b176f2019-09-30 11:19:05 +02004129 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004130 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004131
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02004132 // Trigger adapt down, expect scaled down framerate only (640x360@24fps).
4133 // Fps diff (input-requested:1) == threshold.
4134 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004135 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(24)));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004136
4137 video_stream_encoder_->Stop();
4138}
4139
4140TEST_F(BalancedDegradationTest, AdaptDownUsesCodecSpecificFps) {
4141 test::ScopedFieldTrials field_trials(
4142 "WebRTC-Video-BalancedDegradationSettings/"
4143 "pixels:57600|129600|230400,fps:7|10|24,vp8_fps:8|11|22/");
4144 SetupTest();
4145
4146 EXPECT_EQ(kVideoCodecVP8, video_encoder_config_.codec_type);
4147
Åsa Persson45b176f2019-09-30 11:19:05 +02004148 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004149 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004150
4151 // Trigger adapt down, expect scaled down framerate (640x360@22fps).
4152 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004153 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(22)));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004154
4155 video_stream_encoder_->Stop();
4156}
4157
Åsa Perssonccfb3402019-09-25 15:13:04 +02004158TEST_F(BalancedDegradationTest, NoAdaptUpIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004159 test::ScopedFieldTrials field_trials(
Åsa Persson1b247f12019-08-14 17:26:39 +02004160 "WebRTC-Video-BalancedDegradationSettings/"
4161 "pixels:57600|129600|230400,fps:7|10|14,kbps:0|0|425/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02004162 SetupTest();
Åsa Persson1b247f12019-08-14 17:26:39 +02004163
Asa Persson606d3cb2021-10-04 10:07:11 +02004164 const DataRate kMinBitrate = DataRate::KilobitsPerSec(425);
4165 const DataRate kTooLowMinBitrate = DataRate::KilobitsPerSec(424);
4166 OnBitrateUpdated(kTooLowMinBitrate);
Åsa Persson1b247f12019-08-14 17:26:39 +02004167
Åsa Persson45b176f2019-09-30 11:19:05 +02004168 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004169 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson1b247f12019-08-14 17:26:39 +02004170 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4171
4172 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
4173 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004174 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004175 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson1b247f12019-08-14 17:26:39 +02004176 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4177
4178 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
4179 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004180 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004181 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson1b247f12019-08-14 17:26:39 +02004182 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4183
Åsa Persson30ab0152019-08-27 12:22:33 +02004184 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
4185 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004186 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004187 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Perssonccfb3402019-09-25 15:13:04 +02004188 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 10);
Åsa Persson30ab0152019-08-27 12:22:33 +02004189 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4190
4191 // Trigger adapt up, expect no upscale in fps (target bitrate < min bitrate).
Åsa Persson1b247f12019-08-14 17:26:39 +02004192 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004193 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02004194 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
Åsa Persson1b247f12019-08-14 17:26:39 +02004195
Åsa Persson30ab0152019-08-27 12:22:33 +02004196 // Trigger adapt up, expect upscaled fps (target bitrate == min bitrate).
Asa Persson606d3cb2021-10-04 10:07:11 +02004197 OnBitrateUpdated(kMinBitrate);
Åsa Persson1b247f12019-08-14 17:26:39 +02004198 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004199 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02004200 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 14);
Åsa Persson30ab0152019-08-27 12:22:33 +02004201 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4202
4203 video_stream_encoder_->Stop();
4204}
4205
Åsa Perssonccfb3402019-09-25 15:13:04 +02004206TEST_F(BalancedDegradationTest,
Åsa Persson45b176f2019-09-30 11:19:05 +02004207 InitialFrameDropAdaptsFpsAndResolutionInOneStep) {
4208 test::ScopedFieldTrials field_trials(
4209 "WebRTC-Video-BalancedDegradationSettings/"
4210 "pixels:57600|129600|230400,fps:7|24|24/");
4211 SetupTest();
Asa Persson606d3cb2021-10-04 10:07:11 +02004212 OnBitrateUpdated(kLowTargetBitrate);
Åsa Persson45b176f2019-09-30 11:19:05 +02004213
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004214 EXPECT_THAT(source_.sink_wants(), UnlimitedSinkWants());
Åsa Persson45b176f2019-09-30 11:19:05 +02004215
4216 // Insert frame, expect scaled down:
4217 // framerate (640x360@24fps) -> resolution (480x270@24fps).
4218 InsertFrame();
4219 EXPECT_FALSE(WaitForFrame(1000));
4220 EXPECT_LT(source_.sink_wants().max_pixel_count, kWidth * kHeight);
4221 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 24);
4222
4223 // Insert frame, expect scaled down:
4224 // resolution (320x180@24fps).
4225 InsertFrame();
4226 EXPECT_FALSE(WaitForFrame(1000));
4227 EXPECT_LT(source_.sink_wants().max_pixel_count,
4228 source_.last_wants().max_pixel_count);
4229 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 24);
4230
4231 // Frame should not be dropped (min pixels per frame reached).
4232 InsertFrameAndWaitForEncoded();
4233
4234 video_stream_encoder_->Stop();
4235}
4236
4237TEST_F(BalancedDegradationTest,
Åsa Persson30ab0152019-08-27 12:22:33 +02004238 NoAdaptUpInResolutionIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004239 test::ScopedFieldTrials field_trials(
Åsa Persson30ab0152019-08-27 12:22:33 +02004240 "WebRTC-Video-BalancedDegradationSettings/"
4241 "pixels:57600|129600|230400,fps:7|10|14,kbps_res:0|0|435/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02004242 SetupTest();
Åsa Persson30ab0152019-08-27 12:22:33 +02004243
Asa Persson606d3cb2021-10-04 10:07:11 +02004244 const DataRate kResolutionMinBitrate = DataRate::KilobitsPerSec(435);
4245 const DataRate kTooLowMinResolutionBitrate = DataRate::KilobitsPerSec(434);
4246 OnBitrateUpdated(kTooLowMinResolutionBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004247
Åsa Persson45b176f2019-09-30 11:19:05 +02004248 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004249 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson30ab0152019-08-27 12:22:33 +02004250 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4251
4252 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
4253 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004254 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004255 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson30ab0152019-08-27 12:22:33 +02004256 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4257
4258 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
4259 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004260 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004261 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004262 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4263
4264 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
4265 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004266 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004267 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Persson1b247f12019-08-14 17:26:39 +02004268 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4269
Åsa Persson30ab0152019-08-27 12:22:33 +02004270 // Trigger adapt up, expect upscaled fps (no bitrate limit) (480x270@14fps).
4271 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004272 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004273 EXPECT_THAT(source_.sink_wants(), FpsGtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004274 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4275
4276 // Trigger adapt up, expect no upscale in res (target bitrate < min bitrate).
4277 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004278 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02004279 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4280
4281 // Trigger adapt up, expect upscaled res (target bitrate == min bitrate).
Asa Persson606d3cb2021-10-04 10:07:11 +02004282 OnBitrateUpdated(kResolutionMinBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004283 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004284 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004285 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionGt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004286 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4287
4288 video_stream_encoder_->Stop();
4289}
4290
Åsa Perssonccfb3402019-09-25 15:13:04 +02004291TEST_F(BalancedDegradationTest,
Åsa Persson30ab0152019-08-27 12:22:33 +02004292 NoAdaptUpInFpsAndResolutionIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004293 test::ScopedFieldTrials field_trials(
Åsa Persson30ab0152019-08-27 12:22:33 +02004294 "WebRTC-Video-BalancedDegradationSettings/"
4295 "pixels:57600|129600|230400,fps:7|10|14,kbps:0|0|425,kbps_res:0|0|435/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02004296 SetupTest();
Åsa Persson30ab0152019-08-27 12:22:33 +02004297
Asa Persson606d3cb2021-10-04 10:07:11 +02004298 const DataRate kMinBitrate = DataRate::KilobitsPerSec(425);
4299 const DataRate kTooLowMinBitrate = DataRate::KilobitsPerSec(424);
4300 const DataRate kResolutionMinBitrate = DataRate::KilobitsPerSec(435);
4301 const DataRate kTooLowMinResolutionBitrate = DataRate::KilobitsPerSec(434);
4302 OnBitrateUpdated(kTooLowMinBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004303
Åsa Persson45b176f2019-09-30 11:19:05 +02004304 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004305 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson30ab0152019-08-27 12:22:33 +02004306 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4307
4308 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
4309 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004310 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004311 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson30ab0152019-08-27 12:22:33 +02004312 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4313
4314 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
4315 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004316 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004317 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004318 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4319
4320 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
4321 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004322 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004323 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004324 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4325
4326 // Trigger adapt up, expect no upscale (target bitrate < min bitrate).
4327 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004328 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02004329 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4330
4331 // Trigger adapt up, expect upscaled fps (target bitrate == min bitrate).
Asa Persson606d3cb2021-10-04 10:07:11 +02004332 OnBitrateUpdated(kMinBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004333 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004334 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004335 EXPECT_THAT(source_.sink_wants(), FpsGtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004336 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4337
4338 // Trigger adapt up, expect no upscale in res (target bitrate < min bitrate).
Asa Persson606d3cb2021-10-04 10:07:11 +02004339 OnBitrateUpdated(kTooLowMinResolutionBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004340 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004341 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02004342 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4343
4344 // Trigger adapt up, expect upscaled res (target bitrate == min bitrate).
Asa Persson606d3cb2021-10-04 10:07:11 +02004345 OnBitrateUpdated(kResolutionMinBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004346 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004347 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004348 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionGt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004349 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4350
Åsa Persson1b247f12019-08-14 17:26:39 +02004351 video_stream_encoder_->Stop();
4352}
4353
mflodmancc3d4422017-08-03 08:27:51 -07004354TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07004355 AdaptsResolutionOnOveruseAndLowQuality_MaintainFramerateMode) {
4356 const int kWidth = 1280;
4357 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02004358 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004359 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07004360
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004361 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004362 AdaptingFrameForwarder source(&time_controller_);
asaperssond0de2952017-04-21 01:47:31 -07004363 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07004364 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004365 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07004366
Åsa Persson8c1bf952018-09-13 10:42:19 +02004367 int64_t timestamp_ms = kFrameIntervalMs;
4368 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004369 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004370 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07004371 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4372 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4373 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4374 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4375
4376 // Trigger cpu adapt down, expect scaled down resolution (960x540).
mflodmancc3d4422017-08-03 08:27:51 -07004377 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004378 timestamp_ms += kFrameIntervalMs;
4379 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4380 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004381 EXPECT_THAT(source.sink_wants(),
4382 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07004383 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4384 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4385 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4386 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4387
4388 // Trigger cpu adapt down, expect scaled down resolution (640x360).
mflodmancc3d4422017-08-03 08:27:51 -07004389 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004390 timestamp_ms += kFrameIntervalMs;
4391 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4392 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004393 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssond0de2952017-04-21 01:47:31 -07004394 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4395 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4396 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4397 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4398
Jonathan Yubc771b72017-12-08 17:04:29 -08004399 // Trigger cpu adapt down, expect scaled down resolution (480x270).
mflodmancc3d4422017-08-03 08:27:51 -07004400 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004401 timestamp_ms += kFrameIntervalMs;
4402 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4403 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004404 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssond0de2952017-04-21 01:47:31 -07004405 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4406 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08004407 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07004408 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4409
Jonathan Yubc771b72017-12-08 17:04:29 -08004410 // Trigger quality adapt down, expect scaled down resolution (320x180).
mflodmancc3d4422017-08-03 08:27:51 -07004411 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004412 timestamp_ms += kFrameIntervalMs;
4413 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4414 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004415 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
Jonathan Yubc771b72017-12-08 17:04:29 -08004416 rtc::VideoSinkWants last_wants = source.sink_wants();
asaperssond0de2952017-04-21 01:47:31 -07004417 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4418 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4419 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4420 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4421
Jonathan Yubc771b72017-12-08 17:04:29 -08004422 // Trigger quality adapt down, expect no change (min resolution reached).
4423 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004424 timestamp_ms += kFrameIntervalMs;
4425 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4426 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004427 EXPECT_THAT(source.sink_wants(), FpsMax());
4428 EXPECT_EQ(source.sink_wants().max_pixel_count, last_wants.max_pixel_count);
Jonathan Yubc771b72017-12-08 17:04:29 -08004429 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4430 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4431 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4432 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4433
Evan Shrubsole64469032020-06-11 10:45:29 +02004434 // Trigger quality adapt up, expect upscaled resolution (480x270).
4435 video_stream_encoder_->TriggerQualityHigh();
4436 timestamp_ms += kFrameIntervalMs;
4437 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4438 WaitForEncodedFrame(timestamp_ms);
4439 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
4440 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4441 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4442 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4443 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4444
4445 // Trigger quality and cpu adapt up since both are most limited, expect
4446 // upscaled resolution (640x360).
Henrik Boström91aa7322020-04-28 12:24:33 +02004447 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole64469032020-06-11 10:45:29 +02004448 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004449 timestamp_ms += kFrameIntervalMs;
4450 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4451 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004452 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Jonathan Yubc771b72017-12-08 17:04:29 -08004453 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4454 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4455 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02004456 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
Jonathan Yubc771b72017-12-08 17:04:29 -08004457
Evan Shrubsole64469032020-06-11 10:45:29 +02004458 // Trigger quality and cpu adapt up since both are most limited, expect
4459 // upscaled resolution (960x540).
Henrik Boström91aa7322020-04-28 12:24:33 +02004460 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole64469032020-06-11 10:45:29 +02004461 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004462 timestamp_ms += kFrameIntervalMs;
4463 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4464 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004465 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssond0de2952017-04-21 01:47:31 -07004466 last_wants = source.sink_wants();
Evan Shrubsole64469032020-06-11 10:45:29 +02004467 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
asaperssond0de2952017-04-21 01:47:31 -07004468 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Evan Shrubsole64469032020-06-11 10:45:29 +02004469 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4470 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07004471
Evan Shrubsole64469032020-06-11 10:45:29 +02004472 // Trigger cpu adapt up, expect no change since not most limited (960x540).
4473 // However the stats will change since the CPU resource is no longer limited.
Henrik Boström91aa7322020-04-28 12:24:33 +02004474 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004475 timestamp_ms += kFrameIntervalMs;
4476 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4477 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004478 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(last_wants));
asaperssond0de2952017-04-21 01:47:31 -07004479 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4480 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08004481 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02004482 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07004483
4484 // Trigger quality adapt up, expect no restriction (1280x720).
mflodmancc3d4422017-08-03 08:27:51 -07004485 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004486 timestamp_ms += kFrameIntervalMs;
4487 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004488 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004489 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004490 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07004491 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4492 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08004493 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02004494 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
kthelgason5e13d412016-12-01 03:59:51 -08004495
mflodmancc3d4422017-08-03 08:27:51 -07004496 video_stream_encoder_->Stop();
kthelgason5e13d412016-12-01 03:59:51 -08004497}
4498
mflodmancc3d4422017-08-03 08:27:51 -07004499TEST_F(VideoStreamEncoderTest, CpuLimitedHistogramIsReported) {
asaperssonfab67072017-04-04 05:51:49 -07004500 const int kWidth = 640;
4501 const int kHeight = 360;
perkj803d97f2016-11-01 11:45:46 -07004502
Henrik Boström381d1092020-05-12 18:49:07 +02004503 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004504 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07004505
perkj803d97f2016-11-01 11:45:46 -07004506 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07004507 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004508 WaitForEncodedFrame(i);
perkj803d97f2016-11-01 11:45:46 -07004509 }
4510
mflodmancc3d4422017-08-03 08:27:51 -07004511 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07004512 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07004513 video_source_.IncomingCapturedFrame(CreateFrame(
4514 SendStatisticsProxy::kMinRequiredMetricsSamples + i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004515 WaitForEncodedFrame(SendStatisticsProxy::kMinRequiredMetricsSamples + i);
perkj803d97f2016-11-01 11:45:46 -07004516 }
4517
mflodmancc3d4422017-08-03 08:27:51 -07004518 video_stream_encoder_->Stop();
4519 video_stream_encoder_.reset();
perkj803d97f2016-11-01 11:45:46 -07004520 stats_proxy_.reset();
sprangf8ee65e2017-02-28 08:49:33 -08004521
Ying Wangef3998f2019-12-09 13:06:53 +01004522 EXPECT_METRIC_EQ(
4523 1, metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
4524 EXPECT_METRIC_EQ(
perkj803d97f2016-11-01 11:45:46 -07004525 1, metrics::NumEvents("WebRTC.Video.CpuLimitedResolutionInPercent", 50));
4526}
4527
mflodmancc3d4422017-08-03 08:27:51 -07004528TEST_F(VideoStreamEncoderTest,
4529 CpuLimitedHistogramIsNotReportedForDisabledDegradation) {
Henrik Boström381d1092020-05-12 18:49:07 +02004530 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004531 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf4e44af2017-04-19 02:01:06 -07004532 const int kWidth = 640;
4533 const int kHeight = 360;
4534
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004535 video_stream_encoder_->SetSource(&video_source_,
4536 webrtc::DegradationPreference::DISABLED);
asaperssonf4e44af2017-04-19 02:01:06 -07004537
4538 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
4539 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004540 WaitForEncodedFrame(i);
asaperssonf4e44af2017-04-19 02:01:06 -07004541 }
4542
mflodmancc3d4422017-08-03 08:27:51 -07004543 video_stream_encoder_->Stop();
4544 video_stream_encoder_.reset();
asaperssonf4e44af2017-04-19 02:01:06 -07004545 stats_proxy_.reset();
4546
4547 EXPECT_EQ(0,
4548 metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
4549}
4550
Per Kjellanderdcef6412020-10-07 15:09:05 +02004551TEST_F(VideoStreamEncoderTest, ReportsVideoBitrateAllocation) {
4552 ResetEncoder("FAKE", 1, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004553 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02004554 kVideoBitrateAllocation);
sprang57c2fff2017-01-16 06:24:02 -08004555
4556 const int kDefaultFps = 30;
Erik Språng566124a2018-04-23 12:32:22 +02004557 const VideoBitrateAllocation expected_bitrate =
Asa Persson606d3cb2021-10-04 10:07:11 +02004558 SimulcastRateAllocator(fake_encoder_.config())
4559 .Allocate(VideoBitrateAllocationParameters(kLowTargetBitrate.bps(),
Florent Castelli8bbdb5b2019-08-02 15:16:28 +02004560 kDefaultFps));
sprang57c2fff2017-01-16 06:24:02 -08004561
Henrik Boström381d1092020-05-12 18:49:07 +02004562 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004563 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
sprang57c2fff2017-01-16 06:24:02 -08004564
sprang57c2fff2017-01-16 06:24:02 -08004565 video_source_.IncomingCapturedFrame(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004566 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4567 WaitForEncodedFrame(CurrentTimeMs());
Per Kjellanderdcef6412020-10-07 15:09:05 +02004568 EXPECT_EQ(sink_.GetLastVideoBitrateAllocation(), expected_bitrate);
4569 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
4570
Erik Språngd7329ca2019-02-21 21:19:53 +01004571 // Check that encoder has been updated too, not just allocation observer.
Erik Språng9d69cbe2020-10-22 17:44:42 +02004572 EXPECT_TRUE(fake_encoder_.GetAndResetLastRateControlSettings().has_value());
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004573 AdvanceTime(TimeDelta::Seconds(1) / kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08004574
Per Kjellanderdcef6412020-10-07 15:09:05 +02004575 // VideoBitrateAllocation not updated on second frame.
sprang57c2fff2017-01-16 06:24:02 -08004576 video_source_.IncomingCapturedFrame(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004577 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4578 WaitForEncodedFrame(CurrentTimeMs());
Per Kjellanderdcef6412020-10-07 15:09:05 +02004579 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004580 AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08004581
Per Kjellanderdcef6412020-10-07 15:09:05 +02004582 // VideoBitrateAllocation updated after a process interval.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004583 const int64_t start_time_ms = CurrentTimeMs();
Per Kjellanderd0a8f512020-10-07 11:28:41 +02004584 while (CurrentTimeMs() - start_time_ms < 5 * kProcessIntervalMs) {
Erik Språngd7329ca2019-02-21 21:19:53 +01004585 video_source_.IncomingCapturedFrame(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004586 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4587 WaitForEncodedFrame(CurrentTimeMs());
4588 AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
Erik Språngd7329ca2019-02-21 21:19:53 +01004589 }
Per Kjellanderdcef6412020-10-07 15:09:05 +02004590 EXPECT_GT(sink_.number_of_bitrate_allocations(), 3);
Erik Språngd7329ca2019-02-21 21:19:53 +01004591
mflodmancc3d4422017-08-03 08:27:51 -07004592 video_stream_encoder_->Stop();
sprang57c2fff2017-01-16 06:24:02 -08004593}
4594
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004595TEST_F(VideoStreamEncoderTest, ReportsVideoLayersAllocationForVP8Simulcast) {
Per Kjellandera9434842020-10-15 17:53:22 +02004596 ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004597 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellandera9434842020-10-15 17:53:22 +02004598 kVideoLayersAllocation);
4599
4600 const int kDefaultFps = 30;
4601
4602 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004603 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Per Kjellandera9434842020-10-15 17:53:22 +02004604
4605 video_source_.IncomingCapturedFrame(
4606 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4607 WaitForEncodedFrame(CurrentTimeMs());
4608 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4609 VideoLayersAllocation last_layer_allocation =
4610 sink_.GetLastVideoLayersAllocation();
Asa Persson606d3cb2021-10-04 10:07:11 +02004611 // kLowTargetBitrate is only enough for one spatial layer.
Per Kjellandera9434842020-10-15 17:53:22 +02004612 ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 1u);
4613
4614 VideoBitrateAllocation bitrate_allocation =
Erik Språng9d69cbe2020-10-22 17:44:42 +02004615 fake_encoder_.GetAndResetLastRateControlSettings()->target_bitrate;
Per Kjellandera9434842020-10-15 17:53:22 +02004616 // Check that encoder has been updated too, not just allocation observer.
Asa Persson606d3cb2021-10-04 10:07:11 +02004617 EXPECT_EQ(bitrate_allocation.get_sum_bps(), kLowTargetBitrate.bps());
Per Kjellandera9434842020-10-15 17:53:22 +02004618 AdvanceTime(TimeDelta::Seconds(1) / kDefaultFps);
4619
Erik Språng9d69cbe2020-10-22 17:44:42 +02004620 // VideoLayersAllocation might be updated if frame rate changes.
Per Kjellandera9434842020-10-15 17:53:22 +02004621 int number_of_layers_allocation = 1;
4622 const int64_t start_time_ms = CurrentTimeMs();
4623 while (CurrentTimeMs() - start_time_ms < 10 * kProcessIntervalMs) {
4624 video_source_.IncomingCapturedFrame(
4625 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4626 WaitForEncodedFrame(CurrentTimeMs());
Per Kjellandera9434842020-10-15 17:53:22 +02004627 if (number_of_layers_allocation != sink_.number_of_layers_allocations()) {
4628 number_of_layers_allocation = sink_.number_of_layers_allocations();
4629 VideoLayersAllocation new_allocation =
4630 sink_.GetLastVideoLayersAllocation();
4631 ASSERT_EQ(new_allocation.active_spatial_layers.size(), 1u);
4632 EXPECT_NE(new_allocation.active_spatial_layers[0].frame_rate_fps,
4633 last_layer_allocation.active_spatial_layers[0].frame_rate_fps);
4634 EXPECT_EQ(new_allocation.active_spatial_layers[0]
4635 .target_bitrate_per_temporal_layer,
4636 last_layer_allocation.active_spatial_layers[0]
4637 .target_bitrate_per_temporal_layer);
4638 last_layer_allocation = new_allocation;
4639 }
4640 }
4641 EXPECT_LE(sink_.number_of_layers_allocations(), 3);
4642 video_stream_encoder_->Stop();
4643}
4644
4645TEST_F(VideoStreamEncoderTest,
Åsa Perssona7e34d32021-01-20 15:36:13 +01004646 ReportsVideoLayersAllocationForVP8WithMiddleLayerDisabled) {
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004647 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4648 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4649 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004650 VideoEncoderConfig video_encoder_config;
4651 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP8,
4652 /* num_streams*/ 3, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004653 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004654 video_encoder_config.content_type =
4655 VideoEncoderConfig::ContentType::kRealtimeVideo;
4656 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004657 rtc::make_ref_counted<VideoEncoderConfig::Vp8EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004658 VideoEncoder::GetDefaultVp8Settings());
4659 for (auto& layer : video_encoder_config.simulcast_layers) {
4660 layer.num_temporal_layers = 2;
4661 }
4662 // Simulcast layers are used for enabling/disabling streams.
4663 video_encoder_config.simulcast_layers[0].active = true;
4664 video_encoder_config.simulcast_layers[1].active = false;
4665 video_encoder_config.simulcast_layers[2].active = true;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004666 ConfigureEncoder(std::move(video_encoder_config),
4667 VideoStreamEncoder::BitrateAllocationCallbackType::
4668 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004669
4670 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004671 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004672
4673 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4674 WaitForEncodedFrame(CurrentTimeMs());
4675 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4676 VideoLayersAllocation last_layer_allocation =
4677 sink_.GetLastVideoLayersAllocation();
4678
4679 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4680 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4681 .target_bitrate_per_temporal_layer,
4682 SizeIs(2));
4683 EXPECT_LT(last_layer_allocation.active_spatial_layers[0].width, 1280);
4684 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 1280);
4685 video_stream_encoder_->Stop();
4686}
4687
4688TEST_F(VideoStreamEncoderTest,
Åsa Perssona7e34d32021-01-20 15:36:13 +01004689 ReportsVideoLayersAllocationForVP8WithMiddleAndHighestLayerDisabled) {
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004690 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4691 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4692 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004693 VideoEncoderConfig video_encoder_config;
4694 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP8,
4695 /* num_streams*/ 3, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004696 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004697 video_encoder_config.content_type =
4698 VideoEncoderConfig::ContentType::kRealtimeVideo;
4699 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004700 rtc::make_ref_counted<VideoEncoderConfig::Vp8EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004701 VideoEncoder::GetDefaultVp8Settings());
4702 for (auto& layer : video_encoder_config.simulcast_layers) {
4703 layer.num_temporal_layers = 2;
4704 }
4705 // Simulcast layers are used for enabling/disabling streams.
4706 video_encoder_config.simulcast_layers[0].active = true;
4707 video_encoder_config.simulcast_layers[1].active = false;
4708 video_encoder_config.simulcast_layers[2].active = false;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004709 ConfigureEncoder(std::move(video_encoder_config),
4710 VideoStreamEncoder::BitrateAllocationCallbackType::
4711 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004712
4713 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004714 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004715
4716 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4717 WaitForEncodedFrame(CurrentTimeMs());
4718 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4719 VideoLayersAllocation last_layer_allocation =
4720 sink_.GetLastVideoLayersAllocation();
4721
4722 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(1));
4723 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4724 .target_bitrate_per_temporal_layer,
4725 SizeIs(2));
4726 EXPECT_LT(last_layer_allocation.active_spatial_layers[0].width, 1280);
4727
4728 video_stream_encoder_->Stop();
4729}
4730
4731TEST_F(VideoStreamEncoderTest,
4732 ReportsVideoLayersAllocationForV9SvcWithTemporalLayerSupport) {
4733 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4734 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004735 VideoEncoderConfig video_encoder_config;
4736 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4737 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004738 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004739 video_encoder_config.content_type =
4740 VideoEncoderConfig::ContentType::kRealtimeVideo;
4741 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4742 vp9_settings.numberOfSpatialLayers = 2;
4743 vp9_settings.numberOfTemporalLayers = 2;
4744 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4745 vp9_settings.automaticResizeOn = false;
4746 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004747 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004748 vp9_settings);
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004749 ConfigureEncoder(std::move(video_encoder_config),
4750 VideoStreamEncoder::BitrateAllocationCallbackType::
4751 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004752
4753 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004754 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004755
4756 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4757 WaitForEncodedFrame(CurrentTimeMs());
4758 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4759 VideoLayersAllocation last_layer_allocation =
4760 sink_.GetLastVideoLayersAllocation();
4761
4762 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4763 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4764 .target_bitrate_per_temporal_layer,
4765 SizeIs(2));
4766 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 640);
4767 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].height, 360);
4768 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].frame_rate_fps, 30);
4769 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4770 .target_bitrate_per_temporal_layer,
4771 SizeIs(2));
4772 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 1280);
4773 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].height, 720);
4774 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].frame_rate_fps, 30);
4775
4776 // Since full SVC is used, expect the top layer to utilize the full target
4777 // rate.
4778 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1]
4779 .target_bitrate_per_temporal_layer[1],
Asa Persson606d3cb2021-10-04 10:07:11 +02004780 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004781 video_stream_encoder_->Stop();
4782}
4783
4784TEST_F(VideoStreamEncoderTest,
4785 ReportsVideoLayersAllocationForV9SvcWithoutTemporalLayerSupport) {
4786 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, false);
4787 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, false);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004788 VideoEncoderConfig video_encoder_config;
4789 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4790 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004791 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004792 video_encoder_config.content_type =
4793 VideoEncoderConfig::ContentType::kRealtimeVideo;
4794 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4795 vp9_settings.numberOfSpatialLayers = 2;
4796 vp9_settings.numberOfTemporalLayers = 2;
4797 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4798 vp9_settings.automaticResizeOn = false;
4799 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004800 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004801 vp9_settings);
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004802 ConfigureEncoder(std::move(video_encoder_config),
4803 VideoStreamEncoder::BitrateAllocationCallbackType::
4804 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004805
4806 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004807 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004808
4809 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4810 WaitForEncodedFrame(CurrentTimeMs());
4811 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4812 VideoLayersAllocation last_layer_allocation =
4813 sink_.GetLastVideoLayersAllocation();
4814
4815 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4816 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4817 .target_bitrate_per_temporal_layer,
4818 SizeIs(1));
4819 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4820 .target_bitrate_per_temporal_layer,
4821 SizeIs(1));
4822 // Since full SVC is used, expect the top layer to utilize the full target
4823 // rate.
4824 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1]
4825 .target_bitrate_per_temporal_layer[0],
Asa Persson606d3cb2021-10-04 10:07:11 +02004826 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004827 video_stream_encoder_->Stop();
4828}
4829
4830TEST_F(VideoStreamEncoderTest,
4831 ReportsVideoLayersAllocationForVP9KSvcWithTemporalLayerSupport) {
4832 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4833 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004834 VideoEncoderConfig video_encoder_config;
4835 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4836 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004837 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004838 video_encoder_config.content_type =
4839 VideoEncoderConfig::ContentType::kRealtimeVideo;
4840 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4841 vp9_settings.numberOfSpatialLayers = 2;
4842 vp9_settings.numberOfTemporalLayers = 2;
4843 vp9_settings.interLayerPred = InterLayerPredMode::kOnKeyPic;
4844 vp9_settings.automaticResizeOn = false;
4845 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004846 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004847 vp9_settings);
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004848 ConfigureEncoder(std::move(video_encoder_config),
4849 VideoStreamEncoder::BitrateAllocationCallbackType::
4850 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004851
4852 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004853 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004854
4855 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4856 WaitForEncodedFrame(CurrentTimeMs());
4857 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4858 VideoLayersAllocation last_layer_allocation =
4859 sink_.GetLastVideoLayersAllocation();
4860
4861 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4862 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4863 .target_bitrate_per_temporal_layer,
4864 SizeIs(2));
4865 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4866 .target_bitrate_per_temporal_layer,
4867 SizeIs(2));
4868 // Since KSVC is, spatial layers are independend except on key frames.
4869 EXPECT_LT(last_layer_allocation.active_spatial_layers[1]
4870 .target_bitrate_per_temporal_layer[1],
Asa Persson606d3cb2021-10-04 10:07:11 +02004871 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004872 video_stream_encoder_->Stop();
4873}
4874
4875TEST_F(VideoStreamEncoderTest,
4876 ReportsVideoLayersAllocationForV9SvcWithLowestLayerDisabled) {
4877 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4878 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4879 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004880 VideoEncoderConfig video_encoder_config;
4881 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4882 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004883 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004884 video_encoder_config.content_type =
4885 VideoEncoderConfig::ContentType::kRealtimeVideo;
4886 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4887 vp9_settings.numberOfSpatialLayers = 3;
4888 vp9_settings.numberOfTemporalLayers = 2;
4889 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4890 vp9_settings.automaticResizeOn = false;
4891 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004892 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004893 vp9_settings);
4894 // Simulcast layers are used for enabling/disabling streams.
4895 video_encoder_config.simulcast_layers.resize(3);
4896 video_encoder_config.simulcast_layers[0].active = false;
4897 video_encoder_config.simulcast_layers[1].active = true;
4898 video_encoder_config.simulcast_layers[2].active = true;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004899 ConfigureEncoder(std::move(video_encoder_config),
4900 VideoStreamEncoder::BitrateAllocationCallbackType::
4901 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004902
4903 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004904 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004905
4906 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4907 WaitForEncodedFrame(CurrentTimeMs());
4908 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4909 VideoLayersAllocation last_layer_allocation =
4910 sink_.GetLastVideoLayersAllocation();
4911
4912 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4913 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4914 .target_bitrate_per_temporal_layer,
4915 SizeIs(2));
4916 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 640);
4917 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].spatial_id, 0);
4918
4919 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 1280);
4920 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].spatial_id, 1);
4921 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4922 .target_bitrate_per_temporal_layer,
4923 SizeIs(2));
4924 // Since full SVC is used, expect the top layer to utilize the full target
4925 // rate.
4926 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1]
4927 .target_bitrate_per_temporal_layer[1],
Asa Persson606d3cb2021-10-04 10:07:11 +02004928 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004929 video_stream_encoder_->Stop();
4930}
4931
4932TEST_F(VideoStreamEncoderTest,
4933 ReportsVideoLayersAllocationForV9SvcWithHighestLayerDisabled) {
4934 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4935 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4936 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004937 VideoEncoderConfig video_encoder_config;
4938 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4939 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004940 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004941 video_encoder_config.content_type =
4942 VideoEncoderConfig::ContentType::kRealtimeVideo;
4943 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4944 vp9_settings.numberOfSpatialLayers = 3;
4945 vp9_settings.numberOfTemporalLayers = 2;
4946 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4947 vp9_settings.automaticResizeOn = false;
4948 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004949 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004950 vp9_settings);
4951 // Simulcast layers are used for enabling/disabling streams.
4952 video_encoder_config.simulcast_layers.resize(3);
4953 video_encoder_config.simulcast_layers[2].active = false;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004954 ConfigureEncoder(std::move(video_encoder_config),
4955 VideoStreamEncoder::BitrateAllocationCallbackType::
4956 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004957
4958 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004959 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004960
4961 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4962 WaitForEncodedFrame(CurrentTimeMs());
4963 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4964 VideoLayersAllocation last_layer_allocation =
4965 sink_.GetLastVideoLayersAllocation();
4966
4967 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4968 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4969 .target_bitrate_per_temporal_layer,
4970 SizeIs(2));
4971 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 320);
4972 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].spatial_id, 0);
4973
4974 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 640);
4975 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].spatial_id, 1);
4976 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4977 .target_bitrate_per_temporal_layer,
4978 SizeIs(2));
4979 video_stream_encoder_->Stop();
4980}
4981
4982TEST_F(VideoStreamEncoderTest,
4983 ReportsVideoLayersAllocationForV9SvcWithAllButHighestLayerDisabled) {
4984 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4985 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4986 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004987 VideoEncoderConfig video_encoder_config;
4988 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4989 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004990 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004991 video_encoder_config.content_type =
4992 VideoEncoderConfig::ContentType::kRealtimeVideo;
4993 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4994 vp9_settings.numberOfSpatialLayers = 3;
4995 vp9_settings.numberOfTemporalLayers = 2;
4996 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4997 vp9_settings.automaticResizeOn = false;
4998 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004999 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005000 vp9_settings);
5001 // Simulcast layers are used for enabling/disabling streams.
5002 video_encoder_config.simulcast_layers.resize(3);
5003 video_encoder_config.simulcast_layers[0].active = false;
5004 video_encoder_config.simulcast_layers[1].active = false;
5005 video_encoder_config.simulcast_layers[2].active = true;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005006 ConfigureEncoder(std::move(video_encoder_config),
5007 VideoStreamEncoder::BitrateAllocationCallbackType::
5008 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005009
5010 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005011 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005012
5013 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
5014 WaitForEncodedFrame(CurrentTimeMs());
5015 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
5016 VideoLayersAllocation last_layer_allocation =
5017 sink_.GetLastVideoLayersAllocation();
5018
5019 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(1));
5020 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
5021 .target_bitrate_per_temporal_layer,
5022 SizeIs(2));
5023 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 1280);
5024 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].spatial_id, 0);
5025 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0]
5026 .target_bitrate_per_temporal_layer[1],
Asa Persson606d3cb2021-10-04 10:07:11 +02005027 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005028 video_stream_encoder_->Stop();
5029}
5030
5031TEST_F(VideoStreamEncoderTest, ReportsVideoLayersAllocationForH264) {
5032 ResetEncoder("H264", 1, 1, 1, false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005033 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005034 kVideoLayersAllocation);
5035 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005036 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005037
5038 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
5039 WaitForEncodedFrame(CurrentTimeMs());
5040 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
5041 VideoLayersAllocation last_layer_allocation =
5042 sink_.GetLastVideoLayersAllocation();
5043
5044 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(1));
5045 ASSERT_THAT(last_layer_allocation.active_spatial_layers[0]
5046 .target_bitrate_per_temporal_layer,
5047 SizeIs(1));
5048 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0]
5049 .target_bitrate_per_temporal_layer[0],
Asa Persson606d3cb2021-10-04 10:07:11 +02005050 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005051 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 1280);
5052 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].height, 720);
5053 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].frame_rate_fps, 30);
5054 video_stream_encoder_->Stop();
5055}
5056
5057TEST_F(VideoStreamEncoderTest,
Per Kjellandera9434842020-10-15 17:53:22 +02005058 ReportsUpdatedVideoLayersAllocationWhenBweChanges) {
5059 ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005060 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellandera9434842020-10-15 17:53:22 +02005061 kVideoLayersAllocation);
5062
5063 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005064 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Per Kjellandera9434842020-10-15 17:53:22 +02005065
5066 video_source_.IncomingCapturedFrame(
5067 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
5068 WaitForEncodedFrame(CurrentTimeMs());
5069 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
5070 VideoLayersAllocation last_layer_allocation =
5071 sink_.GetLastVideoLayersAllocation();
Asa Persson606d3cb2021-10-04 10:07:11 +02005072 // kLowTargetBitrate is only enough for one spatial layer.
Per Kjellandera9434842020-10-15 17:53:22 +02005073 ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 1u);
5074 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0]
5075 .target_bitrate_per_temporal_layer[0],
Asa Persson606d3cb2021-10-04 10:07:11 +02005076 kLowTargetBitrate);
Per Kjellandera9434842020-10-15 17:53:22 +02005077
5078 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005079 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
5080 0, 0, 0);
Per Kjellandera9434842020-10-15 17:53:22 +02005081 video_source_.IncomingCapturedFrame(
5082 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
5083 WaitForEncodedFrame(CurrentTimeMs());
5084
5085 EXPECT_EQ(sink_.number_of_layers_allocations(), 2);
5086 last_layer_allocation = sink_.GetLastVideoLayersAllocation();
5087 ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 2u);
5088 EXPECT_GT(last_layer_allocation.active_spatial_layers[1]
5089 .target_bitrate_per_temporal_layer[0],
5090 DataRate::Zero());
5091
5092 video_stream_encoder_->Stop();
5093}
5094
Per Kjellander4190ce92020-12-15 17:24:55 +01005095TEST_F(VideoStreamEncoderTest,
5096 ReportsUpdatedVideoLayersAllocationWhenResolutionChanges) {
5097 ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005098 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellander4190ce92020-12-15 17:24:55 +01005099 kVideoLayersAllocation);
5100
5101 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005102 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
5103 0, 0, 0);
Per Kjellander4190ce92020-12-15 17:24:55 +01005104
5105 video_source_.IncomingCapturedFrame(
5106 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
5107 WaitForEncodedFrame(CurrentTimeMs());
5108 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
5109 ASSERT_THAT(sink_.GetLastVideoLayersAllocation().active_spatial_layers,
5110 SizeIs(2));
5111 EXPECT_EQ(sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].width,
5112 codec_width_);
5113 EXPECT_EQ(
5114 sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].height,
5115 codec_height_);
5116
5117 video_source_.IncomingCapturedFrame(
5118 CreateFrame(CurrentTimeMs(), codec_width_ / 2, codec_height_ / 2));
5119 WaitForEncodedFrame(CurrentTimeMs());
5120 EXPECT_EQ(sink_.number_of_layers_allocations(), 2);
5121 ASSERT_THAT(sink_.GetLastVideoLayersAllocation().active_spatial_layers,
5122 SizeIs(2));
5123 EXPECT_EQ(sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].width,
5124 codec_width_ / 2);
5125 EXPECT_EQ(
5126 sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].height,
5127 codec_height_ / 2);
5128
5129 video_stream_encoder_->Stop();
5130}
5131
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005132TEST_F(VideoStreamEncoderTest, TemporalLayersNotDisabledIfSupported) {
5133 // 2 TLs configured, temporal layers supported by encoder.
5134 const int kNumTemporalLayers = 2;
Per Kjellanderdcef6412020-10-07 15:09:05 +02005135 ResetEncoder("VP8", 1, kNumTemporalLayers, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005136 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02005137 kVideoBitrateAllocation);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005138 fake_encoder_.SetTemporalLayersSupported(0, true);
5139
5140 // Bitrate allocated across temporal layers.
Asa Persson606d3cb2021-10-04 10:07:11 +02005141 const int kTl0Bps = kTargetBitrate.bps() *
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005142 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01005143 kNumTemporalLayers, /*temporal_id*/ 0,
5144 /*base_heavy_tl3_alloc*/ false);
Asa Persson606d3cb2021-10-04 10:07:11 +02005145 const int kTl1Bps = kTargetBitrate.bps() *
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005146 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01005147 kNumTemporalLayers, /*temporal_id*/ 1,
5148 /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005149 VideoBitrateAllocation expected_bitrate;
5150 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTl0Bps);
5151 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kTl1Bps - kTl0Bps);
5152
5153 VerifyAllocatedBitrate(expected_bitrate);
5154 video_stream_encoder_->Stop();
5155}
5156
5157TEST_F(VideoStreamEncoderTest, TemporalLayersDisabledIfNotSupported) {
5158 // 2 TLs configured, temporal layers not supported by encoder.
Per Kjellanderdcef6412020-10-07 15:09:05 +02005159 ResetEncoder("VP8", 1, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005160 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02005161 kVideoBitrateAllocation);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005162 fake_encoder_.SetTemporalLayersSupported(0, false);
5163
5164 // Temporal layers not supported by the encoder.
5165 // Total bitrate should be at ti:0.
5166 VideoBitrateAllocation expected_bitrate;
Asa Persson606d3cb2021-10-04 10:07:11 +02005167 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTargetBitrate.bps());
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005168
5169 VerifyAllocatedBitrate(expected_bitrate);
5170 video_stream_encoder_->Stop();
5171}
5172
5173TEST_F(VideoStreamEncoderTest, VerifyBitrateAllocationForTwoStreams) {
Per Kjellanderdcef6412020-10-07 15:09:05 +02005174 webrtc::test::ScopedFieldTrials field_trials(
5175 "WebRTC-Video-QualityScalerSettings/"
5176 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
5177 // Reset encoder for field trials to take effect.
5178 ConfigureEncoder(video_encoder_config_.Copy());
5179
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005180 // 2 TLs configured, temporal layers only supported for first stream.
Per Kjellanderdcef6412020-10-07 15:09:05 +02005181 ResetEncoder("VP8", 2, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005182 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02005183 kVideoBitrateAllocation);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005184 fake_encoder_.SetTemporalLayersSupported(0, true);
5185 fake_encoder_.SetTemporalLayersSupported(1, false);
5186
5187 const int kS0Bps = 150000;
5188 const int kS0Tl0Bps =
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01005189 kS0Bps *
5190 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
5191 /*num_layers*/ 2, /*temporal_id*/ 0, /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005192 const int kS0Tl1Bps =
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01005193 kS0Bps *
5194 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
5195 /*num_layers*/ 2, /*temporal_id*/ 1, /*base_heavy_tl3_alloc*/ false);
Asa Persson606d3cb2021-10-04 10:07:11 +02005196 const int kS1Bps = kTargetBitrate.bps() - kS0Tl1Bps;
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005197 // Temporal layers not supported by si:1.
5198 VideoBitrateAllocation expected_bitrate;
5199 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kS0Tl0Bps);
5200 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kS0Tl1Bps - kS0Tl0Bps);
5201 expected_bitrate.SetBitrate(/*si*/ 1, /*ti*/ 0, kS1Bps);
5202
5203 VerifyAllocatedBitrate(expected_bitrate);
5204 video_stream_encoder_->Stop();
5205}
5206
Niels Möller7dc26b72017-12-06 10:27:48 +01005207TEST_F(VideoStreamEncoderTest, OveruseDetectorUpdatedOnReconfigureAndAdaption) {
5208 const int kFrameWidth = 1280;
5209 const int kFrameHeight = 720;
5210 const int kFramerate = 24;
5211
Henrik Boström381d1092020-05-12 18:49:07 +02005212 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005213 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller7dc26b72017-12-06 10:27:48 +01005214 test::FrameForwarder source;
5215 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005216 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 10:27:48 +01005217
5218 // Insert a single frame, triggering initial configuration.
5219 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
5220 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5221
5222 EXPECT_EQ(
5223 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5224 kDefaultFramerate);
5225
5226 // Trigger reconfigure encoder (without resetting the entire instance).
5227 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +02005228 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5229 video_encoder_config.simulcast_layers[0].max_framerate = kFramerate;
Asa Persson606d3cb2021-10-04 10:07:11 +02005230 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
Niels Möller7dc26b72017-12-06 10:27:48 +01005231 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02005232 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01005233 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5234
5235 // Detector should be updated with fps limit from codec config.
5236 EXPECT_EQ(
5237 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5238 kFramerate);
5239
5240 // Trigger overuse, max framerate should be reduced.
5241 VideoSendStream::Stats stats = stats_proxy_->GetStats();
5242 stats.input_frame_rate = kFramerate;
5243 stats_proxy_->SetMockStats(stats);
5244 video_stream_encoder_->TriggerCpuOveruse();
5245 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5246 int adapted_framerate =
5247 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
5248 EXPECT_LT(adapted_framerate, kFramerate);
5249
5250 // Trigger underuse, max framerate should go back to codec configured fps.
5251 // Set extra low fps, to make sure it's actually reset, not just incremented.
5252 stats = stats_proxy_->GetStats();
5253 stats.input_frame_rate = adapted_framerate / 2;
5254 stats_proxy_->SetMockStats(stats);
Henrik Boström91aa7322020-04-28 12:24:33 +02005255 video_stream_encoder_->TriggerCpuUnderuse();
Niels Möller7dc26b72017-12-06 10:27:48 +01005256 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5257 EXPECT_EQ(
5258 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5259 kFramerate);
5260
5261 video_stream_encoder_->Stop();
5262}
5263
5264TEST_F(VideoStreamEncoderTest,
5265 OveruseDetectorUpdatedRespectsFramerateAfterUnderuse) {
5266 const int kFrameWidth = 1280;
5267 const int kFrameHeight = 720;
5268 const int kLowFramerate = 15;
5269 const int kHighFramerate = 25;
5270
Henrik Boström381d1092020-05-12 18:49:07 +02005271 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005272 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller7dc26b72017-12-06 10:27:48 +01005273 test::FrameForwarder source;
5274 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005275 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 10:27:48 +01005276
5277 // Trigger initial configuration.
5278 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +02005279 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5280 video_encoder_config.simulcast_layers[0].max_framerate = kLowFramerate;
Asa Persson606d3cb2021-10-04 10:07:11 +02005281 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
Niels Möller7dc26b72017-12-06 10:27:48 +01005282 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
Åsa Persson17107062020-10-08 08:57:51 +02005283 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
Niels Möllerf1338562018-04-26 09:51:47 +02005284 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01005285 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5286
5287 EXPECT_EQ(
5288 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5289 kLowFramerate);
5290
5291 // Trigger overuse, max framerate should be reduced.
5292 VideoSendStream::Stats stats = stats_proxy_->GetStats();
5293 stats.input_frame_rate = kLowFramerate;
5294 stats_proxy_->SetMockStats(stats);
5295 video_stream_encoder_->TriggerCpuOveruse();
5296 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5297 int adapted_framerate =
5298 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
5299 EXPECT_LT(adapted_framerate, kLowFramerate);
5300
5301 // Reconfigure the encoder with a new (higher max framerate), max fps should
5302 // still respect the adaptation.
Åsa Persson17107062020-10-08 08:57:51 +02005303 video_encoder_config.simulcast_layers[0].max_framerate = kHighFramerate;
Niels Möller7dc26b72017-12-06 10:27:48 +01005304 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
5305 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02005306 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01005307 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5308
5309 EXPECT_EQ(
5310 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5311 adapted_framerate);
5312
5313 // Trigger underuse, max framerate should go back to codec configured fps.
5314 stats = stats_proxy_->GetStats();
5315 stats.input_frame_rate = adapted_framerate;
5316 stats_proxy_->SetMockStats(stats);
Henrik Boström91aa7322020-04-28 12:24:33 +02005317 video_stream_encoder_->TriggerCpuUnderuse();
Niels Möller7dc26b72017-12-06 10:27:48 +01005318 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5319 EXPECT_EQ(
5320 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5321 kHighFramerate);
5322
5323 video_stream_encoder_->Stop();
5324}
5325
mflodmancc3d4422017-08-03 08:27:51 -07005326TEST_F(VideoStreamEncoderTest,
5327 OveruseDetectorUpdatedOnDegradationPreferenceChange) {
sprangfda496a2017-06-15 04:21:07 -07005328 const int kFrameWidth = 1280;
5329 const int kFrameHeight = 720;
5330 const int kFramerate = 24;
5331
Henrik Boström381d1092020-05-12 18:49:07 +02005332 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005333 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprangfda496a2017-06-15 04:21:07 -07005334 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07005335 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005336 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangfda496a2017-06-15 04:21:07 -07005337
5338 // Trigger initial configuration.
5339 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +02005340 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5341 video_encoder_config.simulcast_layers[0].max_framerate = kFramerate;
Asa Persson606d3cb2021-10-04 10:07:11 +02005342 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
sprangfda496a2017-06-15 04:21:07 -07005343 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
mflodmancc3d4422017-08-03 08:27:51 -07005344 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02005345 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -07005346 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprangfda496a2017-06-15 04:21:07 -07005347
Niels Möller7dc26b72017-12-06 10:27:48 +01005348 EXPECT_EQ(
5349 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5350 kFramerate);
sprangfda496a2017-06-15 04:21:07 -07005351
5352 // Trigger overuse, max framerate should be reduced.
5353 VideoSendStream::Stats stats = stats_proxy_->GetStats();
5354 stats.input_frame_rate = kFramerate;
5355 stats_proxy_->SetMockStats(stats);
mflodmancc3d4422017-08-03 08:27:51 -07005356 video_stream_encoder_->TriggerCpuOveruse();
5357 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Niels Möller7dc26b72017-12-06 10:27:48 +01005358 int adapted_framerate =
5359 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
sprangfda496a2017-06-15 04:21:07 -07005360 EXPECT_LT(adapted_framerate, kFramerate);
5361
5362 // Change degradation preference to not enable framerate scaling. Target
5363 // framerate should be changed to codec defined limit.
Henrik Boström381d1092020-05-12 18:49:07 +02005364 video_stream_encoder_->SetSourceAndWaitForFramerateUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005365 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Niels Möller7dc26b72017-12-06 10:27:48 +01005366 EXPECT_EQ(
5367 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5368 kFramerate);
sprangfda496a2017-06-15 04:21:07 -07005369
mflodmancc3d4422017-08-03 08:27:51 -07005370 video_stream_encoder_->Stop();
sprangfda496a2017-06-15 04:21:07 -07005371}
5372
mflodmancc3d4422017-08-03 08:27:51 -07005373TEST_F(VideoStreamEncoderTest, DropsFramesAndScalesWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 05:51:49 -07005374 const int kTooLowBitrateForFrameSizeBps = 10000;
Henrik Boström381d1092020-05-12 18:49:07 +02005375 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005376 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5377 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5378 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07005379 const int kWidth = 640;
5380 const int kHeight = 360;
kthelgason2bc68642017-02-07 07:02:22 -08005381
asaperssonfab67072017-04-04 05:51:49 -07005382 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 07:02:22 -08005383
5384 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 07:06:52 -07005385 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08005386
5387 // Expect the sink_wants to specify a scaled frame.
Henrik Boström2671dac2020-05-19 16:29:09 +02005388 EXPECT_TRUE_WAIT(
5389 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
kthelgason2bc68642017-02-07 07:02:22 -08005390
sprangc5d62e22017-04-02 23:53:04 -07005391 int last_pixel_count = video_source_.sink_wants().max_pixel_count;
kthelgason2bc68642017-02-07 07:02:22 -08005392
asaperssonfab67072017-04-04 05:51:49 -07005393 // Next frame is scaled.
kthelgason2bc68642017-02-07 07:02:22 -08005394 video_source_.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07005395 CreateFrame(2, kWidth * 3 / 4, kHeight * 3 / 4));
kthelgason2bc68642017-02-07 07:02:22 -08005396
5397 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 07:06:52 -07005398 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08005399
Henrik Boström2671dac2020-05-19 16:29:09 +02005400 EXPECT_TRUE_WAIT(
5401 video_source_.sink_wants().max_pixel_count < last_pixel_count, 5000);
kthelgason2bc68642017-02-07 07:02:22 -08005402
mflodmancc3d4422017-08-03 08:27:51 -07005403 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08005404}
5405
mflodmancc3d4422017-08-03 08:27:51 -07005406TEST_F(VideoStreamEncoderTest,
5407 NumberOfDroppedFramesLimitedWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 05:51:49 -07005408 const int kTooLowBitrateForFrameSizeBps = 10000;
Henrik Boström381d1092020-05-12 18:49:07 +02005409 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005410 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5411 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5412 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07005413 const int kWidth = 640;
5414 const int kHeight = 360;
kthelgason2bc68642017-02-07 07:02:22 -08005415
5416 // We expect the n initial frames to get dropped.
5417 int i;
5418 for (i = 1; i <= kMaxInitialFramedrop; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07005419 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005420 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08005421 }
5422 // The n+1th frame should not be dropped, even though it's size is too large.
asaperssonfab67072017-04-04 05:51:49 -07005423 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005424 WaitForEncodedFrame(i);
kthelgason2bc68642017-02-07 07:02:22 -08005425
5426 // Expect the sink_wants to specify a scaled frame.
asaperssonfab67072017-04-04 05:51:49 -07005427 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason2bc68642017-02-07 07:02:22 -08005428
mflodmancc3d4422017-08-03 08:27:51 -07005429 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08005430}
5431
mflodmancc3d4422017-08-03 08:27:51 -07005432TEST_F(VideoStreamEncoderTest,
5433 InitialFrameDropOffWithMaintainResolutionPreference) {
asaperssonfab67072017-04-04 05:51:49 -07005434 const int kWidth = 640;
5435 const int kHeight = 360;
Henrik Boström381d1092020-05-12 18:49:07 +02005436 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005437 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
kthelgason2bc68642017-02-07 07:02:22 -08005438
5439 // Set degradation preference.
mflodmancc3d4422017-08-03 08:27:51 -07005440 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005441 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason2bc68642017-02-07 07:02:22 -08005442
asaperssonfab67072017-04-04 05:51:49 -07005443 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 07:02:22 -08005444 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 07:06:52 -07005445 WaitForEncodedFrame(1);
kthelgason2bc68642017-02-07 07:02:22 -08005446
mflodmancc3d4422017-08-03 08:27:51 -07005447 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08005448}
5449
mflodmancc3d4422017-08-03 08:27:51 -07005450TEST_F(VideoStreamEncoderTest, InitialFrameDropOffWhenEncoderDisabledScaling) {
asaperssonfab67072017-04-04 05:51:49 -07005451 const int kWidth = 640;
5452 const int kHeight = 360;
kthelgasonad9010c2017-02-14 00:46:51 -08005453 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 09:04:13 +02005454
5455 VideoEncoderConfig video_encoder_config;
5456 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5457 // Make format different, to force recreation of encoder.
5458 video_encoder_config.video_format.parameters["foo"] = "foo";
5459 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02005460 kMaxPayloadLength);
Henrik Boström381d1092020-05-12 18:49:07 +02005461 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005462 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07005463
kthelgasonb83797b2017-02-14 11:57:25 -08005464 // Force quality scaler reconfiguration by resetting the source.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005465 video_stream_encoder_->SetSource(&video_source_,
5466 webrtc::DegradationPreference::BALANCED);
kthelgasonad9010c2017-02-14 00:46:51 -08005467
asaperssonfab67072017-04-04 05:51:49 -07005468 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgasonad9010c2017-02-14 00:46:51 -08005469 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 07:06:52 -07005470 WaitForEncodedFrame(1);
kthelgasonad9010c2017-02-14 00:46:51 -08005471
mflodmancc3d4422017-08-03 08:27:51 -07005472 video_stream_encoder_->Stop();
kthelgasonad9010c2017-02-14 00:46:51 -08005473 fake_encoder_.SetQualityScaling(true);
5474}
5475
Åsa Persson139f4dc2019-08-02 09:29:58 +02005476TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenBweDrops) {
5477 webrtc::test::ScopedFieldTrials field_trials(
5478 "WebRTC-Video-QualityScalerSettings/"
5479 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
5480 // Reset encoder for field trials to take effect.
5481 ConfigureEncoder(video_encoder_config_.Copy());
Asa Persson606d3cb2021-10-04 10:07:11 +02005482 const int kNotTooLowBitrateForFrameSizeBps = kTargetBitrate.bps() * 0.2;
5483 const int kTooLowBitrateForFrameSizeBps = kTargetBitrate.bps() * 0.19;
Åsa Persson139f4dc2019-08-02 09:29:58 +02005484 const int kWidth = 640;
5485 const int kHeight = 360;
5486
Henrik Boström381d1092020-05-12 18:49:07 +02005487 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005488 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005489 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5490 // Frame should not be dropped.
5491 WaitForEncodedFrame(1);
5492
Henrik Boström381d1092020-05-12 18:49:07 +02005493 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005494 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5495 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5496 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005497 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5498 // Frame should not be dropped.
5499 WaitForEncodedFrame(2);
5500
Henrik Boström381d1092020-05-12 18:49:07 +02005501 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005502 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5503 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5504 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005505 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5506 // Expect to drop this frame, the wait should time out.
5507 ExpectDroppedFrame();
5508
5509 // Expect the sink_wants to specify a scaled frame.
Henrik Boström2671dac2020-05-19 16:29:09 +02005510 EXPECT_TRUE_WAIT(
5511 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005512 video_stream_encoder_->Stop();
5513}
5514
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02005515TEST_F(VideoStreamEncoderTest,
5516 InitialFrameDropNotReactivatedWhenBweDropsWhenScalingDisabled) {
5517 webrtc::test::ScopedFieldTrials field_trials(
5518 "WebRTC-Video-QualityScalerSettings/"
5519 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
5520 fake_encoder_.SetQualityScaling(false);
5521 ConfigureEncoder(video_encoder_config_.Copy());
Asa Persson606d3cb2021-10-04 10:07:11 +02005522 const int kNotTooLowBitrateForFrameSizeBps = kTargetBitrate.bps() * 0.2;
5523 const int kTooLowBitrateForFrameSizeBps = kTargetBitrate.bps() * 0.19;
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02005524 const int kWidth = 640;
5525 const int kHeight = 360;
5526
5527 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005528 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02005529 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5530 // Frame should not be dropped.
5531 WaitForEncodedFrame(1);
5532
5533 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5534 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5535 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5536 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps), 0, 0, 0);
5537 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5538 // Frame should not be dropped.
5539 WaitForEncodedFrame(2);
5540
5541 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5542 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5543 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5544 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
5545 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5546 // Not dropped since quality scaling is disabled.
5547 WaitForEncodedFrame(3);
5548
5549 // Expect the sink_wants to specify a scaled frame.
Evan Shrubsole85728412020-08-25 13:12:12 +02005550 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02005551 EXPECT_THAT(video_source_.sink_wants(), ResolutionMax());
5552
5553 video_stream_encoder_->Stop();
5554}
5555
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005556TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenLayersChange) {
Asa Persson606d3cb2021-10-04 10:07:11 +02005557 const DataRate kLowTargetBitrate = DataRate::KilobitsPerSec(400);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005558 // Set simulcast.
5559 ResetEncoder("VP8", 3, 1, 1, false);
5560 fake_encoder_.SetQualityScaling(true);
5561 const int kWidth = 1280;
5562 const int kHeight = 720;
5563 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005564 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005565 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5566 // Frame should not be dropped.
5567 WaitForEncodedFrame(1);
5568
5569 // Trigger QVGA "singlecast"
5570 // Update the config.
5571 VideoEncoderConfig video_encoder_config;
5572 test::FillEncoderConfiguration(PayloadStringToCodecType("VP8"), 3,
5573 &video_encoder_config);
Åsa Persson7f354f82021-02-04 15:52:15 +01005574 video_encoder_config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005575 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Persson7f354f82021-02-04 15:52:15 +01005576 "VP8", /*max qp*/ 56, /*screencast*/ false,
5577 /*screenshare enabled*/ false);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005578 for (auto& layer : video_encoder_config.simulcast_layers) {
5579 layer.num_temporal_layers = 1;
5580 layer.max_framerate = kDefaultFramerate;
5581 }
Asa Persson606d3cb2021-10-04 10:07:11 +02005582 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005583 video_encoder_config.content_type =
5584 VideoEncoderConfig::ContentType::kRealtimeVideo;
5585
5586 video_encoder_config.simulcast_layers[0].active = true;
5587 video_encoder_config.simulcast_layers[1].active = false;
5588 video_encoder_config.simulcast_layers[2].active = false;
5589
5590 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5591 kMaxPayloadLength);
5592 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5593
5594 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5595 // Frame should not be dropped.
5596 WaitForEncodedFrame(2);
5597
5598 // Trigger HD "singlecast"
5599 video_encoder_config.simulcast_layers[0].active = false;
5600 video_encoder_config.simulcast_layers[1].active = false;
5601 video_encoder_config.simulcast_layers[2].active = true;
5602
5603 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5604 kMaxPayloadLength);
5605 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5606
5607 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5608 // Frame should be dropped because of initial frame drop.
5609 ExpectDroppedFrame();
5610
5611 // Expect the sink_wants to specify a scaled frame.
5612 EXPECT_TRUE_WAIT(
5613 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
5614 video_stream_encoder_->Stop();
5615}
5616
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005617TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenSVCLayersChange) {
Asa Persson606d3cb2021-10-04 10:07:11 +02005618 const DataRate kLowTargetBitrate = DataRate::KilobitsPerSec(400);
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005619 // Set simulcast.
5620 ResetEncoder("VP9", 1, 1, 3, false);
5621 fake_encoder_.SetQualityScaling(true);
5622 const int kWidth = 1280;
5623 const int kHeight = 720;
5624 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005625 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005626 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5627 // Frame should not be dropped.
5628 WaitForEncodedFrame(1);
5629
5630 // Trigger QVGA "singlecast"
5631 // Update the config.
5632 VideoEncoderConfig video_encoder_config;
5633 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5634 &video_encoder_config);
5635 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5636 vp9_settings.numberOfSpatialLayers = 3;
5637 // Since only one layer is active - automatic resize should be enabled.
5638 vp9_settings.automaticResizeOn = true;
5639 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005640 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005641 vp9_settings);
Asa Persson606d3cb2021-10-04 10:07:11 +02005642 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005643 video_encoder_config.content_type =
5644 VideoEncoderConfig::ContentType::kRealtimeVideo;
Artem Titovab30d722021-07-27 16:22:11 +02005645 // Currently simulcast layers `active` flags are used to inidicate
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005646 // which SVC layers are active.
5647 video_encoder_config.simulcast_layers.resize(3);
5648
5649 video_encoder_config.simulcast_layers[0].active = true;
5650 video_encoder_config.simulcast_layers[1].active = false;
5651 video_encoder_config.simulcast_layers[2].active = false;
5652
5653 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5654 kMaxPayloadLength);
5655 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5656
5657 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5658 // Frame should not be dropped.
5659 WaitForEncodedFrame(2);
5660
5661 // Trigger HD "singlecast"
5662 video_encoder_config.simulcast_layers[0].active = false;
5663 video_encoder_config.simulcast_layers[1].active = false;
5664 video_encoder_config.simulcast_layers[2].active = true;
5665
5666 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5667 kMaxPayloadLength);
5668 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5669
5670 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5671 // Frame should be dropped because of initial frame drop.
5672 ExpectDroppedFrame();
5673
5674 // Expect the sink_wants to specify a scaled frame.
5675 EXPECT_TRUE_WAIT(
5676 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
5677 video_stream_encoder_->Stop();
5678}
5679
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005680TEST_F(VideoStreamEncoderTest,
Åsa Perssonc91c4232021-02-01 09:20:05 +01005681 EncoderMaxAndMinBitratesUsedIfMiddleStreamActive) {
5682 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
5683 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
5684 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
5685 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
5686 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
5687 1280 * 720, 54 * 1000, 31 * 1000, 2500 * 1000);
5688 fake_encoder_.SetResolutionBitrateLimits(
5689 {kEncoderLimits270p, kEncoderLimits360p, kEncoderLimits720p});
5690
5691 VideoEncoderConfig video_encoder_config;
5692 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5693 &video_encoder_config);
5694 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5695 vp9_settings.numberOfSpatialLayers = 3;
5696 // Since only one layer is active - automatic resize should be enabled.
5697 vp9_settings.automaticResizeOn = true;
5698 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005699 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Åsa Perssonc91c4232021-02-01 09:20:05 +01005700 vp9_settings);
Asa Persson606d3cb2021-10-04 10:07:11 +02005701 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Åsa Perssonc91c4232021-02-01 09:20:05 +01005702 video_encoder_config.content_type =
5703 VideoEncoderConfig::ContentType::kRealtimeVideo;
5704 // Simulcast layers are used to indicate which spatial layers are active.
5705 video_encoder_config.simulcast_layers.resize(3);
5706 video_encoder_config.simulcast_layers[0].active = false;
5707 video_encoder_config.simulcast_layers[1].active = true;
5708 video_encoder_config.simulcast_layers[2].active = false;
5709
5710 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5711 kMaxPayloadLength);
5712 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5713
5714 // The encoder bitrate limits for 360p should be used.
5715 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5716 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02005717 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5718 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5719 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
5720 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5721 EXPECT_EQ(640, fake_encoder_.config().spatialLayers[0].width);
5722 EXPECT_EQ(360, fake_encoder_.config().spatialLayers[0].height);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005723 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005724 fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005725 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005726 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005727
5728 // The encoder bitrate limits for 270p should be used.
5729 video_source_.IncomingCapturedFrame(CreateFrame(2, 960, 540));
5730 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02005731 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5732 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5733 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
5734 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5735 EXPECT_EQ(480, fake_encoder_.config().spatialLayers[0].width);
5736 EXPECT_EQ(270, fake_encoder_.config().spatialLayers[0].height);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005737 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005738 fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005739 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005740 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005741
5742 video_stream_encoder_->Stop();
5743}
5744
5745TEST_F(VideoStreamEncoderTest,
Åsa Persson258e9892021-02-25 10:39:51 +01005746 DefaultMaxAndMinBitratesUsedIfMiddleStreamActive) {
5747 VideoEncoderConfig video_encoder_config;
5748 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5749 &video_encoder_config);
5750 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5751 vp9_settings.numberOfSpatialLayers = 3;
5752 // Since only one layer is active - automatic resize should be enabled.
5753 vp9_settings.automaticResizeOn = true;
5754 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005755 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Åsa Persson258e9892021-02-25 10:39:51 +01005756 vp9_settings);
Asa Persson606d3cb2021-10-04 10:07:11 +02005757 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Åsa Persson258e9892021-02-25 10:39:51 +01005758 video_encoder_config.content_type =
5759 VideoEncoderConfig::ContentType::kRealtimeVideo;
5760 // Simulcast layers are used to indicate which spatial layers are active.
5761 video_encoder_config.simulcast_layers.resize(3);
5762 video_encoder_config.simulcast_layers[0].active = false;
5763 video_encoder_config.simulcast_layers[1].active = true;
5764 video_encoder_config.simulcast_layers[2].active = false;
5765
5766 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5767 kMaxPayloadLength);
5768 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5769
5770 // The default bitrate limits for 360p should be used.
5771 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits360p =
Sergey Silkina86b29b2021-03-05 13:29:19 +01005772 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5773 kVideoCodecVP9, 640 * 360);
Åsa Persson258e9892021-02-25 10:39:51 +01005774 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5775 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02005776 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5777 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5778 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
5779 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5780 EXPECT_EQ(640, fake_encoder_.config().spatialLayers[0].width);
5781 EXPECT_EQ(360, fake_encoder_.config().spatialLayers[0].height);
Åsa Persson258e9892021-02-25 10:39:51 +01005782 EXPECT_EQ(static_cast<uint32_t>(kLimits360p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005783 fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005784 EXPECT_EQ(static_cast<uint32_t>(kLimits360p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005785 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005786
5787 // The default bitrate limits for 270p should be used.
5788 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits270p =
Sergey Silkina86b29b2021-03-05 13:29:19 +01005789 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5790 kVideoCodecVP9, 480 * 270);
Åsa Persson258e9892021-02-25 10:39:51 +01005791 video_source_.IncomingCapturedFrame(CreateFrame(2, 960, 540));
5792 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02005793 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5794 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5795 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
5796 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5797 EXPECT_EQ(480, fake_encoder_.config().spatialLayers[0].width);
5798 EXPECT_EQ(270, fake_encoder_.config().spatialLayers[0].height);
Åsa Persson258e9892021-02-25 10:39:51 +01005799 EXPECT_EQ(static_cast<uint32_t>(kLimits270p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005800 fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005801 EXPECT_EQ(static_cast<uint32_t>(kLimits270p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005802 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005803
5804 video_stream_encoder_->Stop();
5805}
5806
5807TEST_F(VideoStreamEncoderTest, DefaultMaxAndMinBitratesNotUsedIfDisabled) {
5808 webrtc::test::ScopedFieldTrials field_trials(
5809 "WebRTC-DefaultBitrateLimitsKillSwitch/Enabled/");
5810 VideoEncoderConfig video_encoder_config;
5811 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5812 &video_encoder_config);
5813 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5814 vp9_settings.numberOfSpatialLayers = 3;
5815 // Since only one layer is active - automatic resize should be enabled.
5816 vp9_settings.automaticResizeOn = true;
5817 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005818 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Åsa Persson258e9892021-02-25 10:39:51 +01005819 vp9_settings);
Asa Persson606d3cb2021-10-04 10:07:11 +02005820 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Åsa Persson258e9892021-02-25 10:39:51 +01005821 video_encoder_config.content_type =
5822 VideoEncoderConfig::ContentType::kRealtimeVideo;
5823 // Simulcast layers are used to indicate which spatial layers are active.
5824 video_encoder_config.simulcast_layers.resize(3);
5825 video_encoder_config.simulcast_layers[0].active = false;
5826 video_encoder_config.simulcast_layers[1].active = true;
5827 video_encoder_config.simulcast_layers[2].active = false;
5828
5829 // Reset encoder for field trials to take effect.
5830 ConfigureEncoder(video_encoder_config.Copy());
5831
5832 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5833 kMaxPayloadLength);
5834 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5835
5836 // The default bitrate limits for 360p should not be used.
5837 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits360p =
Sergey Silkina86b29b2021-03-05 13:29:19 +01005838 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5839 kVideoCodecVP9, 640 * 360);
Åsa Persson258e9892021-02-25 10:39:51 +01005840 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5841 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02005842 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5843 EXPECT_EQ(fake_encoder_.config().codecType, kVideoCodecVP9);
5844 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
5845 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5846 EXPECT_EQ(640, fake_encoder_.config().spatialLayers[0].width);
5847 EXPECT_EQ(360, fake_encoder_.config().spatialLayers[0].height);
Åsa Persson258e9892021-02-25 10:39:51 +01005848 EXPECT_NE(static_cast<uint32_t>(kLimits360p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005849 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005850
5851 video_stream_encoder_->Stop();
5852}
5853
5854TEST_F(VideoStreamEncoderTest, SinglecastBitrateLimitsNotUsedForOneStream) {
5855 ResetEncoder("VP9", /*num_streams=*/1, /*num_temporal_layers=*/1,
5856 /*num_spatial_layers=*/1, /*screenshare=*/false);
5857
5858 // The default singlecast bitrate limits for 720p should not be used.
5859 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits720p =
Sergey Silkina86b29b2021-03-05 13:29:19 +01005860 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5861 kVideoCodecVP9, 1280 * 720);
Åsa Persson258e9892021-02-25 10:39:51 +01005862 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5863 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02005864 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5865 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5866 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 1);
5867 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5868 EXPECT_EQ(1280, fake_encoder_.config().spatialLayers[0].width);
5869 EXPECT_EQ(720, fake_encoder_.config().spatialLayers[0].height);
Åsa Persson258e9892021-02-25 10:39:51 +01005870 EXPECT_NE(static_cast<uint32_t>(kLimits720p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005871 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005872
5873 video_stream_encoder_->Stop();
5874}
5875
5876TEST_F(VideoStreamEncoderTest,
Åsa Perssonc91c4232021-02-01 09:20:05 +01005877 EncoderMaxAndMinBitratesNotUsedIfLowestStreamActive) {
5878 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits180p(
5879 320 * 180, 34 * 1000, 12 * 1000, 1234 * 1000);
5880 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
5881 1280 * 720, 54 * 1000, 31 * 1000, 2500 * 1000);
5882 fake_encoder_.SetResolutionBitrateLimits(
5883 {kEncoderLimits180p, kEncoderLimits720p});
5884
5885 VideoEncoderConfig video_encoder_config;
5886 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5887 &video_encoder_config);
5888 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5889 vp9_settings.numberOfSpatialLayers = 3;
5890 // Since only one layer is active - automatic resize should be enabled.
5891 vp9_settings.automaticResizeOn = true;
5892 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005893 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Åsa Perssonc91c4232021-02-01 09:20:05 +01005894 vp9_settings);
Asa Persson606d3cb2021-10-04 10:07:11 +02005895 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Åsa Perssonc91c4232021-02-01 09:20:05 +01005896 video_encoder_config.content_type =
5897 VideoEncoderConfig::ContentType::kRealtimeVideo;
5898 // Simulcast layers are used to indicate which spatial layers are active.
5899 video_encoder_config.simulcast_layers.resize(3);
5900 video_encoder_config.simulcast_layers[0].active = true;
5901 video_encoder_config.simulcast_layers[1].active = false;
5902 video_encoder_config.simulcast_layers[2].active = false;
5903
5904 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5905 kMaxPayloadLength);
5906 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5907
5908 // Limits not applied on lowest stream, limits for 180p should not be used.
5909 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5910 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02005911 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5912 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5913 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 3);
5914 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5915 EXPECT_EQ(320, fake_encoder_.config().spatialLayers[0].width);
5916 EXPECT_EQ(180, fake_encoder_.config().spatialLayers[0].height);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005917 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits180p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005918 fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005919 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits180p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005920 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005921
5922 video_stream_encoder_->Stop();
5923}
5924
5925TEST_F(VideoStreamEncoderTest,
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005926 InitialFrameDropActivatesWhenResolutionIncreases) {
5927 const int kWidth = 640;
5928 const int kHeight = 360;
5929
5930 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005931 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005932 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth / 2, kHeight / 2));
5933 // Frame should not be dropped.
5934 WaitForEncodedFrame(1);
5935
5936 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005937 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005938 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth / 2, kHeight / 2));
5939 // Frame should not be dropped, bitrate not too low for frame.
5940 WaitForEncodedFrame(2);
5941
5942 // Incoming resolution increases.
5943 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5944 // Expect to drop this frame, bitrate too low for frame.
5945 ExpectDroppedFrame();
5946
5947 // Expect the sink_wants to specify a scaled frame.
5948 EXPECT_TRUE_WAIT(
5949 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
5950 video_stream_encoder_->Stop();
5951}
5952
5953TEST_F(VideoStreamEncoderTest, InitialFrameDropIsNotReactivatedWhenAdaptingUp) {
5954 const int kWidth = 640;
5955 const int kHeight = 360;
5956 // So that quality scaling doesn't happen by itself.
5957 fake_encoder_.SetQp(kQpHigh);
5958
5959 AdaptingFrameForwarder source(&time_controller_);
5960 source.set_adaptation_enabled(true);
5961 video_stream_encoder_->SetSource(
5962 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
5963
5964 int timestamp = 1;
5965
5966 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005967 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005968 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
5969 WaitForEncodedFrame(timestamp);
5970 timestamp += 9000;
5971 // Long pause to disable all first BWE drop logic.
5972 AdvanceTime(TimeDelta::Millis(1000));
5973
5974 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005975 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005976 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
5977 // Not dropped frame, as initial frame drop is disabled by now.
5978 WaitForEncodedFrame(timestamp);
5979 timestamp += 9000;
5980 AdvanceTime(TimeDelta::Millis(100));
5981
5982 // Quality adaptation down.
5983 video_stream_encoder_->TriggerQualityLow();
5984
5985 // Adaptation has an effect.
5986 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < kWidth * kHeight,
5987 5000);
5988
5989 // Frame isn't dropped as initial frame dropper is disabled.
5990 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
5991 WaitForEncodedFrame(timestamp);
5992 timestamp += 9000;
5993 AdvanceTime(TimeDelta::Millis(100));
5994
5995 // Quality adaptation up.
5996 video_stream_encoder_->TriggerQualityHigh();
5997
5998 // Adaptation has an effect.
5999 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count > kWidth * kHeight,
6000 5000);
6001
6002 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
6003 // Frame should not be dropped, as initial framedropper is off.
6004 WaitForEncodedFrame(timestamp);
6005
6006 video_stream_encoder_->Stop();
6007}
6008
Åsa Persson7f354f82021-02-04 15:52:15 +01006009TEST_F(VideoStreamEncoderTest,
6010 FrameDroppedWhenResolutionIncreasesAndLinkAllocationIsLow) {
6011 const int kMinStartBps360p = 222000;
6012 fake_encoder_.SetResolutionBitrateLimits(
6013 {VideoEncoder::ResolutionBitrateLimits(320 * 180, 0, 30000, 400000),
6014 VideoEncoder::ResolutionBitrateLimits(640 * 360, kMinStartBps360p, 30000,
6015 800000)});
6016
6017 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
6018 DataRate::BitsPerSec(kMinStartBps360p - 1), // target_bitrate
6019 DataRate::BitsPerSec(kMinStartBps360p - 1), // stable_target_bitrate
6020 DataRate::BitsPerSec(kMinStartBps360p - 1), // link_allocation
6021 0, 0, 0);
6022 // Frame should not be dropped, bitrate not too low for frame.
6023 video_source_.IncomingCapturedFrame(CreateFrame(1, 320, 180));
6024 WaitForEncodedFrame(1);
6025
6026 // Incoming resolution increases, initial frame drop activates.
6027 // Frame should be dropped, link allocation too low for frame.
6028 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
6029 ExpectDroppedFrame();
6030
6031 // Expect sink_wants to specify a scaled frame.
6032 EXPECT_TRUE_WAIT(video_source_.sink_wants().max_pixel_count < 640 * 360,
6033 5000);
6034 video_stream_encoder_->Stop();
6035}
6036
6037TEST_F(VideoStreamEncoderTest,
6038 FrameNotDroppedWhenResolutionIncreasesAndLinkAllocationIsHigh) {
6039 const int kMinStartBps360p = 222000;
6040 fake_encoder_.SetResolutionBitrateLimits(
6041 {VideoEncoder::ResolutionBitrateLimits(320 * 180, 0, 30000, 400000),
6042 VideoEncoder::ResolutionBitrateLimits(640 * 360, kMinStartBps360p, 30000,
6043 800000)});
6044
6045 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
6046 DataRate::BitsPerSec(kMinStartBps360p - 1), // target_bitrate
6047 DataRate::BitsPerSec(kMinStartBps360p - 1), // stable_target_bitrate
6048 DataRate::BitsPerSec(kMinStartBps360p), // link_allocation
6049 0, 0, 0);
6050 // Frame should not be dropped, bitrate not too low for frame.
6051 video_source_.IncomingCapturedFrame(CreateFrame(1, 320, 180));
6052 WaitForEncodedFrame(1);
6053
6054 // Incoming resolution increases, initial frame drop activates.
6055 // Frame should be dropped, link allocation not too low for frame.
6056 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
6057 WaitForEncodedFrame(2);
6058
6059 video_stream_encoder_->Stop();
6060}
6061
Åsa Perssone644a032019-11-08 15:56:00 +01006062TEST_F(VideoStreamEncoderTest, RampsUpInQualityWhenBwIsHigh) {
6063 webrtc::test::ScopedFieldTrials field_trials(
Åsa Persson06defc42021-09-10 15:28:48 +02006064 "WebRTC-Video-QualityRampupSettings/"
6065 "min_pixels:921600,min_duration_ms:2000/");
6066
6067 const int kWidth = 1280;
6068 const int kHeight = 720;
6069 const int kFps = 10;
6070 max_framerate_ = kFps;
Åsa Perssone644a032019-11-08 15:56:00 +01006071
6072 // Reset encoder for field trials to take effect.
6073 VideoEncoderConfig config = video_encoder_config_.Copy();
Asa Persson606d3cb2021-10-04 10:07:11 +02006074 config.max_bitrate_bps = kTargetBitrate.bps();
Evan Shrubsoledff79252020-04-16 11:34:32 +02006075 DataRate max_bitrate = DataRate::BitsPerSec(config.max_bitrate_bps);
Åsa Perssone644a032019-11-08 15:56:00 +01006076 ConfigureEncoder(std::move(config));
6077 fake_encoder_.SetQp(kQpLow);
6078
6079 // Enable MAINTAIN_FRAMERATE preference.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006080 AdaptingFrameForwarder source(&time_controller_);
Åsa Perssone644a032019-11-08 15:56:00 +01006081 source.set_adaptation_enabled(true);
6082 video_stream_encoder_->SetSource(&source,
6083 DegradationPreference::MAINTAIN_FRAMERATE);
6084
6085 // Start at low bitrate.
Asa Persson606d3cb2021-10-04 10:07:11 +02006086 const DataRate kLowBitrate = DataRate::KilobitsPerSec(200);
Henrik Boström381d1092020-05-12 18:49:07 +02006087 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006088 kLowBitrate, kLowBitrate, kLowBitrate, 0, 0, 0);
Åsa Perssone644a032019-11-08 15:56:00 +01006089
6090 // Expect first frame to be dropped and resolution to be limited.
Åsa Persson06defc42021-09-10 15:28:48 +02006091 const int64_t kFrameIntervalMs = 1000 / kFps;
Åsa Perssone644a032019-11-08 15:56:00 +01006092 int64_t timestamp_ms = kFrameIntervalMs;
6093 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6094 ExpectDroppedFrame();
Henrik Boström2671dac2020-05-19 16:29:09 +02006095 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < kWidth * kHeight,
6096 5000);
Åsa Perssone644a032019-11-08 15:56:00 +01006097
6098 // Increase bitrate to encoder max.
Henrik Boström381d1092020-05-12 18:49:07 +02006099 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
6100 max_bitrate, max_bitrate, max_bitrate, 0, 0, 0);
Åsa Perssone644a032019-11-08 15:56:00 +01006101
Artem Titovab30d722021-07-27 16:22:11 +02006102 // Insert frames and advance `min_duration_ms`.
Åsa Persson06defc42021-09-10 15:28:48 +02006103 const int64_t start_bw_high_ms = CurrentTimeMs();
Åsa Perssone644a032019-11-08 15:56:00 +01006104 for (size_t i = 1; i <= 10; i++) {
6105 timestamp_ms += kFrameIntervalMs;
6106 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6107 WaitForEncodedFrame(timestamp_ms);
6108 }
Åsa Persson06defc42021-09-10 15:28:48 +02006109
6110 // Advance to `min_duration_ms` - 1, frame should not trigger high BW.
6111 int64_t elapsed_bw_high_ms = CurrentTimeMs() - start_bw_high_ms;
6112 AdvanceTime(TimeDelta::Millis(2000 - elapsed_bw_high_ms - 1));
6113 timestamp_ms += kFrameIntervalMs;
6114 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6115 WaitForEncodedFrame(timestamp_ms);
Åsa Perssone644a032019-11-08 15:56:00 +01006116 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6117 EXPECT_LT(source.sink_wants().max_pixel_count, kWidth * kHeight);
6118
Åsa Persson06defc42021-09-10 15:28:48 +02006119 // Frame should trigger high BW and release quality limitation.
Åsa Perssone644a032019-11-08 15:56:00 +01006120 timestamp_ms += kFrameIntervalMs;
6121 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6122 WaitForEncodedFrame(timestamp_ms);
Henrik Boström381d1092020-05-12 18:49:07 +02006123 // The ramp-up code involves the adaptation queue, give it time to execute.
6124 // TODO(hbos): Can we await an appropriate event instead?
Evan Shrubsole85728412020-08-25 13:12:12 +02006125 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006126 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
Åsa Perssone644a032019-11-08 15:56:00 +01006127
6128 // Frame should not be adapted.
6129 timestamp_ms += kFrameIntervalMs;
6130 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6131 WaitForEncodedFrame(kWidth, kHeight);
6132 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6133
6134 video_stream_encoder_->Stop();
6135}
6136
mflodmancc3d4422017-08-03 08:27:51 -07006137TEST_F(VideoStreamEncoderTest,
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02006138 QualityScalerAdaptationsRemovedWhenQualityScalingDisabled) {
Ilya Nikolaevskiy483b31c2021-02-03 17:19:31 +01006139 webrtc::test::ScopedFieldTrials field_trials(
6140 "WebRTC-Video-QualityScaling/Disabled/");
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006141 AdaptingFrameForwarder source(&time_controller_);
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02006142 source.set_adaptation_enabled(true);
6143 video_stream_encoder_->SetSource(&source,
6144 DegradationPreference::MAINTAIN_FRAMERATE);
6145 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006146 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02006147 fake_encoder_.SetQp(kQpHigh + 1);
6148 const int kWidth = 1280;
6149 const int kHeight = 720;
6150 const int64_t kFrameIntervalMs = 100;
6151 int64_t timestamp_ms = kFrameIntervalMs;
6152 for (size_t i = 1; i <= 100; i++) {
6153 timestamp_ms += kFrameIntervalMs;
6154 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6155 WaitForEncodedFrame(timestamp_ms);
6156 }
6157 // Wait for QualityScaler, which will wait for 2000*2.5 ms until checking QP
6158 // for the first time.
6159 // TODO(eshr): We should avoid these waits by using threads with simulated
6160 // time.
6161 EXPECT_TRUE_WAIT(stats_proxy_->GetStats().bw_limited_resolution,
6162 2000 * 2.5 * 2);
6163 timestamp_ms += kFrameIntervalMs;
6164 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6165 WaitForEncodedFrame(timestamp_ms);
6166 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
6167 EXPECT_THAT(source.sink_wants(), WantsMaxPixels(Lt(kWidth * kHeight)));
6168 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6169
6170 // Disable Quality scaling by turning off scaler on the encoder and
6171 // reconfiguring.
6172 fake_encoder_.SetQualityScaling(false);
6173 video_stream_encoder_->ConfigureEncoder(video_encoder_config_.Copy(),
6174 kMaxPayloadLength);
6175 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Markus Handell28c71802021-11-08 10:11:55 +01006176 AdvanceTime(TimeDelta::Zero());
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02006177 // Since we turned off the quality scaler, the adaptations made by it are
6178 // removed.
6179 EXPECT_THAT(source.sink_wants(), ResolutionMax());
6180 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6181
6182 video_stream_encoder_->Stop();
6183}
6184
6185TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07006186 ResolutionNotAdaptedForTooSmallFrame_MaintainFramerateMode) {
6187 const int kTooSmallWidth = 10;
6188 const int kTooSmallHeight = 10;
Henrik Boström381d1092020-05-12 18:49:07 +02006189 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006190 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07006191
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006192 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asaperssond0de2952017-04-21 01:47:31 -07006193 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07006194 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006195 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006196 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
asaperssond0de2952017-04-21 01:47:31 -07006197 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6198
6199 // Trigger adapt down, too small frame, expect no change.
6200 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07006201 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 08:27:51 -07006202 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006203 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07006204 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6205 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6206
mflodmancc3d4422017-08-03 08:27:51 -07006207 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07006208}
6209
mflodmancc3d4422017-08-03 08:27:51 -07006210TEST_F(VideoStreamEncoderTest,
6211 ResolutionNotAdaptedForTooSmallFrame_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07006212 const int kTooSmallWidth = 10;
6213 const int kTooSmallHeight = 10;
6214 const int kFpsLimit = 7;
Henrik Boström381d1092020-05-12 18:49:07 +02006215 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006216 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07006217
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006218 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07006219 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006220 video_stream_encoder_->SetSource(&source,
6221 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006222 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
asaperssonf7e294d2017-06-13 23:25:22 -07006223 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6224 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6225
6226 // Trigger adapt down, expect limited framerate.
6227 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07006228 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 08:27:51 -07006229 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006230 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07006231 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6232 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6233 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6234
6235 // Trigger adapt down, too small frame, expect no change.
6236 source.IncomingCapturedFrame(CreateFrame(2, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07006237 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07006238 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006239 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07006240 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6241 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6242 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6243
mflodmancc3d4422017-08-03 08:27:51 -07006244 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006245}
6246
mflodmancc3d4422017-08-03 08:27:51 -07006247TEST_F(VideoStreamEncoderTest, FailingInitEncodeDoesntCauseCrash) {
asapersson02465b82017-04-10 01:12:52 -07006248 fake_encoder_.ForceInitEncodeFailure(true);
Henrik Boström381d1092020-05-12 18:49:07 +02006249 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006250 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möllerf1338562018-04-26 09:51:47 +02006251 ResetEncoder("VP8", 2, 1, 1, false);
asapersson02465b82017-04-10 01:12:52 -07006252 const int kFrameWidth = 1280;
6253 const int kFrameHeight = 720;
6254 video_source_.IncomingCapturedFrame(
6255 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006256 ExpectDroppedFrame();
mflodmancc3d4422017-08-03 08:27:51 -07006257 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07006258}
6259
sprangb1ca0732017-02-01 08:38:12 -08006260// TODO(sprang): Extend this with fps throttling and any "balanced" extensions.
mflodmancc3d4422017-08-03 08:27:51 -07006261TEST_F(VideoStreamEncoderTest,
6262 AdaptsResolutionOnOveruse_MaintainFramerateMode) {
Henrik Boström381d1092020-05-12 18:49:07 +02006263 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006264 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprangb1ca0732017-02-01 08:38:12 -08006265
6266 const int kFrameWidth = 1280;
6267 const int kFrameHeight = 720;
6268 // Enabled default VideoAdapter downscaling. First step is 3/4, not 3/5 as
mflodmancc3d4422017-08-03 08:27:51 -07006269 // requested by
6270 // VideoStreamEncoder::VideoSourceProxy::RequestResolutionLowerThan().
sprangb1ca0732017-02-01 08:38:12 -08006271 video_source_.set_adaptation_enabled(true);
6272
6273 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02006274 CreateFrame(1 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006275 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 08:38:12 -08006276
6277 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 08:27:51 -07006278 video_stream_encoder_->TriggerCpuOveruse();
sprangb1ca0732017-02-01 08:38:12 -08006279 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02006280 CreateFrame(2 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006281 WaitForEncodedFrame((kFrameWidth * 3) / 4, (kFrameHeight * 3) / 4);
sprangb1ca0732017-02-01 08:38:12 -08006282
asaperssonfab67072017-04-04 05:51:49 -07006283 // Trigger CPU normal use, return to original resolution.
Henrik Boström91aa7322020-04-28 12:24:33 +02006284 video_stream_encoder_->TriggerCpuUnderuse();
sprangb1ca0732017-02-01 08:38:12 -08006285 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02006286 CreateFrame(3 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006287 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 08:38:12 -08006288
mflodmancc3d4422017-08-03 08:27:51 -07006289 video_stream_encoder_->Stop();
sprangb1ca0732017-02-01 08:38:12 -08006290}
sprangfe627f32017-03-29 08:24:59 -07006291
mflodmancc3d4422017-08-03 08:27:51 -07006292TEST_F(VideoStreamEncoderTest,
6293 AdaptsFramerateOnOveruse_MaintainResolutionMode) {
sprangc5d62e22017-04-02 23:53:04 -07006294 const int kFrameWidth = 1280;
6295 const int kFrameHeight = 720;
sprangc5d62e22017-04-02 23:53:04 -07006296
Henrik Boström381d1092020-05-12 18:49:07 +02006297 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006298 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07006299 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006300 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07006301 video_source_.set_adaptation_enabled(true);
6302
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006303 int64_t timestamp_ms = CurrentTimeMs();
sprangc5d62e22017-04-02 23:53:04 -07006304
6305 video_source_.IncomingCapturedFrame(
6306 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006307 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-02 23:53:04 -07006308
6309 // Try to trigger overuse. No fps estimate available => no effect.
mflodmancc3d4422017-08-03 08:27:51 -07006310 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07006311
6312 // Insert frames for one second to get a stable estimate.
sprang4847ae62017-06-27 07:06:52 -07006313 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006314 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006315 video_source_.IncomingCapturedFrame(
6316 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006317 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-02 23:53:04 -07006318 }
6319
6320 // Trigger CPU overuse, reduce framerate by 2/3.
mflodmancc3d4422017-08-03 08:27:51 -07006321 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07006322 int num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07006323 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006324 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006325 video_source_.IncomingCapturedFrame(
6326 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006327 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07006328 ++num_frames_dropped;
6329 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01006330 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07006331 }
6332 }
6333
sprang4847ae62017-06-27 07:06:52 -07006334 // Add some slack to account for frames dropped by the frame dropper.
6335 const int kErrorMargin = 1;
6336 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-02 23:53:04 -07006337 kErrorMargin);
6338
6339 // Trigger CPU overuse, reduce framerate by 2/3 again.
mflodmancc3d4422017-08-03 08:27:51 -07006340 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07006341 num_frames_dropped = 0;
Åsa Persson8c1bf952018-09-13 10:42:19 +02006342 for (int i = 0; i <= max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006343 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006344 video_source_.IncomingCapturedFrame(
6345 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006346 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07006347 ++num_frames_dropped;
6348 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01006349 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07006350 }
6351 }
sprang4847ae62017-06-27 07:06:52 -07006352 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 4 / 9),
sprangc5d62e22017-04-02 23:53:04 -07006353 kErrorMargin);
6354
6355 // Go back up one step.
Henrik Boström91aa7322020-04-28 12:24:33 +02006356 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07006357 num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07006358 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006359 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006360 video_source_.IncomingCapturedFrame(
6361 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006362 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07006363 ++num_frames_dropped;
6364 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01006365 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07006366 }
6367 }
sprang4847ae62017-06-27 07:06:52 -07006368 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-02 23:53:04 -07006369 kErrorMargin);
6370
6371 // Go back up to original mode.
Henrik Boström91aa7322020-04-28 12:24:33 +02006372 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07006373 num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07006374 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006375 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006376 video_source_.IncomingCapturedFrame(
6377 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006378 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07006379 ++num_frames_dropped;
6380 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01006381 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07006382 }
6383 }
6384 EXPECT_NEAR(num_frames_dropped, 0, kErrorMargin);
6385
mflodmancc3d4422017-08-03 08:27:51 -07006386 video_stream_encoder_->Stop();
sprangc5d62e22017-04-02 23:53:04 -07006387}
6388
mflodmancc3d4422017-08-03 08:27:51 -07006389TEST_F(VideoStreamEncoderTest, DoesntAdaptDownPastMinFramerate) {
sprangc5d62e22017-04-02 23:53:04 -07006390 const int kFramerateFps = 5;
6391 const int kFrameIntervalMs = rtc::kNumMillisecsPerSec / kFramerateFps;
sprangc5d62e22017-04-02 23:53:04 -07006392 const int kFrameWidth = 1280;
6393 const int kFrameHeight = 720;
6394
sprang4847ae62017-06-27 07:06:52 -07006395 // Reconfigure encoder with two temporal layers and screensharing, which will
6396 // disable frame dropping and make testing easier.
Niels Möllerf1338562018-04-26 09:51:47 +02006397 ResetEncoder("VP8", 1, 2, 1, true);
sprang4847ae62017-06-27 07:06:52 -07006398
Henrik Boström381d1092020-05-12 18:49:07 +02006399 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006400 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07006401 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006402 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07006403 video_source_.set_adaptation_enabled(true);
6404
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006405 int64_t timestamp_ms = CurrentTimeMs();
sprangc5d62e22017-04-02 23:53:04 -07006406
6407 // Trigger overuse as much as we can.
Jonathan Yubc771b72017-12-08 17:04:29 -08006408 rtc::VideoSinkWants last_wants;
6409 do {
6410 last_wants = video_source_.sink_wants();
6411
sprangc5d62e22017-04-02 23:53:04 -07006412 // Insert frames to get a new fps estimate...
6413 for (int j = 0; j < kFramerateFps; ++j) {
6414 video_source_.IncomingCapturedFrame(
6415 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
Jonathan Yubc771b72017-12-08 17:04:29 -08006416 if (video_source_.last_sent_width()) {
6417 sink_.WaitForEncodedFrame(timestamp_ms);
6418 }
sprangc5d62e22017-04-02 23:53:04 -07006419 timestamp_ms += kFrameIntervalMs;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006420 AdvanceTime(TimeDelta::Millis(kFrameIntervalMs));
sprangc5d62e22017-04-02 23:53:04 -07006421 }
6422 // ...and then try to adapt again.
mflodmancc3d4422017-08-03 08:27:51 -07006423 video_stream_encoder_->TriggerCpuOveruse();
Jonathan Yubc771b72017-12-08 17:04:29 -08006424 } while (video_source_.sink_wants().max_framerate_fps <
6425 last_wants.max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07006426
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006427 EXPECT_THAT(video_source_.sink_wants(),
6428 FpsMatchesResolutionMax(Eq(kMinFramerateFps)));
asaperssonf7e294d2017-06-13 23:25:22 -07006429
mflodmancc3d4422017-08-03 08:27:51 -07006430 video_stream_encoder_->Stop();
sprangc5d62e22017-04-02 23:53:04 -07006431}
asaperssonf7e294d2017-06-13 23:25:22 -07006432
mflodmancc3d4422017-08-03 08:27:51 -07006433TEST_F(VideoStreamEncoderTest,
6434 AdaptsResolutionAndFramerateForLowQuality_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07006435 const int kWidth = 1280;
6436 const int kHeight = 720;
6437 const int64_t kFrameIntervalMs = 150;
6438 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02006439 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006440 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07006441
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006442 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006443 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07006444 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006445 video_stream_encoder_->SetSource(&source,
6446 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07006447 timestamp_ms += kFrameIntervalMs;
6448 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006449 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006450 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006451 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6452 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6453 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6454
6455 // Trigger adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006456 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006457 timestamp_ms += kFrameIntervalMs;
6458 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006459 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006460 EXPECT_THAT(source.sink_wants(),
6461 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07006462 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6463 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6464 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6465
6466 // Trigger adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006467 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006468 timestamp_ms += kFrameIntervalMs;
6469 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006470 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006471 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006472 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6473 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6474 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6475
6476 // Trigger adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006477 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006478 timestamp_ms += kFrameIntervalMs;
6479 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006480 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006481 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006482 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6483 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6484 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6485
6486 // Trigger adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006487 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006488 timestamp_ms += kFrameIntervalMs;
6489 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006490 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006491 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006492 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6493 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6494 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6495
6496 // Restrict bitrate, trigger adapt down, expect reduced fps (480x270@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07006497 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006498 timestamp_ms += kFrameIntervalMs;
6499 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006500 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006501 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006502 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6503 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6504 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6505
6506 // Trigger adapt down, expect scaled down resolution (320x180@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07006507 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006508 timestamp_ms += kFrameIntervalMs;
6509 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006510 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006511 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006512 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6513 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6514 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6515
6516 // Trigger adapt down, expect reduced fps (320x180@7fps).
mflodmancc3d4422017-08-03 08:27:51 -07006517 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006518 timestamp_ms += kFrameIntervalMs;
6519 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006520 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006521 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006522 rtc::VideoSinkWants last_wants = source.sink_wants();
6523 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6524 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6525 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6526
6527 // Trigger adapt down, min resolution reached, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006528 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006529 timestamp_ms += kFrameIntervalMs;
6530 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006531 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006532 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(last_wants));
asaperssonf7e294d2017-06-13 23:25:22 -07006533 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6534 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6535 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6536
Åsa Perssonf5f7e8e2021-06-09 22:55:38 +02006537 // Trigger adapt up, expect increased fps (320x180@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07006538 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006539 timestamp_ms += kFrameIntervalMs;
6540 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006541 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006542 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006543 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6544 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6545 EXPECT_EQ(8, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6546
6547 // Trigger adapt up, expect upscaled resolution (480x270@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07006548 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006549 timestamp_ms += kFrameIntervalMs;
6550 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006551 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006552 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006553 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6554 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6555 EXPECT_EQ(9, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6556
6557 // Increase bitrate, trigger adapt up, expect increased fps (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006558 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006559 timestamp_ms += kFrameIntervalMs;
6560 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006561 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006562 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006563 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6564 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6565 EXPECT_EQ(10, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6566
6567 // Trigger adapt up, expect upscaled resolution (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006568 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006569 timestamp_ms += kFrameIntervalMs;
6570 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006571 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006572 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006573 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6574 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6575 EXPECT_EQ(11, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6576
6577 // Trigger adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006578 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006579 timestamp_ms += kFrameIntervalMs;
6580 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006581 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006582 EXPECT_THAT(source.sink_wants(), FpsMax());
6583 EXPECT_EQ(source.sink_wants().max_pixel_count,
6584 source.last_wants().max_pixel_count);
asaperssonf7e294d2017-06-13 23:25:22 -07006585 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6586 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6587 EXPECT_EQ(12, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6588
6589 // Trigger adapt up, expect upscaled resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006590 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006591 timestamp_ms += kFrameIntervalMs;
6592 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006593 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006594 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006595 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6596 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6597 EXPECT_EQ(13, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6598
Åsa Persson30ab0152019-08-27 12:22:33 +02006599 // Trigger adapt up, expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006600 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006601 timestamp_ms += kFrameIntervalMs;
6602 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006603 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006604 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006605 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006606 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6607 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6608 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6609
6610 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006611 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006612 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006613 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6614
mflodmancc3d4422017-08-03 08:27:51 -07006615 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006616}
6617
mflodmancc3d4422017-08-03 08:27:51 -07006618TEST_F(VideoStreamEncoderTest, AdaptWithTwoReasonsAndDifferentOrder_Framerate) {
asaperssonf7e294d2017-06-13 23:25:22 -07006619 const int kWidth = 1280;
6620 const int kHeight = 720;
6621 const int64_t kFrameIntervalMs = 150;
6622 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02006623 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006624 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07006625
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006626 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006627 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07006628 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006629 video_stream_encoder_->SetSource(&source,
6630 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07006631 timestamp_ms += kFrameIntervalMs;
6632 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006633 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006634 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006635 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6636 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6637 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6638 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6639 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6640 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6641
6642 // Trigger cpu adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006643 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07006644 timestamp_ms += kFrameIntervalMs;
6645 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006646 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006647 EXPECT_THAT(source.sink_wants(),
6648 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07006649 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6650 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6651 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6652 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6653 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6654 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6655
6656 // Trigger cpu adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006657 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07006658 timestamp_ms += kFrameIntervalMs;
6659 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006660 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006661 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006662 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6663 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6664 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6665 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6666 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6667 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6668
6669 // Trigger quality adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006670 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006671 timestamp_ms += kFrameIntervalMs;
6672 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006673 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006674 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
Evan Shrubsole64469032020-06-11 10:45:29 +02006675 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
asaperssonf7e294d2017-06-13 23:25:22 -07006676 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6677 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6678 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6679 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6680 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6681
Evan Shrubsole64469032020-06-11 10:45:29 +02006682 // Trigger cpu adapt up, expect no change since QP is most limited.
6683 {
6684 // Store current sink wants since we expect no change and if there is no
6685 // change then last_wants() is not updated.
6686 auto previous_sink_wants = source.sink_wants();
6687 video_stream_encoder_->TriggerCpuUnderuse();
6688 timestamp_ms += kFrameIntervalMs;
6689 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6690 WaitForEncodedFrame(timestamp_ms);
6691 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(previous_sink_wants));
6692 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6693 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6694 }
6695
6696 // Trigger quality adapt up, expect increased fps (640x360@30fps).
6697 video_stream_encoder_->TriggerQualityHigh();
6698 timestamp_ms += kFrameIntervalMs;
6699 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6700 WaitForEncodedFrame(timestamp_ms);
6701 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
6702 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6703 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6704 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6705 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6706 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6707 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6708
6709 // Trigger quality adapt up and Cpu adapt up since both are most limited,
6710 // expect increased resolution (960x540@30fps).
6711 video_stream_encoder_->TriggerQualityHigh();
Henrik Boström91aa7322020-04-28 12:24:33 +02006712 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07006713 timestamp_ms += kFrameIntervalMs;
6714 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006715 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole64469032020-06-11 10:45:29 +02006716 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006717 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6718 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6719 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6720 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6721 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006722 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006723
Evan Shrubsole64469032020-06-11 10:45:29 +02006724 // Trigger quality adapt up and Cpu adapt up since both are most limited,
6725 // expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006726 video_stream_encoder_->TriggerQualityHigh();
Henrik Boström91aa7322020-04-28 12:24:33 +02006727 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07006728 timestamp_ms += kFrameIntervalMs;
6729 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006730 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006731 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006732 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006733 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6734 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6735 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6736 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6737 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006738 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006739
6740 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006741 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006742 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006743 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006744 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006745
mflodmancc3d4422017-08-03 08:27:51 -07006746 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006747}
6748
mflodmancc3d4422017-08-03 08:27:51 -07006749TEST_F(VideoStreamEncoderTest,
6750 AdaptWithTwoReasonsAndDifferentOrder_Resolution) {
asaperssonf7e294d2017-06-13 23:25:22 -07006751 const int kWidth = 640;
6752 const int kHeight = 360;
6753 const int kFpsLimit = 15;
6754 const int64_t kFrameIntervalMs = 150;
6755 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02006756 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006757 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07006758
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006759 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006760 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07006761 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006762 video_stream_encoder_->SetSource(&source,
6763 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07006764 timestamp_ms += kFrameIntervalMs;
6765 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006766 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006767 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006768 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6769 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6770 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6771 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6772 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6773 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6774
6775 // Trigger cpu adapt down, expect scaled down framerate (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006776 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07006777 timestamp_ms += kFrameIntervalMs;
6778 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006779 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006780 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07006781 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6782 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6783 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6784 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
6785 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6786 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6787
6788 // Trigger quality adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006789 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006790 timestamp_ms += kFrameIntervalMs;
6791 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006792 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006793 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006794 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Evan Shrubsole64469032020-06-11 10:45:29 +02006795 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
asaperssonf7e294d2017-06-13 23:25:22 -07006796 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6797 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
6798 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6799 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6800
Evan Shrubsole64469032020-06-11 10:45:29 +02006801 // Trigger cpu adapt up, expect no change because quality is most limited.
6802 {
6803 auto previous_sink_wants = source.sink_wants();
6804 // Store current sink wants since we expect no change ind if there is no
6805 // change then last__wants() is not updated.
6806 video_stream_encoder_->TriggerCpuUnderuse();
6807 timestamp_ms += kFrameIntervalMs;
6808 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6809 WaitForEncodedFrame(timestamp_ms);
6810 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(previous_sink_wants));
6811 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6812 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6813 }
6814
6815 // Trigger quality adapt up, expect upscaled resolution (640x360@15fps).
6816 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006817 timestamp_ms += kFrameIntervalMs;
6818 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006819 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006820 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006821 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6822 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6823 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Evan Shrubsole64469032020-06-11 10:45:29 +02006824 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
6825 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6826 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006827
Evan Shrubsole64469032020-06-11 10:45:29 +02006828 // Trigger quality and cpu adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006829 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole64469032020-06-11 10:45:29 +02006830 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07006831 timestamp_ms += kFrameIntervalMs;
6832 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006833 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006834 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006835 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6836 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6837 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6838 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6839 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006840 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006841
6842 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006843 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006844 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006845 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006846 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006847
mflodmancc3d4422017-08-03 08:27:51 -07006848 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006849}
6850
mflodmancc3d4422017-08-03 08:27:51 -07006851TEST_F(VideoStreamEncoderTest, AcceptsFullHdAdaptedDownSimulcastFrames) {
ilnik6b826ef2017-06-16 06:53:48 -07006852 const int kFrameWidth = 1920;
6853 const int kFrameHeight = 1080;
6854 // 3/4 of 1920.
6855 const int kAdaptedFrameWidth = 1440;
6856 // 3/4 of 1080 rounded down to multiple of 4.
6857 const int kAdaptedFrameHeight = 808;
6858 const int kFramerate = 24;
6859
Henrik Boström381d1092020-05-12 18:49:07 +02006860 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006861 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
ilnik6b826ef2017-06-16 06:53:48 -07006862 // Trigger reconfigure encoder (without resetting the entire instance).
6863 VideoEncoderConfig video_encoder_config;
Åsa Persson17b29b92020-10-17 12:57:58 +02006864 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
6865 video_encoder_config.simulcast_layers[0].max_framerate = kFramerate;
Asa Persson606d3cb2021-10-04 10:07:11 +02006866 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
ilnik6b826ef2017-06-16 06:53:48 -07006867 video_encoder_config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02006868 rtc::make_ref_counted<CroppingVideoStreamFactory>();
mflodmancc3d4422017-08-03 08:27:51 -07006869 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02006870 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -07006871 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
ilnik6b826ef2017-06-16 06:53:48 -07006872
6873 video_source_.set_adaptation_enabled(true);
6874
6875 video_source_.IncomingCapturedFrame(
6876 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006877 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
ilnik6b826ef2017-06-16 06:53:48 -07006878
6879 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 08:27:51 -07006880 video_stream_encoder_->TriggerCpuOveruse();
ilnik6b826ef2017-06-16 06:53:48 -07006881 video_source_.IncomingCapturedFrame(
6882 CreateFrame(2, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006883 WaitForEncodedFrame(kAdaptedFrameWidth, kAdaptedFrameHeight);
ilnik6b826ef2017-06-16 06:53:48 -07006884
mflodmancc3d4422017-08-03 08:27:51 -07006885 video_stream_encoder_->Stop();
ilnik6b826ef2017-06-16 06:53:48 -07006886}
6887
mflodmancc3d4422017-08-03 08:27:51 -07006888TEST_F(VideoStreamEncoderTest, PeriodicallyUpdatesChannelParameters) {
sprang4847ae62017-06-27 07:06:52 -07006889 const int kFrameWidth = 1280;
6890 const int kFrameHeight = 720;
6891 const int kLowFps = 2;
6892 const int kHighFps = 30;
6893
Henrik Boström381d1092020-05-12 18:49:07 +02006894 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006895 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07006896
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006897 int64_t timestamp_ms = CurrentTimeMs();
sprang4847ae62017-06-27 07:06:52 -07006898 max_framerate_ = kLowFps;
6899
6900 // Insert 2 seconds of 2fps video.
6901 for (int i = 0; i < kLowFps * 2; ++i) {
6902 video_source_.IncomingCapturedFrame(
6903 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6904 WaitForEncodedFrame(timestamp_ms);
6905 timestamp_ms += 1000 / kLowFps;
6906 }
6907
6908 // Make sure encoder is updated with new target.
Henrik Boström381d1092020-05-12 18:49:07 +02006909 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006910 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07006911 video_source_.IncomingCapturedFrame(
6912 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6913 WaitForEncodedFrame(timestamp_ms);
6914 timestamp_ms += 1000 / kLowFps;
6915
6916 EXPECT_EQ(kLowFps, fake_encoder_.GetConfiguredInputFramerate());
6917
6918 // Insert 30fps frames for just a little more than the forced update period.
Niels Möllerfe407b72019-09-10 10:48:48 +02006919 const int kVcmTimerIntervalFrames = (kProcessIntervalMs * kHighFps) / 1000;
sprang4847ae62017-06-27 07:06:52 -07006920 const int kFrameIntervalMs = 1000 / kHighFps;
6921 max_framerate_ = kHighFps;
6922 for (int i = 0; i < kVcmTimerIntervalFrames + 2; ++i) {
6923 video_source_.IncomingCapturedFrame(
6924 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6925 // Wait for encoded frame, but skip ahead if it doesn't arrive as it might
6926 // be dropped if the encoder hans't been updated with the new higher target
6927 // framerate yet, causing it to overshoot the target bitrate and then
6928 // suffering the wrath of the media optimizer.
6929 TimedWaitForEncodedFrame(timestamp_ms, 2 * kFrameIntervalMs);
6930 timestamp_ms += kFrameIntervalMs;
6931 }
6932
6933 // Don expect correct measurement just yet, but it should be higher than
6934 // before.
6935 EXPECT_GT(fake_encoder_.GetConfiguredInputFramerate(), kLowFps);
6936
mflodmancc3d4422017-08-03 08:27:51 -07006937 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 07:06:52 -07006938}
6939
mflodmancc3d4422017-08-03 08:27:51 -07006940TEST_F(VideoStreamEncoderTest, DoesNotUpdateBitrateAllocationWhenSuspended) {
sprang4847ae62017-06-27 07:06:52 -07006941 const int kFrameWidth = 1280;
6942 const int kFrameHeight = 720;
Per Kjellanderdcef6412020-10-07 15:09:05 +02006943 ResetEncoder("FAKE", 1, 1, 1, false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01006944 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02006945 kVideoBitrateAllocation);
sprang4847ae62017-06-27 07:06:52 -07006946
Henrik Boström381d1092020-05-12 18:49:07 +02006947 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006948 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07006949 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprang4847ae62017-06-27 07:06:52 -07006950
6951 // Insert a first video frame, causes another bitrate update.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006952 int64_t timestamp_ms = CurrentTimeMs();
sprang4847ae62017-06-27 07:06:52 -07006953 video_source_.IncomingCapturedFrame(
6954 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6955 WaitForEncodedFrame(timestamp_ms);
Per Kjellanderdcef6412020-10-07 15:09:05 +02006956 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
sprang4847ae62017-06-27 07:06:52 -07006957
6958 // Next, simulate video suspension due to pacer queue overrun.
Henrik Boström381d1092020-05-12 18:49:07 +02006959 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006960 DataRate::Zero(), DataRate::Zero(), DataRate::Zero(), 0, 1, 0);
sprang4847ae62017-06-27 07:06:52 -07006961
6962 // Skip ahead until a new periodic parameter update should have occured.
Niels Möllerfe407b72019-09-10 10:48:48 +02006963 timestamp_ms += kProcessIntervalMs;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006964 AdvanceTime(TimeDelta::Millis(kProcessIntervalMs));
sprang4847ae62017-06-27 07:06:52 -07006965
Per Kjellanderdcef6412020-10-07 15:09:05 +02006966 // No more allocations has been made.
sprang4847ae62017-06-27 07:06:52 -07006967 video_source_.IncomingCapturedFrame(
6968 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6969 ExpectDroppedFrame();
Per Kjellanderdcef6412020-10-07 15:09:05 +02006970 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
sprang4847ae62017-06-27 07:06:52 -07006971
mflodmancc3d4422017-08-03 08:27:51 -07006972 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 07:06:52 -07006973}
ilnik6b826ef2017-06-16 06:53:48 -07006974
Niels Möller4db138e2018-04-19 09:04:13 +02006975TEST_F(VideoStreamEncoderTest,
6976 DefaultCpuAdaptationThresholdsForSoftwareEncoder) {
6977 const int kFrameWidth = 1280;
6978 const int kFrameHeight = 720;
6979 const CpuOveruseOptions default_options;
Henrik Boström381d1092020-05-12 18:49:07 +02006980 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006981 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller4db138e2018-04-19 09:04:13 +02006982 video_source_.IncomingCapturedFrame(
6983 CreateFrame(1, kFrameWidth, kFrameHeight));
6984 WaitForEncodedFrame(1);
6985 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
6986 .low_encode_usage_threshold_percent,
6987 default_options.low_encode_usage_threshold_percent);
6988 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
6989 .high_encode_usage_threshold_percent,
6990 default_options.high_encode_usage_threshold_percent);
6991 video_stream_encoder_->Stop();
6992}
6993
6994TEST_F(VideoStreamEncoderTest,
6995 HigherCpuAdaptationThresholdsForHardwareEncoder) {
6996 const int kFrameWidth = 1280;
6997 const int kFrameHeight = 720;
6998 CpuOveruseOptions hardware_options;
6999 hardware_options.low_encode_usage_threshold_percent = 150;
7000 hardware_options.high_encode_usage_threshold_percent = 200;
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01007001 fake_encoder_.SetIsHardwareAccelerated(true);
Niels Möller4db138e2018-04-19 09:04:13 +02007002
Henrik Boström381d1092020-05-12 18:49:07 +02007003 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007004 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller4db138e2018-04-19 09:04:13 +02007005 video_source_.IncomingCapturedFrame(
7006 CreateFrame(1, kFrameWidth, kFrameHeight));
7007 WaitForEncodedFrame(1);
7008 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7009 .low_encode_usage_threshold_percent,
7010 hardware_options.low_encode_usage_threshold_percent);
7011 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7012 .high_encode_usage_threshold_percent,
7013 hardware_options.high_encode_usage_threshold_percent);
7014 video_stream_encoder_->Stop();
7015}
7016
Jakob Ivarsson461b1d92021-01-22 16:27:43 +01007017TEST_F(VideoStreamEncoderTest,
7018 CpuAdaptationThresholdsUpdatesWhenHardwareAccelerationChange) {
7019 const int kFrameWidth = 1280;
7020 const int kFrameHeight = 720;
7021
7022 const CpuOveruseOptions default_options;
7023 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007024 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Jakob Ivarsson461b1d92021-01-22 16:27:43 +01007025 video_source_.IncomingCapturedFrame(
7026 CreateFrame(1, kFrameWidth, kFrameHeight));
7027 WaitForEncodedFrame(1);
7028 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7029 .low_encode_usage_threshold_percent,
7030 default_options.low_encode_usage_threshold_percent);
7031 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7032 .high_encode_usage_threshold_percent,
7033 default_options.high_encode_usage_threshold_percent);
7034
7035 CpuOveruseOptions hardware_options;
7036 hardware_options.low_encode_usage_threshold_percent = 150;
7037 hardware_options.high_encode_usage_threshold_percent = 200;
7038 fake_encoder_.SetIsHardwareAccelerated(true);
7039
7040 video_source_.IncomingCapturedFrame(
7041 CreateFrame(2, kFrameWidth, kFrameHeight));
7042 WaitForEncodedFrame(2);
7043
7044 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7045 .low_encode_usage_threshold_percent,
7046 hardware_options.low_encode_usage_threshold_percent);
7047 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7048 .high_encode_usage_threshold_percent,
7049 hardware_options.high_encode_usage_threshold_percent);
7050
7051 video_stream_encoder_->Stop();
7052}
7053
Niels Möller6bb5ab92019-01-11 11:11:10 +01007054TEST_F(VideoStreamEncoderTest, DropsFramesWhenEncoderOvershoots) {
7055 const int kFrameWidth = 320;
7056 const int kFrameHeight = 240;
7057 const int kFps = 30;
Asa Persson606d3cb2021-10-04 10:07:11 +02007058 const DataRate kTargetBitrate = DataRate::KilobitsPerSec(120);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007059 const int kNumFramesInRun = kFps * 5; // Runs of five seconds.
7060
Henrik Boström381d1092020-05-12 18:49:07 +02007061 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007062 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007063
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007064 int64_t timestamp_ms = CurrentTimeMs();
Niels Möller6bb5ab92019-01-11 11:11:10 +01007065 max_framerate_ = kFps;
7066
7067 // Insert 3 seconds of video, verify number of drops with normal bitrate.
7068 fake_encoder_.SimulateOvershoot(1.0);
7069 int num_dropped = 0;
7070 for (int i = 0; i < kNumFramesInRun; ++i) {
7071 video_source_.IncomingCapturedFrame(
7072 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7073 // Wait up to two frame durations for a frame to arrive.
7074 if (!TimedWaitForEncodedFrame(timestamp_ms, 2 * 1000 / kFps)) {
7075 ++num_dropped;
7076 }
7077 timestamp_ms += 1000 / kFps;
7078 }
7079
Erik Språnga8d48ab2019-02-08 14:17:40 +01007080 // Framerate should be measured to be near the expected target rate.
7081 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
7082
7083 // Frame drops should be within 5% of expected 0%.
7084 EXPECT_NEAR(num_dropped, 0, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007085
7086 // Make encoder produce frames at double the expected bitrate during 3 seconds
7087 // of video, verify number of drops. Rate needs to be slightly changed in
7088 // order to force the rate to be reconfigured.
Erik Språng7ca375c2019-02-06 16:20:17 +01007089 double overshoot_factor = 2.0;
Erik Språng9d69cbe2020-10-22 17:44:42 +02007090 const RateControlSettings trials =
7091 RateControlSettings::ParseFromFieldTrials();
7092 if (trials.UseEncoderBitrateAdjuster()) {
Erik Språng7ca375c2019-02-06 16:20:17 +01007093 // With bitrate adjuster, when need to overshoot even more to trigger
Erik Språng9d69cbe2020-10-22 17:44:42 +02007094 // frame dropping since the adjuter will try to just lower the target
7095 // bitrate rather than drop frames. If network headroom can be used, it
7096 // doesn't push back as hard so we don't need quite as much overshoot.
7097 // These numbers are unfortunately a bit magical but there's not trivial
7098 // way to algebraically infer them.
Erik Språng73cf80a2021-03-24 11:10:09 +01007099 overshoot_factor = 3.0;
Erik Språng7ca375c2019-02-06 16:20:17 +01007100 }
7101 fake_encoder_.SimulateOvershoot(overshoot_factor);
Henrik Boström381d1092020-05-12 18:49:07 +02007102 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007103 kTargetBitrate + DataRate::KilobitsPerSec(1),
7104 kTargetBitrate + DataRate::KilobitsPerSec(1),
7105 kTargetBitrate + DataRate::KilobitsPerSec(1), 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007106 num_dropped = 0;
7107 for (int i = 0; i < kNumFramesInRun; ++i) {
7108 video_source_.IncomingCapturedFrame(
7109 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7110 // Wait up to two frame durations for a frame to arrive.
7111 if (!TimedWaitForEncodedFrame(timestamp_ms, 2 * 1000 / kFps)) {
7112 ++num_dropped;
7113 }
7114 timestamp_ms += 1000 / kFps;
7115 }
7116
Henrik Boström381d1092020-05-12 18:49:07 +02007117 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007118 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Erik Språnga8d48ab2019-02-08 14:17:40 +01007119
7120 // Target framerate should be still be near the expected target, despite
7121 // the frame drops.
7122 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
7123
7124 // Frame drops should be within 5% of expected 50%.
7125 EXPECT_NEAR(num_dropped, kNumFramesInRun / 2, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007126
7127 video_stream_encoder_->Stop();
7128}
7129
7130TEST_F(VideoStreamEncoderTest, ConfiguresCorrectFrameRate) {
7131 const int kFrameWidth = 320;
7132 const int kFrameHeight = 240;
7133 const int kActualInputFps = 24;
Asa Persson606d3cb2021-10-04 10:07:11 +02007134 const DataRate kTargetBitrate = DataRate::KilobitsPerSec(120);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007135
7136 ASSERT_GT(max_framerate_, kActualInputFps);
7137
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007138 int64_t timestamp_ms = CurrentTimeMs();
Niels Möller6bb5ab92019-01-11 11:11:10 +01007139 max_framerate_ = kActualInputFps;
Henrik Boström381d1092020-05-12 18:49:07 +02007140 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007141 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007142
7143 // Insert 3 seconds of video, with an input fps lower than configured max.
7144 for (int i = 0; i < kActualInputFps * 3; ++i) {
7145 video_source_.IncomingCapturedFrame(
7146 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7147 // Wait up to two frame durations for a frame to arrive.
7148 WaitForEncodedFrame(timestamp_ms);
7149 timestamp_ms += 1000 / kActualInputFps;
7150 }
7151
7152 EXPECT_NEAR(kActualInputFps, fake_encoder_.GetLastFramerate(), 1);
7153
7154 video_stream_encoder_->Stop();
7155}
7156
Markus Handell9a478b52021-11-18 16:07:01 +01007157TEST_F(VideoStreamEncoderTest, AccumulatesUpdateRectOnDroppedFrames) {
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01007158 VideoFrame::UpdateRect rect;
Markus Handell9a478b52021-11-18 16:07:01 +01007159 test::FrameForwarder source;
7160 video_stream_encoder_->SetSource(&source,
7161 DegradationPreference::MAINTAIN_FRAMERATE);
Henrik Boström381d1092020-05-12 18:49:07 +02007162 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007163 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01007164
Markus Handell9a478b52021-11-18 16:07:01 +01007165 source.IncomingCapturedFrame(CreateFrameWithUpdatedPixel(1, nullptr, 0));
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01007166 WaitForEncodedFrame(1);
7167 // On the very first frame full update should be forced.
7168 rect = fake_encoder_.GetLastUpdateRect();
7169 EXPECT_EQ(rect.offset_x, 0);
7170 EXPECT_EQ(rect.offset_y, 0);
7171 EXPECT_EQ(rect.height, codec_height_);
7172 EXPECT_EQ(rect.width, codec_width_);
Markus Handell9a478b52021-11-18 16:07:01 +01007173 // Frame with NTP timestamp 2 will be dropped due to outstanding frames
7174 // scheduled for processing during encoder queue processing of frame 2.
7175 source.IncomingCapturedFrame(CreateFrameWithUpdatedPixel(2, nullptr, 1));
7176 source.IncomingCapturedFrame(CreateFrameWithUpdatedPixel(3, nullptr, 10));
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01007177 WaitForEncodedFrame(3);
7178 // Updates to pixels 1 and 10 should be accumulated to one 10x1 rect.
7179 rect = fake_encoder_.GetLastUpdateRect();
7180 EXPECT_EQ(rect.offset_x, 1);
7181 EXPECT_EQ(rect.offset_y, 0);
7182 EXPECT_EQ(rect.width, 10);
7183 EXPECT_EQ(rect.height, 1);
7184
Markus Handell9a478b52021-11-18 16:07:01 +01007185 source.IncomingCapturedFrame(CreateFrameWithUpdatedPixel(4, nullptr, 0));
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01007186 WaitForEncodedFrame(4);
7187 // Previous frame was encoded, so no accumulation should happen.
7188 rect = fake_encoder_.GetLastUpdateRect();
7189 EXPECT_EQ(rect.offset_x, 0);
7190 EXPECT_EQ(rect.offset_y, 0);
7191 EXPECT_EQ(rect.width, 1);
7192 EXPECT_EQ(rect.height, 1);
7193
7194 video_stream_encoder_->Stop();
7195}
7196
Erik Språngd7329ca2019-02-21 21:19:53 +01007197TEST_F(VideoStreamEncoderTest, SetsFrameTypes) {
Henrik Boström381d1092020-05-12 18:49:07 +02007198 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007199 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01007200
7201 // First frame is always keyframe.
7202 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7203 WaitForEncodedFrame(1);
Niels Möller8f7ce222019-03-21 15:43:58 +01007204 EXPECT_THAT(
7205 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007206 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007207
7208 // Insert delta frame.
7209 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
7210 WaitForEncodedFrame(2);
Niels Möller8f7ce222019-03-21 15:43:58 +01007211 EXPECT_THAT(
7212 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007213 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007214
7215 // Request next frame be a key-frame.
7216 video_stream_encoder_->SendKeyFrame();
7217 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
7218 WaitForEncodedFrame(3);
Niels Möller8f7ce222019-03-21 15:43:58 +01007219 EXPECT_THAT(
7220 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007221 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007222
7223 video_stream_encoder_->Stop();
7224}
7225
7226TEST_F(VideoStreamEncoderTest, SetsFrameTypesSimulcast) {
7227 // Setup simulcast with three streams.
7228 ResetEncoder("VP8", 3, 1, 1, false);
Henrik Boström381d1092020-05-12 18:49:07 +02007229 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007230 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
7231 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01007232 // Wait for all three layers before triggering event.
7233 sink_.SetNumExpectedLayers(3);
7234
7235 // First frame is always keyframe.
7236 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7237 WaitForEncodedFrame(1);
7238 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007239 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7240 VideoFrameType::kVideoFrameKey,
7241 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007242
7243 // Insert delta frame.
7244 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
7245 WaitForEncodedFrame(2);
7246 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007247 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
7248 VideoFrameType::kVideoFrameDelta,
7249 VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007250
7251 // Request next frame be a key-frame.
7252 // Only first stream is configured to produce key-frame.
7253 video_stream_encoder_->SendKeyFrame();
7254 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
7255 WaitForEncodedFrame(3);
Sergey Silkine62a08a2019-05-13 13:45:39 +02007256
7257 // TODO(webrtc:10615): Map keyframe request to spatial layer. Currently
7258 // keyframe request on any layer triggers keyframe on all layers.
Erik Språngd7329ca2019-02-21 21:19:53 +01007259 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007260 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
Sergey Silkine62a08a2019-05-13 13:45:39 +02007261 VideoFrameType::kVideoFrameKey,
7262 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007263
7264 video_stream_encoder_->Stop();
7265}
7266
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007267TEST_F(VideoStreamEncoderTest, DoesNotRewriteH264BitstreamWithOptimalSps) {
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007268 // SPS contains VUI with restrictions on the maximum number of reordered
7269 // pictures, there is no need to rewrite the bitstream to enable faster
7270 // decoding.
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007271 ResetEncoder("H264", 1, 1, 1, false);
7272
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007273 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007274 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007275 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007276
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007277 fake_encoder_.SetEncodedImageData(
Asa Persson606d3cb2021-10-04 10:07:11 +02007278 EncodedImageBuffer::Create(kOptimalSps, sizeof(kOptimalSps)));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007279
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007280 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7281 WaitForEncodedFrame(1);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007282
7283 EXPECT_THAT(sink_.GetLastEncodedImageData(),
Asa Persson606d3cb2021-10-04 10:07:11 +02007284 testing::ElementsAreArray(kOptimalSps));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007285
7286 video_stream_encoder_->Stop();
7287}
7288
7289TEST_F(VideoStreamEncoderTest, RewritesH264BitstreamWithNonOptimalSps) {
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007290 // SPS does not contain VUI, the bitstream is will be rewritten with added
7291 // VUI with restrictions on the maximum number of reordered pictures to
7292 // enable faster decoding.
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007293 uint8_t original_sps[] = {0, 0, 0, 1, H264::NaluType::kSps,
7294 0x00, 0x00, 0x03, 0x03, 0xF4,
7295 0x05, 0x03, 0xC7, 0xC0};
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007296 ResetEncoder("H264", 1, 1, 1, false);
7297
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007298 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007299 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007300 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007301
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007302 fake_encoder_.SetEncodedImageData(
7303 EncodedImageBuffer::Create(original_sps, sizeof(original_sps)));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007304
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007305 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7306 WaitForEncodedFrame(1);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007307
7308 EXPECT_THAT(sink_.GetLastEncodedImageData(),
Asa Persson606d3cb2021-10-04 10:07:11 +02007309 testing::ElementsAreArray(kOptimalSps));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007310
7311 video_stream_encoder_->Stop();
7312}
7313
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007314TEST_F(VideoStreamEncoderTest, CopiesVideoFrameMetadataAfterDownscale) {
7315 const int kFrameWidth = 1280;
7316 const int kFrameHeight = 720;
Asa Persson606d3cb2021-10-04 10:07:11 +02007317 const DataRate kTargetBitrate =
7318 DataRate::KilobitsPerSec(300); // Too low for HD resolution.
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007319
Henrik Boström381d1092020-05-12 18:49:07 +02007320 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007321 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007322 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7323
7324 // Insert a first video frame. It should be dropped because of downscale in
7325 // resolution.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007326 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007327 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7328 frame.set_rotation(kVideoRotation_270);
7329 video_source_.IncomingCapturedFrame(frame);
7330
7331 ExpectDroppedFrame();
7332
7333 // Second frame is downscaled.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007334 timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007335 frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
7336 frame.set_rotation(kVideoRotation_90);
7337 video_source_.IncomingCapturedFrame(frame);
7338
7339 WaitForEncodedFrame(timestamp_ms);
7340 sink_.CheckLastFrameRotationMatches(kVideoRotation_90);
7341
7342 // Insert another frame, also downscaled.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007343 timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007344 frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
7345 frame.set_rotation(kVideoRotation_180);
7346 video_source_.IncomingCapturedFrame(frame);
7347
7348 WaitForEncodedFrame(timestamp_ms);
7349 sink_.CheckLastFrameRotationMatches(kVideoRotation_180);
7350
7351 video_stream_encoder_->Stop();
7352}
7353
Erik Språng5056af02019-09-02 15:53:11 +02007354TEST_F(VideoStreamEncoderTest, BandwidthAllocationLowerBound) {
7355 const int kFrameWidth = 320;
7356 const int kFrameHeight = 180;
7357
7358 // Initial rate.
Henrik Boström381d1092020-05-12 18:49:07 +02007359 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007360 /*target_bitrate=*/DataRate::KilobitsPerSec(300),
7361 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(300),
7362 /*link_allocation=*/DataRate::KilobitsPerSec(300),
Erik Språng5056af02019-09-02 15:53:11 +02007363 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007364 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007365 /*cwnd_reduce_ratio=*/0);
Erik Språng5056af02019-09-02 15:53:11 +02007366
7367 // Insert a first video frame so that encoder gets configured.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007368 int64_t timestamp_ms = CurrentTimeMs();
Erik Språng5056af02019-09-02 15:53:11 +02007369 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7370 frame.set_rotation(kVideoRotation_270);
7371 video_source_.IncomingCapturedFrame(frame);
7372 WaitForEncodedFrame(timestamp_ms);
7373
7374 // Set a target rate below the minimum allowed by the codec settings.
Asa Persson606d3cb2021-10-04 10:07:11 +02007375 VideoCodec codec_config = fake_encoder_.config();
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007376 DataRate min_rate = DataRate::KilobitsPerSec(codec_config.minBitrate);
7377 DataRate target_rate = min_rate - DataRate::KilobitsPerSec(1);
Henrik Boström381d1092020-05-12 18:49:07 +02007378 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Erik Språng5056af02019-09-02 15:53:11 +02007379 /*target_bitrate=*/target_rate,
Florent Castellia8336d32019-09-09 13:36:55 +02007380 /*stable_target_bitrate=*/target_rate,
Erik Språng5056af02019-09-02 15:53:11 +02007381 /*link_allocation=*/target_rate,
7382 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007383 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007384 /*cwnd_reduce_ratio=*/0);
Erik Språng5056af02019-09-02 15:53:11 +02007385 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7386
7387 // Target bitrate and bandwidth allocation should both be capped at min_rate.
7388 auto rate_settings = fake_encoder_.GetAndResetLastRateControlSettings();
7389 ASSERT_TRUE(rate_settings.has_value());
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007390 DataRate allocation_sum =
7391 DataRate::BitsPerSec(rate_settings->bitrate.get_sum_bps());
Erik Språng5056af02019-09-02 15:53:11 +02007392 EXPECT_EQ(min_rate, allocation_sum);
7393 EXPECT_EQ(rate_settings->bandwidth_allocation, min_rate);
7394
7395 video_stream_encoder_->Stop();
7396}
7397
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01007398TEST_F(VideoStreamEncoderTest, EncoderRatesPropagatedOnReconfigure) {
Henrik Boström381d1092020-05-12 18:49:07 +02007399 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007400 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02007401 // Capture a frame and wait for it to synchronize with the encoder thread.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007402 int64_t timestamp_ms = CurrentTimeMs();
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02007403 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
7404 WaitForEncodedFrame(1);
7405
7406 auto prev_rate_settings = fake_encoder_.GetAndResetLastRateControlSettings();
7407 ASSERT_TRUE(prev_rate_settings.has_value());
7408 EXPECT_EQ(static_cast<int>(prev_rate_settings->framerate_fps),
7409 kDefaultFramerate);
7410
7411 // Send 1s of video to ensure the framerate is stable at kDefaultFramerate.
7412 for (int i = 0; i < 2 * kDefaultFramerate; i++) {
7413 timestamp_ms += 1000 / kDefaultFramerate;
7414 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
7415 WaitForEncodedFrame(timestamp_ms);
7416 }
7417 EXPECT_EQ(static_cast<int>(fake_encoder_.GetLastFramerate()),
7418 kDefaultFramerate);
7419 // Capture larger frame to trigger a reconfigure.
7420 codec_height_ *= 2;
7421 codec_width_ *= 2;
7422 timestamp_ms += 1000 / kDefaultFramerate;
7423 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
7424 WaitForEncodedFrame(timestamp_ms);
7425
7426 EXPECT_EQ(2, sink_.number_of_reconfigurations());
7427 auto current_rate_settings =
7428 fake_encoder_.GetAndResetLastRateControlSettings();
7429 // Ensure we have actually reconfigured twice
7430 // The rate settings should have been set again even though
7431 // they haven't changed.
7432 ASSERT_TRUE(current_rate_settings.has_value());
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007433 EXPECT_EQ(prev_rate_settings, current_rate_settings);
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02007434
7435 video_stream_encoder_->Stop();
7436}
7437
philipeld9cc8c02019-09-16 14:53:40 +02007438struct MockEncoderSwitchRequestCallback : public EncoderSwitchRequestCallback {
Danil Chapovalov91fdc602020-05-14 19:17:51 +02007439 MOCK_METHOD(void, RequestEncoderFallback, (), (override));
7440 MOCK_METHOD(void, RequestEncoderSwitch, (const Config& conf), (override));
7441 MOCK_METHOD(void,
7442 RequestEncoderSwitch,
7443 (const webrtc::SdpVideoFormat& format),
7444 (override));
philipeld9cc8c02019-09-16 14:53:40 +02007445};
7446
philipel9b058032020-02-10 11:30:00 +01007447TEST_F(VideoStreamEncoderTest, EncoderSelectorCurrentEncoderIsSignaled) {
7448 constexpr int kDontCare = 100;
7449 StrictMock<MockEncoderSelector> encoder_selector;
7450 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7451 &fake_encoder_, &encoder_selector);
7452 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7453
7454 // Reset encoder for new configuration to take effect.
7455 ConfigureEncoder(video_encoder_config_.Copy());
7456
7457 EXPECT_CALL(encoder_selector, OnCurrentEncoder(_));
7458
7459 video_source_.IncomingCapturedFrame(
7460 CreateFrame(kDontCare, kDontCare, kDontCare));
Markus Handell28c71802021-11-08 10:11:55 +01007461 AdvanceTime(TimeDelta::Zero());
philipel9b058032020-02-10 11:30:00 +01007462 video_stream_encoder_->Stop();
7463
7464 // The encoders produces by the VideoEncoderProxyFactory have a pointer back
7465 // to it's factory, so in order for the encoder instance in the
Artem Titovab30d722021-07-27 16:22:11 +02007466 // `video_stream_encoder_` to be destroyed before the `encoder_factory` we
7467 // reset the `video_stream_encoder_` here.
philipel9b058032020-02-10 11:30:00 +01007468 video_stream_encoder_.reset();
7469}
7470
7471TEST_F(VideoStreamEncoderTest, EncoderSelectorBitrateSwitch) {
7472 constexpr int kDontCare = 100;
7473
7474 NiceMock<MockEncoderSelector> encoder_selector;
7475 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7476 video_send_config_.encoder_settings.encoder_switch_request_callback =
7477 &switch_callback;
7478 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7479 &fake_encoder_, &encoder_selector);
7480 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7481
7482 // Reset encoder for new configuration to take effect.
7483 ConfigureEncoder(video_encoder_config_.Copy());
7484
Mirta Dvornicic4f34d782020-02-26 13:01:19 +01007485 ON_CALL(encoder_selector, OnAvailableBitrate(_))
philipel9b058032020-02-10 11:30:00 +01007486 .WillByDefault(Return(SdpVideoFormat("AV1")));
7487 EXPECT_CALL(switch_callback,
7488 RequestEncoderSwitch(Matcher<const SdpVideoFormat&>(
7489 Field(&SdpVideoFormat::name, "AV1"))));
7490
Henrik Boström381d1092020-05-12 18:49:07 +02007491 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007492 /*target_bitrate=*/DataRate::KilobitsPerSec(50),
7493 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(kDontCare),
7494 /*link_allocation=*/DataRate::KilobitsPerSec(kDontCare),
philipel9b058032020-02-10 11:30:00 +01007495 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007496 /*round_trip_time_ms=*/0,
philipel9b058032020-02-10 11:30:00 +01007497 /*cwnd_reduce_ratio=*/0);
Markus Handell28c71802021-11-08 10:11:55 +01007498 AdvanceTime(TimeDelta::Zero());
philipel9b058032020-02-10 11:30:00 +01007499
7500 video_stream_encoder_->Stop();
7501}
7502
7503TEST_F(VideoStreamEncoderTest, EncoderSelectorBrokenEncoderSwitch) {
7504 constexpr int kSufficientBitrateToNotDrop = 1000;
7505 constexpr int kDontCare = 100;
7506
7507 NiceMock<MockVideoEncoder> video_encoder;
7508 NiceMock<MockEncoderSelector> encoder_selector;
7509 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7510 video_send_config_.encoder_settings.encoder_switch_request_callback =
7511 &switch_callback;
7512 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7513 &video_encoder, &encoder_selector);
7514 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7515
7516 // Reset encoder for new configuration to take effect.
7517 ConfigureEncoder(video_encoder_config_.Copy());
7518
7519 // The VideoStreamEncoder needs some bitrate before it can start encoding,
7520 // setting some bitrate so that subsequent calls to WaitForEncodedFrame does
7521 // not fail.
Henrik Boström381d1092020-05-12 18:49:07 +02007522 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007523 /*target_bitrate=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
7524 /*stable_target_bitrate=*/
7525 DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
7526 /*link_allocation=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
philipel9b058032020-02-10 11:30:00 +01007527 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007528 /*round_trip_time_ms=*/0,
philipel9b058032020-02-10 11:30:00 +01007529 /*cwnd_reduce_ratio=*/0);
7530
7531 ON_CALL(video_encoder, Encode(_, _))
7532 .WillByDefault(Return(WEBRTC_VIDEO_CODEC_ENCODER_FAILURE));
7533 ON_CALL(encoder_selector, OnEncoderBroken())
7534 .WillByDefault(Return(SdpVideoFormat("AV2")));
7535
7536 rtc::Event encode_attempted;
7537 EXPECT_CALL(switch_callback,
7538 RequestEncoderSwitch(Matcher<const SdpVideoFormat&>(_)))
7539 .WillOnce([&encode_attempted](const SdpVideoFormat& format) {
7540 EXPECT_EQ(format.name, "AV2");
7541 encode_attempted.Set();
7542 });
7543
7544 video_source_.IncomingCapturedFrame(CreateFrame(1, kDontCare, kDontCare));
7545 encode_attempted.Wait(3000);
7546
Markus Handell28c71802021-11-08 10:11:55 +01007547 AdvanceTime(TimeDelta::Zero());
Tomas Gunnarssond41c2a62020-09-21 15:56:42 +02007548
philipel9b058032020-02-10 11:30:00 +01007549 video_stream_encoder_->Stop();
7550
7551 // The encoders produces by the VideoEncoderProxyFactory have a pointer back
7552 // to it's factory, so in order for the encoder instance in the
Artem Titovab30d722021-07-27 16:22:11 +02007553 // `video_stream_encoder_` to be destroyed before the `encoder_factory` we
7554 // reset the `video_stream_encoder_` here.
philipel9b058032020-02-10 11:30:00 +01007555 video_stream_encoder_.reset();
7556}
7557
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007558TEST_F(VideoStreamEncoderTest,
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01007559 AllocationPropagatedToEncoderWhenTargetRateChanged) {
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007560 const int kFrameWidth = 320;
7561 const int kFrameHeight = 180;
7562
7563 // Set initial rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007564 auto rate = DataRate::KilobitsPerSec(100);
Henrik Boström381d1092020-05-12 18:49:07 +02007565 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007566 /*target_bitrate=*/rate,
7567 /*stable_target_bitrate=*/rate,
7568 /*link_allocation=*/rate,
7569 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007570 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007571 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007572
7573 // Insert a first video frame so that encoder gets configured.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007574 int64_t timestamp_ms = CurrentTimeMs();
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007575 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7576 frame.set_rotation(kVideoRotation_270);
7577 video_source_.IncomingCapturedFrame(frame);
7578 WaitForEncodedFrame(timestamp_ms);
7579 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
7580
7581 // Change of target bitrate propagates to the encoder.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007582 auto new_stable_rate = rate - DataRate::KilobitsPerSec(5);
Henrik Boström381d1092020-05-12 18:49:07 +02007583 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007584 /*target_bitrate=*/new_stable_rate,
7585 /*stable_target_bitrate=*/new_stable_rate,
7586 /*link_allocation=*/rate,
7587 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007588 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007589 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007590 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7591 EXPECT_EQ(2, fake_encoder_.GetNumSetRates());
7592 video_stream_encoder_->Stop();
7593}
7594
7595TEST_F(VideoStreamEncoderTest,
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01007596 AllocationNotPropagatedToEncoderWhenTargetRateUnchanged) {
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007597 const int kFrameWidth = 320;
7598 const int kFrameHeight = 180;
7599
7600 // Set initial rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007601 auto rate = DataRate::KilobitsPerSec(100);
Henrik Boström381d1092020-05-12 18:49:07 +02007602 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007603 /*target_bitrate=*/rate,
7604 /*stable_target_bitrate=*/rate,
7605 /*link_allocation=*/rate,
7606 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007607 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007608 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007609
7610 // Insert a first video frame so that encoder gets configured.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007611 int64_t timestamp_ms = CurrentTimeMs();
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007612 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7613 frame.set_rotation(kVideoRotation_270);
7614 video_source_.IncomingCapturedFrame(frame);
7615 WaitForEncodedFrame(timestamp_ms);
7616 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
7617
7618 // Set a higher target rate without changing the link_allocation. Should not
7619 // reset encoder's rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007620 auto new_stable_rate = rate - DataRate::KilobitsPerSec(5);
Henrik Boström381d1092020-05-12 18:49:07 +02007621 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007622 /*target_bitrate=*/rate,
7623 /*stable_target_bitrate=*/new_stable_rate,
7624 /*link_allocation=*/rate,
7625 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007626 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007627 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007628 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7629 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
7630 video_stream_encoder_->Stop();
7631}
7632
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007633TEST_F(VideoStreamEncoderTest, AutomaticAnimationDetection) {
7634 test::ScopedFieldTrials field_trials(
7635 "WebRTC-AutomaticAnimationDetectionScreenshare/"
7636 "enabled:true,min_fps:20,min_duration_ms:1000,min_area_ratio:0.8/");
7637 const int kFramerateFps = 30;
7638 const int kWidth = 1920;
7639 const int kHeight = 1080;
7640 const int kNumFrames = 2 * kFramerateFps; // >1 seconds of frames.
7641 // Works on screenshare mode.
7642 ResetEncoder("VP8", 1, 1, 1, /*screenshare*/ true);
7643 // We rely on the automatic resolution adaptation, but we handle framerate
7644 // adaptation manually by mocking the stats proxy.
7645 video_source_.set_adaptation_enabled(true);
7646
7647 // BALANCED degradation preference is required for this feature.
Henrik Boström381d1092020-05-12 18:49:07 +02007648 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007649 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007650 video_stream_encoder_->SetSource(&video_source_,
7651 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02007652 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007653
7654 VideoFrame frame = CreateFrame(1, kWidth, kHeight);
7655 frame.set_update_rect(VideoFrame::UpdateRect{0, 0, kWidth, kHeight});
7656
7657 // Pass enough frames with the full update to trigger animation detection.
7658 for (int i = 0; i < kNumFrames; ++i) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007659 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007660 frame.set_ntp_time_ms(timestamp_ms);
7661 frame.set_timestamp_us(timestamp_ms * 1000);
7662 video_source_.IncomingCapturedFrame(frame);
7663 WaitForEncodedFrame(timestamp_ms);
7664 }
7665
7666 // Resolution should be limited.
7667 rtc::VideoSinkWants expected;
7668 expected.max_framerate_fps = kFramerateFps;
7669 expected.max_pixel_count = 1280 * 720 + 1;
Evan Shrubsole5fd40602020-05-25 16:19:54 +02007670 EXPECT_THAT(video_source_.sink_wants(), FpsEqResolutionLt(expected));
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007671
7672 // Pass one frame with no known update.
7673 // Resolution cap should be removed immediately.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007674 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007675 frame.set_ntp_time_ms(timestamp_ms);
7676 frame.set_timestamp_us(timestamp_ms * 1000);
7677 frame.clear_update_rect();
7678
7679 video_source_.IncomingCapturedFrame(frame);
7680 WaitForEncodedFrame(timestamp_ms);
7681
7682 // Resolution should be unlimited now.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02007683 EXPECT_THAT(video_source_.sink_wants(),
7684 FpsMatchesResolutionMax(Eq(kFramerateFps)));
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007685
7686 video_stream_encoder_->Stop();
7687}
7688
Ilya Nikolaevskiy09eb6e22020-06-05 12:36:32 +02007689TEST_F(VideoStreamEncoderTest, ConfiguresVp9SvcAtOddResolutions) {
7690 const int kWidth = 720; // 540p adapted down.
7691 const int kHeight = 405;
7692 const int kNumFrames = 3;
7693 // Works on screenshare mode.
7694 ResetEncoder("VP9", /*num_streams=*/1, /*num_temporal_layers=*/1,
7695 /*num_spatial_layers=*/2, /*screenshare=*/true);
7696
7697 video_source_.set_adaptation_enabled(true);
7698
7699 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007700 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy09eb6e22020-06-05 12:36:32 +02007701
7702 VideoFrame frame = CreateFrame(1, kWidth, kHeight);
7703
7704 // Pass enough frames with the full update to trigger animation detection.
7705 for (int i = 0; i < kNumFrames; ++i) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007706 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiy09eb6e22020-06-05 12:36:32 +02007707 frame.set_ntp_time_ms(timestamp_ms);
7708 frame.set_timestamp_us(timestamp_ms * 1000);
7709 video_source_.IncomingCapturedFrame(frame);
7710 WaitForEncodedFrame(timestamp_ms);
7711 }
7712
7713 video_stream_encoder_->Stop();
7714}
7715
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007716TEST_F(VideoStreamEncoderTest, EncoderResetAccordingToParameterChange) {
7717 const float downscale_factors[] = {4.0, 2.0, 1.0};
7718 const int number_layers =
7719 sizeof(downscale_factors) / sizeof(downscale_factors[0]);
7720 VideoEncoderConfig config;
7721 test::FillEncoderConfiguration(kVideoCodecVP8, number_layers, &config);
7722 for (int i = 0; i < number_layers; ++i) {
7723 config.simulcast_layers[i].scale_resolution_down_by = downscale_factors[i];
7724 config.simulcast_layers[i].active = true;
7725 }
7726 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02007727 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007728 "VP8", /*max qp*/ 56, /*screencast*/ false,
7729 /*screenshare enabled*/ false);
7730 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007731 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
7732 0, 0, 0);
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007733
7734 // First initialization.
7735 // Encoder should be initialized. Next frame should be key frame.
7736 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7737 sink_.SetNumExpectedLayers(number_layers);
7738 int64_t timestamp_ms = kFrameIntervalMs;
7739 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7740 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02007741 EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007742 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7743 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7744 VideoFrameType::kVideoFrameKey,
7745 VideoFrameType::kVideoFrameKey}));
7746
7747 // Disable top layer.
7748 // Encoder shouldn't be re-initialized. Next frame should be delta frame.
7749 config.simulcast_layers[number_layers - 1].active = false;
7750 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7751 sink_.SetNumExpectedLayers(number_layers - 1);
7752 timestamp_ms += kFrameIntervalMs;
7753 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7754 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02007755 EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007756 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7757 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
7758 VideoFrameType::kVideoFrameDelta,
7759 VideoFrameType::kVideoFrameDelta}));
7760
7761 // Re-enable top layer.
7762 // Encoder should be re-initialized. Next frame should be key frame.
7763 config.simulcast_layers[number_layers - 1].active = true;
7764 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7765 sink_.SetNumExpectedLayers(number_layers);
7766 timestamp_ms += kFrameIntervalMs;
7767 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7768 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02007769 EXPECT_EQ(2, fake_encoder_.GetNumInitializations());
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007770 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7771 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7772 VideoFrameType::kVideoFrameKey,
7773 VideoFrameType::kVideoFrameKey}));
7774
7775 // Top layer max rate change.
7776 // Encoder shouldn't be re-initialized. Next frame should be delta frame.
7777 config.simulcast_layers[number_layers - 1].max_bitrate_bps -= 100;
7778 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7779 sink_.SetNumExpectedLayers(number_layers);
7780 timestamp_ms += kFrameIntervalMs;
7781 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7782 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02007783 EXPECT_EQ(2, fake_encoder_.GetNumInitializations());
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007784 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7785 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
7786 VideoFrameType::kVideoFrameDelta,
7787 VideoFrameType::kVideoFrameDelta}));
7788
7789 // Top layer resolution change.
7790 // Encoder should be re-initialized. Next frame should be key frame.
7791 config.simulcast_layers[number_layers - 1].scale_resolution_down_by += 0.1;
7792 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7793 sink_.SetNumExpectedLayers(number_layers);
7794 timestamp_ms += kFrameIntervalMs;
7795 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7796 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02007797 EXPECT_EQ(3, fake_encoder_.GetNumInitializations());
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007798 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7799 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7800 VideoFrameType::kVideoFrameKey,
7801 VideoFrameType::kVideoFrameKey}));
7802 video_stream_encoder_->Stop();
7803}
7804
Henrik Boström1124ed12021-02-25 10:30:39 +01007805TEST_F(VideoStreamEncoderTest, EncoderResolutionsExposedInSinglecast) {
7806 const int kFrameWidth = 1280;
7807 const int kFrameHeight = 720;
7808
7809 SetUp();
7810 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007811 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Henrik Boström1124ed12021-02-25 10:30:39 +01007812
7813 // Capturing a frame should reconfigure the encoder and expose the encoder
7814 // resolution, which is the same as the input frame.
7815 int64_t timestamp_ms = kFrameIntervalMs;
7816 video_source_.IncomingCapturedFrame(
7817 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7818 WaitForEncodedFrame(timestamp_ms);
7819 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7820 EXPECT_THAT(video_source_.sink_wants().resolutions,
7821 ::testing::ElementsAreArray(
7822 {rtc::VideoSinkWants::FrameSize(kFrameWidth, kFrameHeight)}));
7823
7824 video_stream_encoder_->Stop();
7825}
7826
7827TEST_F(VideoStreamEncoderTest, EncoderResolutionsExposedInSimulcast) {
7828 // Pick downscale factors such that we never encode at full resolution - this
7829 // is an interesting use case. The frame resolution influences the encoder
Artem Titovab30d722021-07-27 16:22:11 +02007830 // resolutions, but if no layer has `scale_resolution_down_by` == 1 then the
Henrik Boström1124ed12021-02-25 10:30:39 +01007831 // encoder should not ask for the frame resolution. This allows video frames
7832 // to have the appearence of one resolution but optimize its internal buffers
7833 // for what is actually encoded.
7834 const size_t kNumSimulcastLayers = 3u;
7835 const float kDownscaleFactors[] = {8.0, 4.0, 2.0};
7836 const int kFrameWidth = 1280;
7837 const int kFrameHeight = 720;
7838 const rtc::VideoSinkWants::FrameSize kLayer0Size(
7839 kFrameWidth / kDownscaleFactors[0], kFrameHeight / kDownscaleFactors[0]);
7840 const rtc::VideoSinkWants::FrameSize kLayer1Size(
7841 kFrameWidth / kDownscaleFactors[1], kFrameHeight / kDownscaleFactors[1]);
7842 const rtc::VideoSinkWants::FrameSize kLayer2Size(
7843 kFrameWidth / kDownscaleFactors[2], kFrameHeight / kDownscaleFactors[2]);
7844
7845 VideoEncoderConfig config;
7846 test::FillEncoderConfiguration(kVideoCodecVP8, kNumSimulcastLayers, &config);
7847 for (size_t i = 0; i < kNumSimulcastLayers; ++i) {
7848 config.simulcast_layers[i].scale_resolution_down_by = kDownscaleFactors[i];
7849 config.simulcast_layers[i].active = true;
7850 }
7851 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02007852 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Henrik Boström1124ed12021-02-25 10:30:39 +01007853 "VP8", /*max qp*/ 56, /*screencast*/ false,
7854 /*screenshare enabled*/ false);
7855 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007856 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
7857 0, 0, 0);
Henrik Boström1124ed12021-02-25 10:30:39 +01007858
7859 // Capture a frame with all layers active.
7860 int64_t timestamp_ms = kFrameIntervalMs;
7861 sink_.SetNumExpectedLayers(kNumSimulcastLayers);
7862 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7863 video_source_.IncomingCapturedFrame(
7864 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7865 WaitForEncodedFrame(timestamp_ms);
7866 // Expect encoded resolutions to match the expected simulcast layers.
7867 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7868 EXPECT_THAT(
7869 video_source_.sink_wants().resolutions,
7870 ::testing::ElementsAreArray({kLayer0Size, kLayer1Size, kLayer2Size}));
7871
7872 // Capture a frame with one of the layers inactive.
7873 timestamp_ms += kFrameIntervalMs;
7874 config.simulcast_layers[2].active = false;
7875 sink_.SetNumExpectedLayers(kNumSimulcastLayers - 1);
7876 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7877 video_source_.IncomingCapturedFrame(
7878 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7879 WaitForEncodedFrame(timestamp_ms);
7880
7881 // Expect encoded resolutions to match the expected simulcast layers.
7882 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7883 EXPECT_THAT(video_source_.sink_wants().resolutions,
7884 ::testing::ElementsAreArray({kLayer0Size, kLayer1Size}));
7885
7886 // Capture a frame with all but one layer turned off.
7887 timestamp_ms += kFrameIntervalMs;
7888 config.simulcast_layers[1].active = false;
7889 sink_.SetNumExpectedLayers(kNumSimulcastLayers - 2);
7890 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7891 video_source_.IncomingCapturedFrame(
7892 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7893 WaitForEncodedFrame(timestamp_ms);
7894
7895 // Expect encoded resolutions to match the expected simulcast layers.
7896 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7897 EXPECT_THAT(video_source_.sink_wants().resolutions,
7898 ::testing::ElementsAreArray({kLayer0Size}));
7899
7900 video_stream_encoder_->Stop();
7901}
7902
Sergey Silkin0e42cf72021-03-15 10:12:57 +01007903TEST_F(VideoStreamEncoderTest, QpPresent_QpKept) {
Sergey Silkin0e42cf72021-03-15 10:12:57 +01007904 ResetEncoder("VP8", 1, 1, 1, false);
7905
Niels Möller8b692902021-06-14 12:04:57 +02007906 // Force encoder reconfig.
7907 video_source_.IncomingCapturedFrame(
7908 CreateFrame(1, codec_width_, codec_height_));
7909 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7910
Sergey Silkin0e42cf72021-03-15 10:12:57 +01007911 // Set QP on encoded frame and pass the frame to encode complete callback.
7912 // Since QP is present QP parsing won't be triggered and the original value
7913 // should be kept.
7914 EncodedImage encoded_image;
7915 encoded_image.qp_ = 123;
7916 encoded_image.SetEncodedData(EncodedImageBuffer::Create(
7917 kCodedFrameVp8Qp25, sizeof(kCodedFrameVp8Qp25)));
7918 CodecSpecificInfo codec_info;
7919 codec_info.codecType = kVideoCodecVP8;
7920 fake_encoder_.InjectEncodedImage(encoded_image, &codec_info);
7921 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
7922 EXPECT_EQ(sink_.GetLastEncodedImage().qp_, 123);
7923 video_stream_encoder_->Stop();
7924}
7925
7926TEST_F(VideoStreamEncoderTest, QpAbsent_QpParsed) {
Sergey Silkin0e42cf72021-03-15 10:12:57 +01007927 ResetEncoder("VP8", 1, 1, 1, false);
7928
Niels Möller8b692902021-06-14 12:04:57 +02007929 // Force encoder reconfig.
7930 video_source_.IncomingCapturedFrame(
7931 CreateFrame(1, codec_width_, codec_height_));
7932 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7933
Sergey Silkin0e42cf72021-03-15 10:12:57 +01007934 // Pass an encoded frame without QP to encode complete callback. QP should be
7935 // parsed and set.
7936 EncodedImage encoded_image;
7937 encoded_image.qp_ = -1;
7938 encoded_image.SetEncodedData(EncodedImageBuffer::Create(
7939 kCodedFrameVp8Qp25, sizeof(kCodedFrameVp8Qp25)));
7940 CodecSpecificInfo codec_info;
7941 codec_info.codecType = kVideoCodecVP8;
7942 fake_encoder_.InjectEncodedImage(encoded_image, &codec_info);
7943 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
7944 EXPECT_EQ(sink_.GetLastEncodedImage().qp_, 25);
7945 video_stream_encoder_->Stop();
7946}
7947
7948TEST_F(VideoStreamEncoderTest, QpAbsentParsingDisabled_QpAbsent) {
7949 webrtc::test::ScopedFieldTrials field_trials(
7950 "WebRTC-QpParsingKillSwitch/Enabled/");
7951
Sergey Silkin0e42cf72021-03-15 10:12:57 +01007952 ResetEncoder("VP8", 1, 1, 1, false);
7953
Niels Möller8b692902021-06-14 12:04:57 +02007954 // Force encoder reconfig.
7955 video_source_.IncomingCapturedFrame(
7956 CreateFrame(1, codec_width_, codec_height_));
7957 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7958
Sergey Silkin0e42cf72021-03-15 10:12:57 +01007959 EncodedImage encoded_image;
7960 encoded_image.qp_ = -1;
7961 encoded_image.SetEncodedData(EncodedImageBuffer::Create(
7962 kCodedFrameVp8Qp25, sizeof(kCodedFrameVp8Qp25)));
7963 CodecSpecificInfo codec_info;
7964 codec_info.codecType = kVideoCodecVP8;
7965 fake_encoder_.InjectEncodedImage(encoded_image, &codec_info);
7966 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
7967 EXPECT_EQ(sink_.GetLastEncodedImage().qp_, -1);
7968 video_stream_encoder_->Stop();
7969}
7970
Sergey Silkind19e3b92021-03-16 10:05:30 +00007971TEST_F(VideoStreamEncoderTest,
7972 QualityScalingNotAllowed_QualityScalingDisabled) {
7973 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
7974
7975 // Disable scaling settings in encoder info.
7976 fake_encoder_.SetQualityScaling(false);
7977 // Disable quality scaling in encoder config.
7978 video_encoder_config.is_quality_scaling_allowed = false;
7979 ConfigureEncoder(std::move(video_encoder_config));
7980
7981 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007982 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkind19e3b92021-03-16 10:05:30 +00007983
7984 test::FrameForwarder source;
7985 video_stream_encoder_->SetSource(
7986 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
7987 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
7988 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
7989
7990 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
7991 WaitForEncodedFrame(1);
7992 video_stream_encoder_->TriggerQualityLow();
7993 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
7994
7995 video_stream_encoder_->Stop();
7996}
7997
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08007998TEST_F(VideoStreamEncoderTest, QualityScalingNotAllowed_IsQpTrustedSetTrue) {
7999 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8000
8001 // Disable scaling settings in encoder info.
8002 fake_encoder_.SetQualityScaling(false);
8003 // Set QP trusted in encoder info.
8004 fake_encoder_.SetIsQpTrusted(true);
8005 // Enable quality scaling in encoder config.
8006 video_encoder_config.is_quality_scaling_allowed = false;
8007 ConfigureEncoder(std::move(video_encoder_config));
8008
8009 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008010 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08008011
8012 test::FrameForwarder source;
8013 video_stream_encoder_->SetSource(
8014 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8015 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8016 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8017
8018 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8019 WaitForEncodedFrame(1);
8020 video_stream_encoder_->TriggerQualityLow();
8021 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8022
8023 video_stream_encoder_->Stop();
8024}
8025
Shuhai Pengf2707702021-09-29 17:19:44 +08008026TEST_F(VideoStreamEncoderTest,
8027 QualityScalingNotAllowedAndQPIsTrusted_BandwidthScalerDisable) {
8028 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8029
8030 // Disable scaling settings in encoder info.
8031 fake_encoder_.SetQualityScaling(false);
8032 // Set QP trusted in encoder info.
8033 fake_encoder_.SetIsQpTrusted(true);
8034 // Enable quality scaling in encoder config.
8035 video_encoder_config.is_quality_scaling_allowed = false;
8036 ConfigureEncoder(std::move(video_encoder_config));
8037
8038 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008039 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008040
8041 test::FrameForwarder source;
8042 video_stream_encoder_->SetSource(
8043 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8044 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8045 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8046
8047 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8048 WaitForEncodedFrame(1);
8049 video_stream_encoder_->TriggerQualityLow();
8050 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8051
8052 video_stream_encoder_->Stop();
8053}
8054
8055TEST_F(VideoStreamEncoderTest,
8056 QualityScalingNotAllowedAndQPIsNotTrusted_BandwidthScalerDisable) {
8057 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8058
8059 // Disable scaling settings in encoder info.
8060 fake_encoder_.SetQualityScaling(false);
8061 // Set QP trusted in encoder info.
8062 fake_encoder_.SetIsQpTrusted(false);
8063 // Enable quality scaling in encoder config.
8064 video_encoder_config.is_quality_scaling_allowed = false;
8065 ConfigureEncoder(std::move(video_encoder_config));
8066
8067 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008068 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008069
8070 test::FrameForwarder source;
8071 video_stream_encoder_->SetSource(
8072 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8073 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8074 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8075
8076 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8077 WaitForEncodedFrame(1);
8078 video_stream_encoder_->TriggerQualityLow();
8079 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8080
8081 video_stream_encoder_->Stop();
8082}
8083
8084TEST_F(VideoStreamEncoderTest, EncoderProvideLimitsWhenQPIsNotTrusted) {
8085 // Set QP trusted in encoder info.
8086 fake_encoder_.SetIsQpTrusted(false);
8087
8088 const int MinEncBitrateKbps = 30;
8089 const int MaxEncBitrateKbps = 100;
8090 const int MinStartBitrateKbp = 50;
8091 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
8092 /*frame_size_pixels=*/codec_width_ * codec_height_,
8093 /*min_start_bitrate_bps=*/MinStartBitrateKbp,
8094 /*min_bitrate_bps=*/MinEncBitrateKbps * 1000,
8095 /*max_bitrate_bps=*/MaxEncBitrateKbps * 1000);
8096
8097 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008098 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008099
8100 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
8101
8102 VideoEncoderConfig video_encoder_config;
8103 test::FillEncoderConfiguration(kVideoCodecH264, 1, &video_encoder_config);
8104 video_encoder_config.max_bitrate_bps = MaxEncBitrateKbps * 1000;
8105 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
8106 MinEncBitrateKbps * 1000;
8107 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
8108 kMaxPayloadLength);
8109
8110 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
8111 WaitForEncodedFrame(1);
8112 EXPECT_EQ(
8113 MaxEncBitrateKbps,
8114 static_cast<int>(bitrate_allocator_factory_.codec_config().maxBitrate));
8115 EXPECT_EQ(
8116 MinEncBitrateKbps,
8117 static_cast<int>(bitrate_allocator_factory_.codec_config().minBitrate));
8118
8119 video_stream_encoder_->Stop();
8120}
8121
8122TEST_F(VideoStreamEncoderTest, EncoderDoesnotProvideLimitsWhenQPIsNotTrusted) {
8123 // Set QP trusted in encoder info.
8124 fake_encoder_.SetIsQpTrusted(false);
8125
8126 absl::optional<VideoEncoder::ResolutionBitrateLimits> suitable_bitrate_limit =
8127 EncoderInfoSettings::
8128 GetSinglecastBitrateLimitForResolutionWhenQpIsUntrusted(
8129 codec_width_ * codec_height_,
8130 EncoderInfoSettings::
8131 GetDefaultSinglecastBitrateLimitsWhenQpIsUntrusted());
8132 EXPECT_TRUE(suitable_bitrate_limit.has_value());
8133
8134 const int MaxEncBitrate = suitable_bitrate_limit->max_bitrate_bps;
8135 const int MinEncBitrate = suitable_bitrate_limit->min_bitrate_bps;
8136 const int TargetEncBitrate = MaxEncBitrate;
8137 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8138 DataRate::BitsPerSec(TargetEncBitrate),
8139 DataRate::BitsPerSec(TargetEncBitrate),
8140 DataRate::BitsPerSec(TargetEncBitrate), 0, 0, 0);
8141
8142 VideoEncoderConfig video_encoder_config;
8143 test::FillEncoderConfiguration(kVideoCodecH264, 1, &video_encoder_config);
8144 video_encoder_config.max_bitrate_bps = MaxEncBitrate;
8145 video_encoder_config.simulcast_layers[0].min_bitrate_bps = MinEncBitrate;
8146 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
8147 kMaxPayloadLength);
8148
8149 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
8150 WaitForEncodedFrame(1);
8151 EXPECT_EQ(
8152 MaxEncBitrate / 1000,
8153 static_cast<int>(bitrate_allocator_factory_.codec_config().maxBitrate));
8154 EXPECT_EQ(
8155 MinEncBitrate / 1000,
8156 static_cast<int>(bitrate_allocator_factory_.codec_config().minBitrate));
8157
8158 video_stream_encoder_->Stop();
8159}
8160
Sergey Silkind19e3b92021-03-16 10:05:30 +00008161#if !defined(WEBRTC_IOS)
8162// TODO(bugs.webrtc.org/12401): Disabled because WebRTC-Video-QualityScaling is
8163// disabled by default on iOS.
8164TEST_F(VideoStreamEncoderTest, QualityScalingAllowed_QualityScalingEnabled) {
8165 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8166
8167 // Disable scaling settings in encoder info.
8168 fake_encoder_.SetQualityScaling(false);
8169 // Enable quality scaling in encoder config.
8170 video_encoder_config.is_quality_scaling_allowed = true;
8171 ConfigureEncoder(std::move(video_encoder_config));
8172
8173 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008174 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkind19e3b92021-03-16 10:05:30 +00008175
8176 test::FrameForwarder source;
8177 video_stream_encoder_->SetSource(
8178 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8179 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8180 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8181
8182 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8183 WaitForEncodedFrame(1);
8184 video_stream_encoder_->TriggerQualityLow();
8185 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8186
8187 video_stream_encoder_->Stop();
8188}
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08008189
8190TEST_F(VideoStreamEncoderTest, QualityScalingAllowed_IsQpTrustedSetTrue) {
8191 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8192
8193 // Disable scaling settings in encoder info.
8194 fake_encoder_.SetQualityScaling(false);
8195 // Set QP trusted in encoder info.
8196 fake_encoder_.SetIsQpTrusted(true);
8197 // Enable quality scaling in encoder config.
8198 video_encoder_config.is_quality_scaling_allowed = true;
8199 ConfigureEncoder(std::move(video_encoder_config));
8200
8201 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008202 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08008203
8204 test::FrameForwarder source;
8205 video_stream_encoder_->SetSource(
8206 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8207 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8208 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8209
8210 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8211 WaitForEncodedFrame(1);
8212 video_stream_encoder_->TriggerQualityLow();
8213 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8214
8215 video_stream_encoder_->Stop();
8216}
Shuhai Pengf2707702021-09-29 17:19:44 +08008217
8218TEST_F(VideoStreamEncoderTest, QualityScalingAllowed_IsQpTrustedSetFalse) {
8219 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8220
8221 // Disable scaling settings in encoder info.
8222 fake_encoder_.SetQualityScaling(false);
8223 // Set QP not trusted in encoder info.
8224 fake_encoder_.SetIsQpTrusted(false);
8225 // Enable quality scaling in encoder config.
8226 video_encoder_config.is_quality_scaling_allowed = true;
8227 ConfigureEncoder(std::move(video_encoder_config));
8228
8229 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008230 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008231
8232 test::FrameForwarder source;
8233 video_stream_encoder_->SetSource(
8234 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8235 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8236 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8237
8238 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8239 WaitForEncodedFrame(1);
8240 video_stream_encoder_->TriggerQualityLow();
8241 // When quality_scaler doesn't work and is_quality_scaling_allowed is
8242 // true,the bandwidth_quality_scaler_ works,so bw_limited_resolution is true.
8243 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8244
8245 video_stream_encoder_->Stop();
8246}
8247
8248TEST_F(VideoStreamEncoderTest,
8249 QualityScalingAllowedAndQPIsTrusted_BandwidthScalerDisable) {
8250 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8251
8252 // Disable scaling settings in encoder info.
8253 fake_encoder_.SetQualityScaling(false);
8254 // Set QP trusted in encoder info.
8255 fake_encoder_.SetIsQpTrusted(true);
8256 // Enable quality scaling in encoder config.
8257 video_encoder_config.is_quality_scaling_allowed = true;
8258 ConfigureEncoder(std::move(video_encoder_config));
8259
8260 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008261 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008262
8263 test::FrameForwarder source;
8264 video_stream_encoder_->SetSource(
8265 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8266 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8267 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8268
8269 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8270 WaitForEncodedFrame(1);
8271 video_stream_encoder_->TriggerQualityLow();
8272 // bandwidth_quality_scaler isn't working, but quality_scaler is working.
8273 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8274
8275 video_stream_encoder_->Stop();
8276}
8277
8278TEST_F(VideoStreamEncoderTest,
8279 QualityScalingAllowedAndQPIsNotTrusted_BandwidthScalerEnabled) {
8280 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8281
8282 // Disable scaling settings in encoder info.
8283 fake_encoder_.SetQualityScaling(false);
8284 // Set QP trusted in encoder info.
8285 fake_encoder_.SetIsQpTrusted(false);
8286 // Enable quality scaling in encoder config.
8287 video_encoder_config.is_quality_scaling_allowed = true;
8288 ConfigureEncoder(std::move(video_encoder_config));
8289
8290 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008291 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008292
8293 test::FrameForwarder source;
8294 video_stream_encoder_->SetSource(
8295 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8296 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8297 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8298
8299 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8300 WaitForEncodedFrame(1);
8301 video_stream_encoder_->TriggerQualityLow();
8302 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8303
8304 video_stream_encoder_->Stop();
8305}
8306
Sergey Silkind19e3b92021-03-16 10:05:30 +00008307#endif
8308
Henrik Boström56db9ff2021-03-24 09:06:45 +01008309// Test parameters: (VideoCodecType codec, bool allow_i420_conversion)
8310class VideoStreamEncoderWithRealEncoderTest
8311 : public VideoStreamEncoderTest,
8312 public ::testing::WithParamInterface<std::pair<VideoCodecType, bool>> {
8313 public:
8314 VideoStreamEncoderWithRealEncoderTest()
8315 : VideoStreamEncoderTest(),
8316 codec_type_(std::get<0>(GetParam())),
8317 allow_i420_conversion_(std::get<1>(GetParam())) {}
8318
8319 void SetUp() override {
8320 VideoStreamEncoderTest::SetUp();
8321 std::unique_ptr<VideoEncoder> encoder;
8322 switch (codec_type_) {
8323 case kVideoCodecVP8:
8324 encoder = VP8Encoder::Create();
8325 break;
8326 case kVideoCodecVP9:
8327 encoder = VP9Encoder::Create();
8328 break;
8329 case kVideoCodecAV1:
8330 encoder = CreateLibaomAv1Encoder();
8331 break;
8332 case kVideoCodecH264:
8333 encoder =
8334 H264Encoder::Create(cricket::VideoCodec(cricket::kH264CodecName));
8335 break;
8336 case kVideoCodecMultiplex:
8337 mock_encoder_factory_for_multiplex_ =
8338 std::make_unique<MockVideoEncoderFactory>();
8339 EXPECT_CALL(*mock_encoder_factory_for_multiplex_, Die);
8340 EXPECT_CALL(*mock_encoder_factory_for_multiplex_, CreateVideoEncoder)
8341 .WillRepeatedly([] { return VP8Encoder::Create(); });
8342 encoder = std::make_unique<MultiplexEncoderAdapter>(
8343 mock_encoder_factory_for_multiplex_.get(), SdpVideoFormat("VP8"),
8344 false);
8345 break;
8346 default:
Artem Titovd3251962021-11-15 16:57:07 +01008347 RTC_DCHECK_NOTREACHED();
Henrik Boström56db9ff2021-03-24 09:06:45 +01008348 }
8349 ConfigureEncoderAndBitrate(codec_type_, std::move(encoder));
8350 }
8351
8352 void TearDown() override {
8353 video_stream_encoder_->Stop();
Artem Titovab30d722021-07-27 16:22:11 +02008354 // Ensure `video_stream_encoder_` is destroyed before
8355 // `encoder_proxy_factory_`.
Henrik Boström56db9ff2021-03-24 09:06:45 +01008356 video_stream_encoder_.reset();
8357 VideoStreamEncoderTest::TearDown();
8358 }
8359
8360 protected:
8361 void ConfigureEncoderAndBitrate(VideoCodecType codec_type,
8362 std::unique_ptr<VideoEncoder> encoder) {
8363 // Configure VSE to use the encoder.
8364 encoder_ = std::move(encoder);
8365 encoder_proxy_factory_ = std::make_unique<test::VideoEncoderProxyFactory>(
8366 encoder_.get(), &encoder_selector_);
8367 video_send_config_.encoder_settings.encoder_factory =
8368 encoder_proxy_factory_.get();
8369 VideoEncoderConfig video_encoder_config;
8370 test::FillEncoderConfiguration(codec_type, 1, &video_encoder_config);
8371 video_encoder_config_ = video_encoder_config.Copy();
8372 ConfigureEncoder(video_encoder_config_.Copy());
8373
8374 // Set bitrate to ensure frame is not dropped.
8375 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008376 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Henrik Boström56db9ff2021-03-24 09:06:45 +01008377 }
8378
8379 const VideoCodecType codec_type_;
8380 const bool allow_i420_conversion_;
8381 NiceMock<MockEncoderSelector> encoder_selector_;
8382 std::unique_ptr<test::VideoEncoderProxyFactory> encoder_proxy_factory_;
8383 std::unique_ptr<VideoEncoder> encoder_;
8384 std::unique_ptr<MockVideoEncoderFactory> mock_encoder_factory_for_multiplex_;
8385};
8386
8387TEST_P(VideoStreamEncoderWithRealEncoderTest, EncoderMapsNativeI420) {
8388 auto native_i420_frame = test::CreateMappableNativeFrame(
8389 1, VideoFrameBuffer::Type::kI420, codec_width_, codec_height_);
8390 video_source_.IncomingCapturedFrame(native_i420_frame);
8391 WaitForEncodedFrame(codec_width_, codec_height_);
8392
8393 auto mappable_native_buffer =
8394 test::GetMappableNativeBufferFromVideoFrame(native_i420_frame);
8395 std::vector<rtc::scoped_refptr<VideoFrameBuffer>> mapped_frame_buffers =
8396 mappable_native_buffer->GetMappedFramedBuffers();
8397 ASSERT_EQ(mapped_frame_buffers.size(), 1u);
8398 EXPECT_EQ(mapped_frame_buffers[0]->width(), codec_width_);
8399 EXPECT_EQ(mapped_frame_buffers[0]->height(), codec_height_);
8400 EXPECT_EQ(mapped_frame_buffers[0]->type(), VideoFrameBuffer::Type::kI420);
8401}
8402
8403TEST_P(VideoStreamEncoderWithRealEncoderTest, EncoderMapsNativeNV12) {
8404 auto native_nv12_frame = test::CreateMappableNativeFrame(
8405 1, VideoFrameBuffer::Type::kNV12, codec_width_, codec_height_);
8406 video_source_.IncomingCapturedFrame(native_nv12_frame);
8407 WaitForEncodedFrame(codec_width_, codec_height_);
8408
8409 auto mappable_native_buffer =
8410 test::GetMappableNativeBufferFromVideoFrame(native_nv12_frame);
8411 std::vector<rtc::scoped_refptr<VideoFrameBuffer>> mapped_frame_buffers =
8412 mappable_native_buffer->GetMappedFramedBuffers();
8413 ASSERT_EQ(mapped_frame_buffers.size(), 1u);
8414 EXPECT_EQ(mapped_frame_buffers[0]->width(), codec_width_);
8415 EXPECT_EQ(mapped_frame_buffers[0]->height(), codec_height_);
8416 EXPECT_EQ(mapped_frame_buffers[0]->type(), VideoFrameBuffer::Type::kNV12);
8417
8418 if (!allow_i420_conversion_) {
8419 EXPECT_FALSE(mappable_native_buffer->DidConvertToI420());
8420 }
8421}
8422
Erik Språng7444b192021-06-02 14:02:13 +02008423TEST_P(VideoStreamEncoderWithRealEncoderTest, HandlesLayerToggling) {
8424 if (codec_type_ == kVideoCodecMultiplex) {
8425 // Multiplex codec here uses wrapped mock codecs, ignore for this test.
8426 return;
8427 }
8428
8429 const size_t kNumSpatialLayers = 3u;
8430 const float kDownscaleFactors[] = {4.0, 2.0, 1.0};
8431 const int kFrameWidth = 1280;
8432 const int kFrameHeight = 720;
8433 const rtc::VideoSinkWants::FrameSize kLayer0Size(
8434 kFrameWidth / kDownscaleFactors[0], kFrameHeight / kDownscaleFactors[0]);
8435 const rtc::VideoSinkWants::FrameSize kLayer1Size(
8436 kFrameWidth / kDownscaleFactors[1], kFrameHeight / kDownscaleFactors[1]);
8437 const rtc::VideoSinkWants::FrameSize kLayer2Size(
8438 kFrameWidth / kDownscaleFactors[2], kFrameHeight / kDownscaleFactors[2]);
8439
8440 VideoEncoderConfig config;
8441 if (codec_type_ == VideoCodecType::kVideoCodecVP9) {
8442 test::FillEncoderConfiguration(codec_type_, 1, &config);
Asa Persson606d3cb2021-10-04 10:07:11 +02008443 config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Erik Språng7444b192021-06-02 14:02:13 +02008444 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
8445 vp9_settings.numberOfSpatialLayers = kNumSpatialLayers;
8446 vp9_settings.numberOfTemporalLayers = 3;
8447 vp9_settings.automaticResizeOn = false;
8448 config.encoder_specific_settings =
8449 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
8450 vp9_settings);
8451 config.spatial_layers = GetSvcConfig(kFrameWidth, kFrameHeight,
8452 /*fps=*/30.0,
8453 /*first_active_layer=*/0,
8454 /*num_spatial_layers=*/3,
8455 /*num_temporal_layers=*/3,
8456 /*is_screenshare=*/false);
8457 } else if (codec_type_ == VideoCodecType::kVideoCodecAV1) {
8458 test::FillEncoderConfiguration(codec_type_, 1, &config);
Asa Persson606d3cb2021-10-04 10:07:11 +02008459 config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Erik Språng7444b192021-06-02 14:02:13 +02008460 config.spatial_layers = GetSvcConfig(kFrameWidth, kFrameHeight,
8461 /*fps=*/30.0,
8462 /*first_active_layer=*/0,
8463 /*num_spatial_layers=*/3,
8464 /*num_temporal_layers=*/3,
8465 /*is_screenshare=*/false);
8466 config.simulcast_layers[0].scalability_mode = "L3T3_KEY";
8467 } else {
8468 // Simulcast for VP8/H264.
8469 test::FillEncoderConfiguration(codec_type_, kNumSpatialLayers, &config);
8470 for (size_t i = 0; i < kNumSpatialLayers; ++i) {
8471 config.simulcast_layers[i].scale_resolution_down_by =
8472 kDownscaleFactors[i];
8473 config.simulcast_layers[i].active = true;
8474 }
8475 if (codec_type_ == VideoCodecType::kVideoCodecH264) {
8476 // Turn off frame dropping to prevent flakiness.
8477 VideoCodecH264 h264_settings = VideoEncoder::GetDefaultH264Settings();
8478 h264_settings.frameDroppingOn = false;
8479 config.encoder_specific_settings = rtc::make_ref_counted<
8480 VideoEncoderConfig::H264EncoderSpecificSettings>(h264_settings);
8481 }
8482 }
8483
8484 auto set_layer_active = [&](int layer_idx, bool active) {
8485 if (codec_type_ == VideoCodecType::kVideoCodecVP9 ||
8486 codec_type_ == VideoCodecType::kVideoCodecAV1) {
8487 config.spatial_layers[layer_idx].active = active;
8488 } else {
8489 config.simulcast_layers[layer_idx].active = active;
8490 }
8491 };
8492
8493 config.video_stream_factory =
8494 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
8495 CodecTypeToPayloadString(codec_type_), /*max qp*/ 56,
8496 /*screencast*/ false,
8497 /*screenshare enabled*/ false);
8498 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008499 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
8500 0, 0, 0);
Erik Språng7444b192021-06-02 14:02:13 +02008501
8502 // Capture a frame with all layers active.
8503 sink_.SetNumExpectedLayers(kNumSpatialLayers);
8504 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8505 int64_t timestamp_ms = kFrameIntervalMs;
8506 video_source_.IncomingCapturedFrame(
8507 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8508
8509 WaitForEncodedFrame(kLayer2Size.width, kLayer2Size.height);
8510 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8511
8512 // Capture a frame with one of the layers inactive.
8513 set_layer_active(2, false);
8514 sink_.SetNumExpectedLayers(kNumSpatialLayers - 1);
8515 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8516 timestamp_ms += kFrameIntervalMs;
8517 video_source_.IncomingCapturedFrame(
8518 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8519 WaitForEncodedFrame(kLayer1Size.width, kLayer1Size.height);
8520
8521 // New target bitrates signaled based on lower resolution.
8522 DataRate kTwoLayerBitrate = DataRate::KilobitsPerSec(833);
8523 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8524 kTwoLayerBitrate, kTwoLayerBitrate, kTwoLayerBitrate, 0, 0, 0);
8525 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8526
8527 // Re-enable the top layer.
8528 set_layer_active(2, true);
8529 sink_.SetNumExpectedLayers(kNumSpatialLayers);
8530 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8531 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8532
8533 // Bitrate target adjusted back up to enable HD layer...
8534 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8535 DataRate::KilobitsPerSec(1800), DataRate::KilobitsPerSec(1800),
8536 DataRate::KilobitsPerSec(1800), 0, 0, 0);
8537 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8538
8539 // ...then add a new frame.
8540 timestamp_ms += kFrameIntervalMs;
8541 video_source_.IncomingCapturedFrame(
8542 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8543 WaitForEncodedFrame(kLayer2Size.width, kLayer2Size.height);
8544 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8545
8546 video_stream_encoder_->Stop();
8547}
8548
Henrik Boström56db9ff2021-03-24 09:06:45 +01008549std::string TestParametersVideoCodecAndAllowI420ConversionToString(
8550 testing::TestParamInfo<std::pair<VideoCodecType, bool>> info) {
8551 VideoCodecType codec_type = std::get<0>(info.param);
8552 bool allow_i420_conversion = std::get<1>(info.param);
8553 std::string str;
8554 switch (codec_type) {
8555 case kVideoCodecGeneric:
8556 str = "Generic";
8557 break;
8558 case kVideoCodecVP8:
8559 str = "VP8";
8560 break;
8561 case kVideoCodecVP9:
8562 str = "VP9";
8563 break;
8564 case kVideoCodecAV1:
8565 str = "AV1";
8566 break;
8567 case kVideoCodecH264:
8568 str = "H264";
8569 break;
8570 case kVideoCodecMultiplex:
8571 str = "Multiplex";
8572 break;
8573 default:
Artem Titovd3251962021-11-15 16:57:07 +01008574 RTC_DCHECK_NOTREACHED();
Henrik Boström56db9ff2021-03-24 09:06:45 +01008575 }
8576 str += allow_i420_conversion ? "_AllowToI420" : "_DisallowToI420";
8577 return str;
8578}
8579
8580constexpr std::pair<VideoCodecType, bool> kVP8DisallowConversion =
8581 std::make_pair(kVideoCodecVP8, /*allow_i420_conversion=*/false);
8582constexpr std::pair<VideoCodecType, bool> kVP9DisallowConversion =
8583 std::make_pair(kVideoCodecVP9, /*allow_i420_conversion=*/false);
8584constexpr std::pair<VideoCodecType, bool> kAV1AllowConversion =
8585 std::make_pair(kVideoCodecAV1, /*allow_i420_conversion=*/true);
8586constexpr std::pair<VideoCodecType, bool> kMultiplexDisallowConversion =
8587 std::make_pair(kVideoCodecMultiplex, /*allow_i420_conversion=*/false);
8588#if defined(WEBRTC_USE_H264)
8589constexpr std::pair<VideoCodecType, bool> kH264AllowConversion =
8590 std::make_pair(kVideoCodecH264, /*allow_i420_conversion=*/true);
8591
8592// The windows compiler does not tolerate #if statements inside the
8593// INSTANTIATE_TEST_SUITE_P() macro, so we have to have two definitions (with
8594// and without H264).
8595INSTANTIATE_TEST_SUITE_P(
8596 All,
8597 VideoStreamEncoderWithRealEncoderTest,
8598 ::testing::Values(kVP8DisallowConversion,
8599 kVP9DisallowConversion,
8600 kAV1AllowConversion,
8601 kMultiplexDisallowConversion,
8602 kH264AllowConversion),
8603 TestParametersVideoCodecAndAllowI420ConversionToString);
8604#else
8605INSTANTIATE_TEST_SUITE_P(
8606 All,
8607 VideoStreamEncoderWithRealEncoderTest,
8608 ::testing::Values(kVP8DisallowConversion,
8609 kVP9DisallowConversion,
8610 kAV1AllowConversion,
8611 kMultiplexDisallowConversion),
8612 TestParametersVideoCodecAndAllowI420ConversionToString);
8613#endif
8614
Åsa Persson4d4f62f2021-09-12 16:14:48 +02008615class ReconfigureEncoderTest : public VideoStreamEncoderTest {
8616 protected:
8617 void RunTest(const std::vector<VideoStream>& configs,
8618 const int expected_num_init_encode) {
8619 ConfigureEncoder(configs[0]);
Asa Persson606d3cb2021-10-04 10:07:11 +02008620 OnBitrateUpdated(kTargetBitrate);
Åsa Persson4d4f62f2021-09-12 16:14:48 +02008621 InsertFrameAndWaitForEncoded();
8622 EXPECT_EQ(1, sink_.number_of_reconfigurations());
8623 ExpectEqual(bitrate_allocator_factory_.codec_config(), configs[0]);
Asa Persson606d3cb2021-10-04 10:07:11 +02008624 EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
8625 ExpectEqual(fake_encoder_.config(), configs[0]);
Åsa Persson4d4f62f2021-09-12 16:14:48 +02008626
8627 // Reconfigure encoder, the encoder should only be reconfigured if needed.
8628 ConfigureEncoder(configs[1]);
8629 InsertFrameAndWaitForEncoded();
8630 EXPECT_EQ(2, sink_.number_of_reconfigurations());
8631 ExpectEqual(bitrate_allocator_factory_.codec_config(), configs[1]);
Asa Persson606d3cb2021-10-04 10:07:11 +02008632 EXPECT_EQ(expected_num_init_encode, fake_encoder_.GetNumInitializations());
Åsa Persson4d4f62f2021-09-12 16:14:48 +02008633 if (expected_num_init_encode > 1)
Asa Persson606d3cb2021-10-04 10:07:11 +02008634 ExpectEqual(fake_encoder_.config(), configs[1]);
Åsa Persson4d4f62f2021-09-12 16:14:48 +02008635
8636 video_stream_encoder_->Stop();
8637 }
8638
8639 void ConfigureEncoder(const VideoStream& stream) {
8640 VideoEncoderConfig config;
8641 test::FillEncoderConfiguration(kVideoCodecVP8, /*num_streams=*/1, &config);
8642 config.max_bitrate_bps = stream.max_bitrate_bps;
8643 config.simulcast_layers[0] = stream;
8644 config.video_stream_factory =
8645 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
8646 /*codec_name=*/"VP8", /*max_qp=*/0, /*is_screenshare=*/false,
8647 /*conference_mode=*/false);
8648 video_stream_encoder_->ConfigureEncoder(std::move(config),
8649 kMaxPayloadLength);
8650 }
8651
8652 void OnBitrateUpdated(DataRate bitrate) {
8653 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8654 bitrate, bitrate, bitrate, 0, 0, 0);
8655 }
8656
8657 void InsertFrameAndWaitForEncoded() {
8658 timestamp_ms_ += kFrameIntervalMs;
8659 video_source_.IncomingCapturedFrame(
8660 CreateFrame(timestamp_ms_, kWidth, kHeight));
8661 sink_.WaitForEncodedFrame(timestamp_ms_);
8662 }
8663
8664 void ExpectEqual(const VideoCodec& actual,
8665 const VideoStream& expected) const {
8666 EXPECT_EQ(actual.numberOfSimulcastStreams, 1);
8667 EXPECT_EQ(actual.simulcastStream[0].maxFramerate, expected.max_framerate);
8668 EXPECT_EQ(actual.simulcastStream[0].minBitrate * 1000,
8669 static_cast<unsigned int>(expected.min_bitrate_bps));
8670 EXPECT_EQ(actual.simulcastStream[0].maxBitrate * 1000,
8671 static_cast<unsigned int>(expected.max_bitrate_bps));
8672 EXPECT_EQ(actual.simulcastStream[0].width,
8673 kWidth / expected.scale_resolution_down_by);
8674 EXPECT_EQ(actual.simulcastStream[0].height,
8675 kHeight / expected.scale_resolution_down_by);
8676 EXPECT_EQ(actual.simulcastStream[0].numberOfTemporalLayers,
8677 expected.num_temporal_layers);
8678 EXPECT_EQ(actual.ScalabilityMode(), expected.scalability_mode);
8679 }
8680
8681 VideoStream DefaultConfig() const {
8682 VideoStream stream;
8683 stream.max_framerate = 25;
8684 stream.min_bitrate_bps = 35000;
8685 stream.max_bitrate_bps = 900000;
8686 stream.scale_resolution_down_by = 1.0;
8687 stream.num_temporal_layers = 1;
8688 stream.bitrate_priority = 1.0;
8689 stream.scalability_mode = "";
8690 return stream;
8691 }
8692
8693 const int kWidth = 640;
8694 const int kHeight = 360;
8695 int64_t timestamp_ms_ = 0;
8696};
8697
8698TEST_F(ReconfigureEncoderTest, NotReconfiguredIfMaxFramerateChanges) {
8699 VideoStream config1 = DefaultConfig();
8700 VideoStream config2 = config1;
8701 config2.max_framerate++;
8702
8703 RunTest({config1, config2}, /*expected_num_init_encode=*/1);
8704}
8705
8706TEST_F(ReconfigureEncoderTest, NotReconfiguredIfMinBitrateChanges) {
8707 VideoStream config1 = DefaultConfig();
8708 VideoStream config2 = config1;
8709 config2.min_bitrate_bps += 10000;
8710
8711 RunTest({config1, config2}, /*expected_num_init_encode=*/1);
8712}
8713
8714TEST_F(ReconfigureEncoderTest, NotReconfiguredIfMaxBitrateChanges) {
8715 VideoStream config1 = DefaultConfig();
8716 VideoStream config2 = config1;
8717 config2.max_bitrate_bps += 100000;
8718
8719 RunTest({config1, config2}, /*expected_num_init_encode=*/1);
8720}
8721
8722TEST_F(ReconfigureEncoderTest, NotReconfiguredIfBitratePriorityChanges) {
8723 VideoStream config1 = DefaultConfig();
8724 VideoStream config2 = config1;
8725 config2.bitrate_priority = config1.bitrate_priority.value() * 2.0;
8726
8727 RunTest({config1, config2}, /*expected_num_init_encode=*/1);
8728}
8729
8730TEST_F(ReconfigureEncoderTest, ReconfiguredIfResolutionChanges) {
8731 VideoStream config1 = DefaultConfig();
8732 VideoStream config2 = config1;
8733 config2.scale_resolution_down_by *= 2;
8734
8735 RunTest({config1, config2}, /*expected_num_init_encode=*/2);
8736}
8737
8738TEST_F(ReconfigureEncoderTest, ReconfiguredIfNumTemporalLayerChanges) {
8739 VideoStream config1 = DefaultConfig();
8740 VideoStream config2 = config1;
8741 config2.num_temporal_layers = config1.num_temporal_layers.value() + 1;
8742
8743 RunTest({config1, config2}, /*expected_num_init_encode=*/2);
8744}
8745
8746TEST_F(ReconfigureEncoderTest, ReconfiguredIfScalabilityModeChanges) {
8747 VideoStream config1 = DefaultConfig();
8748 VideoStream config2 = config1;
8749 config2.scalability_mode = "L1T2";
8750
8751 RunTest({config1, config2}, /*expected_num_init_encode=*/2);
8752}
8753
Markus Handellb4e96d42021-11-05 12:00:55 +01008754TEST(VideoStreamEncoderFrameCadenceTest, ActivatesFrameCadenceOnContentType) {
8755 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
8756 auto* adapter_ptr = adapter.get();
8757 SimpleVideoStreamEncoderFactory factory;
Markus Handell8d87c462021-12-16 11:37:16 +01008758 FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback =
8759 nullptr;
8760 EXPECT_CALL(*adapter_ptr, Initialize)
8761 .WillOnce(Invoke([&video_stream_encoder_callback](
8762 FrameCadenceAdapterInterface::Callback* callback) {
8763 video_stream_encoder_callback = callback;
8764 }));
8765 TaskQueueBase* encoder_queue = nullptr;
8766 auto video_stream_encoder =
8767 factory.Create(std::move(adapter), &encoder_queue);
Markus Handellb4e96d42021-11-05 12:00:55 +01008768
Markus Handell8d87c462021-12-16 11:37:16 +01008769 EXPECT_CALL(*adapter_ptr, SetZeroHertzModeEnabled(Not(Eq(absl::nullopt))));
Markus Handellb4e96d42021-11-05 12:00:55 +01008770 VideoEncoderConfig config;
Markus Handell8d87c462021-12-16 11:37:16 +01008771 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &config);
Markus Handellb4e96d42021-11-05 12:00:55 +01008772 config.content_type = VideoEncoderConfig::ContentType::kScreen;
8773 video_stream_encoder->ConfigureEncoder(std::move(config), 0);
Markus Handell8d87c462021-12-16 11:37:16 +01008774 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/1);
Markus Handell9a478b52021-11-18 16:07:01 +01008775 factory.DepleteTaskQueues();
Markus Handellb4e96d42021-11-05 12:00:55 +01008776 Mock::VerifyAndClearExpectations(adapter_ptr);
8777
Markus Handell8d87c462021-12-16 11:37:16 +01008778 EXPECT_CALL(*adapter_ptr, SetZeroHertzModeEnabled(Eq(absl::nullopt)));
Markus Handellb4e96d42021-11-05 12:00:55 +01008779 VideoEncoderConfig config2;
Markus Handell8d87c462021-12-16 11:37:16 +01008780 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &config2);
Markus Handellb4e96d42021-11-05 12:00:55 +01008781 config2.content_type = VideoEncoderConfig::ContentType::kRealtimeVideo;
8782 video_stream_encoder->ConfigureEncoder(std::move(config2), 0);
Markus Handell8d87c462021-12-16 11:37:16 +01008783 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/2);
Markus Handell9a478b52021-11-18 16:07:01 +01008784 factory.DepleteTaskQueues();
Markus Handellb4e96d42021-11-05 12:00:55 +01008785}
8786
8787TEST(VideoStreamEncoderFrameCadenceTest,
8788 ForwardsFramesIntoFrameCadenceAdapter) {
8789 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
8790 auto* adapter_ptr = adapter.get();
8791 test::FrameForwarder video_source;
8792 SimpleVideoStreamEncoderFactory factory;
8793 auto video_stream_encoder = factory.Create(std::move(adapter));
8794 video_stream_encoder->SetSource(
8795 &video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8796
8797 EXPECT_CALL(*adapter_ptr, OnFrame);
8798 auto buffer = rtc::make_ref_counted<NV12Buffer>(/*width=*/16, /*height=*/16);
8799 video_source.IncomingCapturedFrame(
Markus Handell8d87c462021-12-16 11:37:16 +01008800 VideoFrame::Builder().set_video_frame_buffer(std::move(buffer)).build());
Markus Handellb4e96d42021-11-05 12:00:55 +01008801}
8802
Markus Handellee225432021-11-29 12:35:12 +01008803TEST(VideoStreamEncoderFrameCadenceTest, UsesFrameCadenceAdapterForFrameRate) {
8804 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
8805 auto* adapter_ptr = adapter.get();
8806 test::FrameForwarder video_source;
8807 SimpleVideoStreamEncoderFactory factory;
8808 FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback =
8809 nullptr;
8810 EXPECT_CALL(*adapter_ptr, Initialize)
8811 .WillOnce(Invoke([&video_stream_encoder_callback](
8812 FrameCadenceAdapterInterface::Callback* callback) {
8813 video_stream_encoder_callback = callback;
8814 }));
8815 TaskQueueBase* encoder_queue = nullptr;
8816 auto video_stream_encoder =
8817 factory.Create(std::move(adapter), &encoder_queue);
8818
8819 // This is just to make the VSE operational. We'll feed a frame directly by
8820 // the callback interface.
8821 video_stream_encoder->SetSource(
8822 &video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8823
8824 VideoEncoderConfig video_encoder_config;
8825 test::FillEncoderConfiguration(kVideoCodecGeneric, 1, &video_encoder_config);
8826 video_stream_encoder->ConfigureEncoder(std::move(video_encoder_config),
8827 /*max_data_payload_length=*/1000);
8828
8829 EXPECT_CALL(*adapter_ptr, GetInputFrameRateFps);
8830 EXPECT_CALL(*adapter_ptr, UpdateFrameRate);
Markus Handell8d87c462021-12-16 11:37:16 +01008831 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/1);
Markus Handellee225432021-11-29 12:35:12 +01008832 factory.DepleteTaskQueues();
8833}
8834
Markus Handell8d87c462021-12-16 11:37:16 +01008835TEST(VideoStreamEncoderFrameCadenceTest,
8836 DeactivatesActivatesLayersOnBitrateChanges) {
8837 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
8838 auto* adapter_ptr = adapter.get();
8839 SimpleVideoStreamEncoderFactory factory;
8840 FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback =
8841 nullptr;
8842 EXPECT_CALL(*adapter_ptr, Initialize)
8843 .WillOnce(Invoke([&video_stream_encoder_callback](
8844 FrameCadenceAdapterInterface::Callback* callback) {
8845 video_stream_encoder_callback = callback;
8846 }));
8847 TaskQueueBase* encoder_queue = nullptr;
8848 auto video_stream_encoder =
8849 factory.Create(std::move(adapter), &encoder_queue);
8850
8851 // Configure 2 simulcast layers. FillEncoderConfiguration sets min bitrates to
8852 // {150000, 450000}.
8853 VideoEncoderConfig video_encoder_config;
8854 test::FillEncoderConfiguration(kVideoCodecVP8, 2, &video_encoder_config);
8855 video_stream_encoder->ConfigureEncoder(video_encoder_config.Copy(),
8856 kMaxPayloadLength);
8857 // Ensure an encoder is created.
8858 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/1);
8859
8860 // Both layers enabled at 1 MBit/s.
8861 video_stream_encoder->OnBitrateUpdated(
8862 DataRate::KilobitsPerSec(1000), DataRate::KilobitsPerSec(1000),
8863 DataRate::KilobitsPerSec(1000), 0, 0, 0);
8864 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(0, /*enabled=*/true));
8865 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(1, /*enabled=*/true));
8866 factory.DepleteTaskQueues();
8867 Mock::VerifyAndClearExpectations(adapter_ptr);
8868
8869 // Layer 1 disabled at 200 KBit/s.
8870 video_stream_encoder->OnBitrateUpdated(
8871 DataRate::KilobitsPerSec(200), DataRate::KilobitsPerSec(200),
8872 DataRate::KilobitsPerSec(200), 0, 0, 0);
8873 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(0, /*enabled=*/true));
8874 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(1, /*enabled=*/false));
8875 factory.DepleteTaskQueues();
8876 Mock::VerifyAndClearExpectations(adapter_ptr);
8877
8878 // All layers off at suspended video.
8879 video_stream_encoder->OnBitrateUpdated(DataRate::Zero(), DataRate::Zero(),
8880 DataRate::Zero(), 0, 0, 0);
8881 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(0, /*enabled=*/false));
8882 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(1, /*enabled=*/false));
8883 factory.DepleteTaskQueues();
8884 Mock::VerifyAndClearExpectations(adapter_ptr);
8885
8886 // Both layers enabled again back at 1 MBit/s.
8887 video_stream_encoder->OnBitrateUpdated(
8888 DataRate::KilobitsPerSec(1000), DataRate::KilobitsPerSec(1000),
8889 DataRate::KilobitsPerSec(1000), 0, 0, 0);
8890 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(0, /*enabled=*/true));
8891 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(1, /*enabled=*/true));
8892 factory.DepleteTaskQueues();
8893}
8894
8895TEST(VideoStreamEncoderFrameCadenceTest, UpdatesQualityConvergence) {
8896 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
8897 auto* adapter_ptr = adapter.get();
8898 SimpleVideoStreamEncoderFactory factory;
8899 FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback =
8900 nullptr;
8901 EXPECT_CALL(*adapter_ptr, Initialize)
8902 .WillOnce(Invoke([&video_stream_encoder_callback](
8903 FrameCadenceAdapterInterface::Callback* callback) {
8904 video_stream_encoder_callback = callback;
8905 }));
8906 TaskQueueBase* encoder_queue = nullptr;
8907 auto video_stream_encoder =
8908 factory.Create(std::move(adapter), &encoder_queue);
8909
8910 // Configure 2 simulcast layers and setup 1 MBit/s to unpause the encoder.
8911 VideoEncoderConfig video_encoder_config;
8912 test::FillEncoderConfiguration(kVideoCodecVP8, 2, &video_encoder_config);
8913 video_stream_encoder->ConfigureEncoder(video_encoder_config.Copy(),
8914 kMaxPayloadLength);
8915 video_stream_encoder->OnBitrateUpdated(
8916 DataRate::KilobitsPerSec(1000), DataRate::KilobitsPerSec(1000),
8917 DataRate::KilobitsPerSec(1000), 0, 0, 0);
8918
8919 // Pass a frame which has unconverged results.
8920 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/1);
8921 EXPECT_CALL(factory.GetMockFakeEncoder(), EncodeHook)
8922 .WillRepeatedly(Invoke([](EncodedImage& encoded_image,
8923 rtc::scoped_refptr<EncodedImageBuffer> buffer) {
8924 EXPECT_FALSE(encoded_image.IsAtTargetQuality());
8925 CodecSpecificInfo codec_specific;
8926 codec_specific.codecType = kVideoCodecGeneric;
8927 return codec_specific;
8928 }));
8929 EXPECT_CALL(*adapter_ptr, UpdateLayerQualityConvergence(0, false));
8930 EXPECT_CALL(*adapter_ptr, UpdateLayerQualityConvergence(1, false));
8931 factory.DepleteTaskQueues();
8932 Mock::VerifyAndClearExpectations(adapter_ptr);
8933 Mock::VerifyAndClearExpectations(&factory.GetMockFakeEncoder());
8934
8935 // Pass a frame which converges in layer 0 and not in layer 1.
8936 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/2);
8937 EXPECT_CALL(factory.GetMockFakeEncoder(), EncodeHook)
8938 .WillRepeatedly(Invoke([](EncodedImage& encoded_image,
8939 rtc::scoped_refptr<EncodedImageBuffer> buffer) {
8940 encoded_image.SetAtTargetQuality(encoded_image.SpatialIndex() == 0);
8941 CodecSpecificInfo codec_specific;
8942 codec_specific.codecType = kVideoCodecGeneric;
8943 return codec_specific;
8944 }));
8945 EXPECT_CALL(*adapter_ptr, UpdateLayerQualityConvergence(0, true));
8946 EXPECT_CALL(*adapter_ptr, UpdateLayerQualityConvergence(1, false));
8947 factory.DepleteTaskQueues();
8948 Mock::VerifyAndClearExpectations(adapter_ptr);
8949 Mock::VerifyAndClearExpectations(&factory.GetMockFakeEncoder());
8950}
8951
perkj26091b12016-09-01 01:17:40 -07008952} // namespace webrtc