Add av1 svc configuration for target bitrates
This configuration mostly copies vp9 configuration for regular video,
but is done separately to allow tune av1 svc bitrates independently of vp9.
Bug: webrtc:12148
Change-Id: Icd11817ada8f9b6135ee2da57204eadb50de3954
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/195329
Reviewed-by: Philip Eliasson <philipel@webrtc.org>
Commit-Queue: Danil Chapovalov <danilchap@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#32713}
diff --git a/modules/video_coding/BUILD.gn b/modules/video_coding/BUILD.gn
index 742a305..2844d52 100644
--- a/modules/video_coding/BUILD.gn
+++ b/modules/video_coding/BUILD.gn
@@ -99,29 +99,6 @@
rtc_library("video_coding") {
visibility = [ "*" ]
- deps = [
- "..:module_fec_api",
- "../../api:array_view",
- "../../api:scoped_refptr",
- "../../api/video:encoded_image",
- "../../api/video:video_adaptation",
- "../../api/video:video_bitrate_allocation",
- "../../api/video:video_bitrate_allocator_factory",
- "../../rtc_base:deprecation",
- "../../rtc_base/system:no_unique_address",
- "../../rtc_base/task_utils:to_queued_task",
- "../../system_wrappers:field_trial",
- "../../system_wrappers:metrics",
- "../rtp_rtcp:rtp_video_header",
- ]
- absl_deps = [
- "//third_party/abseil-cpp/absl/base:core_headers",
- "//third_party/abseil-cpp/absl/container:inlined_vector",
- "//third_party/abseil-cpp/absl/memory",
- "//third_party/abseil-cpp/absl/types:optional",
- "//third_party/abseil-cpp/absl/types:variant",
- ]
-
sources = [
"codec_timer.cc",
"codec_timer.h",
@@ -167,7 +144,7 @@
"video_receiver2.h",
]
- deps += [
+ deps = [
":codec_globals_headers",
":encoded_frame",
":video_codec_interface",
@@ -175,15 +152,22 @@
":webrtc_vp9_helpers",
"..:module_api",
"..:module_api_public",
+ "..:module_fec_api",
+ "../../api:array_view",
"../../api:fec_controller_api",
"../../api:rtp_headers",
"../../api:rtp_packet_info",
+ "../../api:scoped_refptr",
"../../api/units:data_rate",
"../../api/units:time_delta",
"../../api/video:builtin_video_bitrate_allocator_factory",
"../../api/video:encoded_frame",
+ "../../api/video:encoded_image",
"../../api/video:video_adaptation",
+ "../../api/video:video_adaptation",
+ "../../api/video:video_bitrate_allocation",
"../../api/video:video_bitrate_allocator",
+ "../../api/video:video_bitrate_allocator_factory",
"../../api/video:video_frame",
"../../api/video:video_frame_type",
"../../api/video:video_rtp_headers",
@@ -191,6 +175,7 @@
"../../common_video",
"../../rtc_base",
"../../rtc_base:checks",
+ "../../rtc_base:deprecation",
"../../rtc_base:rtc_base_approved",
"../../rtc_base:rtc_numerics",
"../../rtc_base:rtc_task_queue",
@@ -202,12 +187,25 @@
"../../rtc_base/experiments:rtt_mult_experiment",
"../../rtc_base/synchronization:mutex",
"../../rtc_base/synchronization:sequence_checker",
+ "../../rtc_base/system:no_unique_address",
"../../rtc_base/task_utils:repeating_task",
+ "../../rtc_base/task_utils:to_queued_task",
"../../rtc_base/third_party/base64",
"../../rtc_base/time:timestamp_extrapolator",
"../../system_wrappers",
+ "../../system_wrappers:field_trial",
+ "../../system_wrappers:metrics",
"../rtp_rtcp",
"../rtp_rtcp:rtp_rtcp_format",
+ "../rtp_rtcp:rtp_video_header",
+ "codecs/av1:av1_svc_config",
+ ]
+ absl_deps = [
+ "//third_party/abseil-cpp/absl/base:core_headers",
+ "//third_party/abseil-cpp/absl/container:inlined_vector",
+ "//third_party/abseil-cpp/absl/memory",
+ "//third_party/abseil-cpp/absl/types:optional",
+ "//third_party/abseil-cpp/absl/types:variant",
]
}
diff --git a/modules/video_coding/codecs/av1/BUILD.gn b/modules/video_coding/codecs/av1/BUILD.gn
index 27b22a0..95b5ad1 100644
--- a/modules/video_coding/codecs/av1/BUILD.gn
+++ b/modules/video_coding/codecs/av1/BUILD.gn
@@ -9,6 +9,20 @@
import("//third_party/libaom/options.gni")
import("../../../../webrtc.gni")
+rtc_library("av1_svc_config") {
+ sources = [
+ "av1_svc_config.cc",
+ "av1_svc_config.h",
+ ]
+ deps = [
+ "../../../../api/video_codecs:video_codecs_api",
+ "../../../../rtc_base:checks",
+ "../../../../rtc_base:logging",
+ "../../svc:scalability_structures",
+ "../../svc:scalable_video_controller",
+ ]
+}
+
rtc_library("libaom_av1_decoder") {
visibility = [ "*" ]
poisonous = [ "software_video_codecs" ]
@@ -70,12 +84,18 @@
rtc_library("video_coding_codecs_av1_tests") {
testonly = true
+ sources = [ "av1_svc_config_unittest.cc" ]
+ deps = [
+ ":av1_svc_config",
+ "../../../../api/video_codecs:video_codecs_api",
+ ]
+
if (enable_libaom) {
- sources = [
+ sources += [
"libaom_av1_encoder_unittest.cc",
"libaom_av1_unittest.cc",
]
- deps = [
+ deps += [
":libaom_av1_decoder",
":libaom_av1_encoder",
"../..:encoded_video_frame_producer",
@@ -84,7 +104,6 @@
"../../../../api/units:data_size",
"../../../../api/units:time_delta",
"../../../../api/video:video_frame",
- "../../../../api/video_codecs:video_codecs_api",
"../../../../test:test_support",
"../../svc:scalability_structures",
"../../svc:scalable_video_controller",
diff --git a/modules/video_coding/codecs/av1/av1_svc_config.cc b/modules/video_coding/codecs/av1/av1_svc_config.cc
new file mode 100644
index 0000000..1e61477
--- /dev/null
+++ b/modules/video_coding/codecs/av1/av1_svc_config.cc
@@ -0,0 +1,73 @@
+/*
+ * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "modules/video_coding/codecs/av1/av1_svc_config.h"
+
+#include <algorithm>
+#include <cmath>
+#include <memory>
+
+#include "modules/video_coding/svc/create_scalability_structure.h"
+#include "modules/video_coding/svc/scalable_video_controller.h"
+#include "rtc_base/checks.h"
+#include "rtc_base/logging.h"
+
+namespace webrtc {
+
+bool SetAv1SvcConfig(VideoCodec& video_codec) {
+ RTC_DCHECK_EQ(video_codec.codecType, kVideoCodecAV1);
+
+ if (video_codec.ScalabilityMode().empty()) {
+ RTC_LOG(LS_INFO) << "No scalability mode set.";
+ return false;
+ }
+ std::unique_ptr<ScalableVideoController> structure =
+ CreateScalabilityStructure(video_codec.ScalabilityMode());
+ if (structure == nullptr) {
+ RTC_LOG(LS_INFO) << "Failed to create structure "
+ << video_codec.ScalabilityMode();
+ return false;
+ }
+ ScalableVideoController::StreamLayersConfig info = structure->StreamConfig();
+ for (int sl_idx = 0; sl_idx < info.num_spatial_layers; ++sl_idx) {
+ SpatialLayer& spatial_layer = video_codec.spatialLayers[sl_idx];
+ spatial_layer.width = video_codec.width * info.scaling_factor_num[sl_idx] /
+ info.scaling_factor_den[sl_idx];
+ spatial_layer.height = video_codec.height *
+ info.scaling_factor_num[sl_idx] /
+ info.scaling_factor_den[sl_idx];
+ spatial_layer.maxFramerate = video_codec.maxFramerate;
+ spatial_layer.numberOfTemporalLayers = info.num_temporal_layers;
+ spatial_layer.active = true;
+ }
+
+ if (info.num_spatial_layers == 1) {
+ SpatialLayer& spatial_layer = video_codec.spatialLayers[0];
+ spatial_layer.minBitrate = video_codec.minBitrate;
+ spatial_layer.targetBitrate = video_codec.startBitrate;
+ spatial_layer.maxBitrate = video_codec.maxBitrate;
+ return true;
+ }
+
+ for (int sl_idx = 0; sl_idx < info.num_spatial_layers; ++sl_idx) {
+ SpatialLayer& spatial_layer = video_codec.spatialLayers[sl_idx];
+ // minBitrate and maxBitrate formulas are copied from vp9 settings and
+ // are not yet tuned for av1.
+ const int num_pixels = spatial_layer.width * spatial_layer.height;
+ int min_bitrate_kbps = (600.0 * std::sqrt(num_pixels) - 95'000.0) / 1000.0;
+ spatial_layer.minBitrate = std::max(min_bitrate_kbps, 20);
+ spatial_layer.maxBitrate = 50 + static_cast<int>(1.6 * num_pixels / 1000.0);
+ spatial_layer.targetBitrate =
+ (spatial_layer.minBitrate + spatial_layer.maxBitrate) / 2;
+ }
+ return true;
+}
+
+} // namespace webrtc
diff --git a/modules/video_coding/codecs/av1/av1_svc_config.h b/modules/video_coding/codecs/av1/av1_svc_config.h
new file mode 100644
index 0000000..15d94e0
--- /dev/null
+++ b/modules/video_coding/codecs/av1/av1_svc_config.h
@@ -0,0 +1,22 @@
+/* Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef MODULES_VIDEO_CODING_CODECS_AV1_AV1_SVC_CONFIG_H_
+#define MODULES_VIDEO_CODING_CODECS_AV1_AV1_SVC_CONFIG_H_
+
+#include "api/video_codecs/video_codec.h"
+
+namespace webrtc {
+
+// Fills `video_codec.spatialLayers` using other members.
+bool SetAv1SvcConfig(VideoCodec& video_codec);
+
+} // namespace webrtc
+
+#endif // MODULES_VIDEO_CODING_CODECS_AV1_AV1_SVC_CONFIG_H_
diff --git a/modules/video_coding/codecs/av1/av1_svc_config_unittest.cc b/modules/video_coding/codecs/av1/av1_svc_config_unittest.cc
new file mode 100644
index 0000000..02ded1c
--- /dev/null
+++ b/modules/video_coding/codecs/av1/av1_svc_config_unittest.cc
@@ -0,0 +1,142 @@
+/*
+ * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "modules/video_coding/codecs/av1/av1_svc_config.h"
+
+#include "api/video_codecs/video_codec.h"
+#include "test/gmock.h"
+#include "test/gtest.h"
+
+namespace webrtc {
+namespace {
+
+TEST(Av1SvcConfigTest, RequireScalabilityMode) {
+ VideoCodec video_codec;
+ video_codec.codecType = kVideoCodecAV1;
+
+ video_codec.SetScalabilityMode("");
+ EXPECT_FALSE(SetAv1SvcConfig(video_codec));
+
+ video_codec.SetScalabilityMode("Unknown");
+ EXPECT_FALSE(SetAv1SvcConfig(video_codec));
+
+ video_codec.SetScalabilityMode("NONE");
+ EXPECT_TRUE(SetAv1SvcConfig(video_codec));
+}
+
+TEST(Av1SvcConfigTest, SetsActiveSpatialLayersFromScalabilityMode) {
+ VideoCodec video_codec;
+ video_codec.codecType = kVideoCodecAV1;
+ video_codec.SetScalabilityMode("L2T1");
+
+ EXPECT_TRUE(SetAv1SvcConfig(video_codec));
+
+ EXPECT_TRUE(video_codec.spatialLayers[0].active);
+ EXPECT_TRUE(video_codec.spatialLayers[1].active);
+ EXPECT_FALSE(video_codec.spatialLayers[2].active);
+}
+
+TEST(Av1SvcConfigTest, ConfiguresDobuleResolutionRatioFromScalabilityMode) {
+ VideoCodec video_codec;
+ video_codec.codecType = kVideoCodecAV1;
+ video_codec.SetScalabilityMode("L2T1");
+ video_codec.width = 1200;
+ video_codec.height = 800;
+
+ EXPECT_TRUE(SetAv1SvcConfig(video_codec));
+
+ EXPECT_EQ(video_codec.spatialLayers[0].width, 600);
+ EXPECT_EQ(video_codec.spatialLayers[0].height, 400);
+ EXPECT_EQ(video_codec.spatialLayers[1].width, 1200);
+ EXPECT_EQ(video_codec.spatialLayers[1].height, 800);
+}
+
+TEST(Av1SvcConfigTest, ConfiguresSmallResolutionRatioFromScalabilityMode) {
+ VideoCodec video_codec;
+ video_codec.codecType = kVideoCodecAV1;
+ // h mode uses 1.5:1 ratio
+ video_codec.SetScalabilityMode("L2T1h");
+ video_codec.width = 1500;
+ video_codec.height = 900;
+
+ EXPECT_TRUE(SetAv1SvcConfig(video_codec));
+
+ EXPECT_EQ(video_codec.spatialLayers[0].width, 1000);
+ EXPECT_EQ(video_codec.spatialLayers[0].height, 600);
+ EXPECT_EQ(video_codec.spatialLayers[1].width, 1500);
+ EXPECT_EQ(video_codec.spatialLayers[1].height, 900);
+}
+
+TEST(Av1SvcConfigTest, CopiesFramrate) {
+ VideoCodec video_codec;
+ video_codec.codecType = kVideoCodecAV1;
+ // h mode uses 1.5:1 ratio
+ video_codec.SetScalabilityMode("L2T1");
+ video_codec.maxFramerate = 27;
+
+ EXPECT_TRUE(SetAv1SvcConfig(video_codec));
+
+ EXPECT_EQ(video_codec.spatialLayers[0].maxFramerate, 27);
+ EXPECT_EQ(video_codec.spatialLayers[1].maxFramerate, 27);
+}
+
+TEST(Av1SvcConfigTest, SetsNumberOfTemporalLayers) {
+ VideoCodec video_codec;
+ video_codec.codecType = kVideoCodecAV1;
+ video_codec.SetScalabilityMode("L1T3");
+
+ EXPECT_TRUE(SetAv1SvcConfig(video_codec));
+
+ EXPECT_EQ(video_codec.spatialLayers[0].numberOfTemporalLayers, 3);
+}
+
+TEST(Av1SvcConfigTest, CopiesBitrateForSingleSpatialLayer) {
+ VideoCodec video_codec;
+ video_codec.codecType = kVideoCodecAV1;
+ video_codec.SetScalabilityMode("L1T3");
+ video_codec.minBitrate = 100;
+ video_codec.startBitrate = 200;
+ video_codec.maxBitrate = 500;
+
+ EXPECT_TRUE(SetAv1SvcConfig(video_codec));
+
+ EXPECT_EQ(video_codec.spatialLayers[0].minBitrate, 100u);
+ EXPECT_EQ(video_codec.spatialLayers[0].targetBitrate, 200u);
+ EXPECT_EQ(video_codec.spatialLayers[0].maxBitrate, 500u);
+}
+
+TEST(Av1SvcConfigTest, SetsBitratesForMultipleSpatialLayers) {
+ VideoCodec video_codec;
+ video_codec.codecType = kVideoCodecAV1;
+ video_codec.SetScalabilityMode("L3T3");
+
+ EXPECT_TRUE(SetAv1SvcConfig(video_codec));
+
+ EXPECT_GT(video_codec.spatialLayers[0].minBitrate, 0u);
+ EXPECT_LE(video_codec.spatialLayers[0].minBitrate,
+ video_codec.spatialLayers[0].targetBitrate);
+ EXPECT_LE(video_codec.spatialLayers[0].targetBitrate,
+ video_codec.spatialLayers[0].maxBitrate);
+
+ EXPECT_GT(video_codec.spatialLayers[1].minBitrate, 0u);
+ EXPECT_LE(video_codec.spatialLayers[1].minBitrate,
+ video_codec.spatialLayers[1].targetBitrate);
+ EXPECT_LE(video_codec.spatialLayers[1].targetBitrate,
+ video_codec.spatialLayers[1].maxBitrate);
+
+ EXPECT_GT(video_codec.spatialLayers[2].minBitrate, 0u);
+ EXPECT_LE(video_codec.spatialLayers[2].minBitrate,
+ video_codec.spatialLayers[2].targetBitrate);
+ EXPECT_LE(video_codec.spatialLayers[2].targetBitrate,
+ video_codec.spatialLayers[2].maxBitrate);
+}
+
+} // namespace
+} // namespace webrtc
diff --git a/modules/video_coding/video_codec_initializer.cc b/modules/video_coding/video_codec_initializer.cc
index 4777fe5..983e2a0 100644
--- a/modules/video_coding/video_codec_initializer.cc
+++ b/modules/video_coding/video_codec_initializer.cc
@@ -20,6 +20,7 @@
#include "api/units/data_rate.h"
#include "api/video/video_bitrate_allocation.h"
#include "api/video_codecs/video_encoder.h"
+#include "modules/video_coding/codecs/av1/av1_svc_config.h"
#include "modules/video_coding/codecs/vp9/svc_config.h"
#include "modules/video_coding/include/video_coding_defines.h"
#include "rtc_base/checks.h"
@@ -56,7 +57,6 @@
RTC_DCHECK_GE(config.min_transmit_bitrate_bps, 0);
VideoCodec video_codec;
- memset(&video_codec, 0, sizeof(video_codec));
video_codec.codecType = config.codec_type;
switch (config.content_type) {
@@ -255,6 +255,11 @@
break;
}
+ case kVideoCodecAV1:
+ if (!SetAv1SvcConfig(video_codec)) {
+ RTC_LOG(LS_WARNING) << "Failed to configure svc bitrates for av1.";
+ }
+ break;
case kVideoCodecH264: {
if (!config.encoder_specific_settings)
*video_codec.H264() = VideoEncoder::GetDefaultH264Settings();