Add support for more scalability modes (1.5:1 resolution ratio).
Added modes:
- S2T1h
- S2T2h
- S2T3h
- S3T1h
- S3T2h
- S3T3h
Bug: webrtc:13960
Change-Id: I618a30c68b0ce1609847ee33a2298fe8fa0720c9
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/273664
Reviewed-by: Florent Castelli <orphis@webrtc.org>
Reviewed-by: Rasmus Brandt <brandtr@webrtc.org>
Commit-Queue: Åsa Persson <asapersson@webrtc.org>
Reviewed-by: Danil Chapovalov <danilchap@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#37968}
diff --git a/api/video_codecs/scalability_mode.cc b/api/video_codecs/scalability_mode.cc
index ab26915..c449b42 100644
--- a/api/video_codecs/scalability_mode.cc
+++ b/api/video_codecs/scalability_mode.cc
@@ -62,16 +62,28 @@
return "L3T3_KEY";
case ScalabilityMode::kS2T1:
return "S2T1";
+ case ScalabilityMode::kS2T1h:
+ return "S2T1h";
case ScalabilityMode::kS2T2:
return "S2T2";
+ case ScalabilityMode::kS2T2h:
+ return "S2T2h";
case ScalabilityMode::kS2T3:
return "S2T3";
+ case ScalabilityMode::kS2T3h:
+ return "S2T3h";
case ScalabilityMode::kS3T1:
return "S3T1";
+ case ScalabilityMode::kS3T1h:
+ return "S3T1h";
case ScalabilityMode::kS3T2:
return "S3T2";
+ case ScalabilityMode::kS3T2h:
+ return "S3T2h";
case ScalabilityMode::kS3T3:
return "S3T3";
+ case ScalabilityMode::kS3T3h:
+ return "S3T3h";
}
RTC_CHECK_NOTREACHED();
}
diff --git a/api/video_codecs/scalability_mode.h b/api/video_codecs/scalability_mode.h
index 09f564e..b26f32e 100644
--- a/api/video_codecs/scalability_mode.h
+++ b/api/video_codecs/scalability_mode.h
@@ -48,11 +48,17 @@
kL3T3h,
kL3T3_KEY,
kS2T1,
+ kS2T1h,
kS2T2,
+ kS2T2h,
kS2T3,
+ kS2T3h,
kS3T1,
+ kS3T1h,
kS3T2,
+ kS3T2h,
kS3T3,
+ kS3T3h,
};
inline constexpr ScalabilityMode kAllScalabilityModes[] = {
@@ -80,11 +86,17 @@
ScalabilityMode::kL3T3h,
ScalabilityMode::kL3T3_KEY,
ScalabilityMode::kS2T1,
+ ScalabilityMode::kS2T1h,
ScalabilityMode::kS2T2,
+ ScalabilityMode::kS2T2h,
ScalabilityMode::kS2T3,
+ ScalabilityMode::kS2T3h,
ScalabilityMode::kS3T1,
+ ScalabilityMode::kS3T1h,
ScalabilityMode::kS3T2,
+ ScalabilityMode::kS3T2h,
ScalabilityMode::kS3T3,
+ ScalabilityMode::kS3T3h,
// clang-format on
};
diff --git a/modules/video_coding/svc/create_scalability_structure.cc b/modules/video_coding/svc/create_scalability_structure.cc
index dd3225c..fbcd27b 100644
--- a/modules/video_coding/svc/create_scalability_structure.cc
+++ b/modules/video_coding/svc/create_scalability_structure.cc
@@ -148,6 +148,13 @@
{1, 1},
{2, 1}};
+constexpr ScalableVideoController::StreamLayersConfig kConfigS2T1h = {
+ /*num_spatial_layers=*/2,
+ /*num_temporal_layers=*/1,
+ /*uses_reference_scaling=*/false,
+ {2, 1},
+ {3, 1}};
+
constexpr ScalableVideoController::StreamLayersConfig kConfigS2T2 = {
/*num_spatial_layers=*/2,
/*num_temporal_layers=*/2,
@@ -155,6 +162,13 @@
{1, 1},
{2, 1}};
+constexpr ScalableVideoController::StreamLayersConfig kConfigS2T2h = {
+ /*num_spatial_layers=*/2,
+ /*num_temporal_layers=*/2,
+ /*uses_reference_scaling=*/false,
+ {2, 1},
+ {3, 1}};
+
constexpr ScalableVideoController::StreamLayersConfig kConfigS2T3 = {
/*num_spatial_layers=*/2,
/*num_temporal_layers=*/3,
@@ -162,6 +176,13 @@
{1, 1},
{2, 1}};
+constexpr ScalableVideoController::StreamLayersConfig kConfigS2T3h = {
+ /*num_spatial_layers=*/2,
+ /*num_temporal_layers=*/3,
+ /*uses_reference_scaling=*/false,
+ {2, 1},
+ {3, 1}};
+
constexpr ScalableVideoController::StreamLayersConfig kConfigS3T1 = {
/*num_spatial_layers=*/3,
/*num_temporal_layers=*/1,
@@ -169,6 +190,13 @@
{1, 1, 1},
{4, 2, 1}};
+constexpr ScalableVideoController::StreamLayersConfig kConfigS3T1h = {
+ /*num_spatial_layers=*/3,
+ /*num_temporal_layers=*/1,
+ /*uses_reference_scaling=*/false,
+ {4, 2, 1},
+ {9, 3, 1}};
+
constexpr ScalableVideoController::StreamLayersConfig kConfigS3T2 = {
/*num_spatial_layers=*/3,
/*num_temporal_layers=*/2,
@@ -176,6 +204,13 @@
{1, 1, 1},
{4, 2, 1}};
+constexpr ScalableVideoController::StreamLayersConfig kConfigS3T2h = {
+ /*num_spatial_layers=*/3,
+ /*num_temporal_layers=*/2,
+ /*uses_reference_scaling=*/false,
+ {4, 2, 1},
+ {9, 3, 1}};
+
constexpr ScalableVideoController::StreamLayersConfig kConfigS3T3 = {
/*num_spatial_layers=*/3,
/*num_temporal_layers=*/3,
@@ -183,6 +218,13 @@
{1, 1, 1},
{4, 2, 1}};
+constexpr ScalableVideoController::StreamLayersConfig kConfigS3T3h = {
+ /*num_spatial_layers=*/3,
+ /*num_temporal_layers=*/3,
+ /*uses_reference_scaling=*/false,
+ {4, 2, 1},
+ {9, 3, 1}};
+
constexpr NamedStructureFactory kFactories[] = {
{ScalabilityMode::kL1T1, Create<ScalableVideoControllerNoLayering>,
kConfigL1T1},
@@ -215,11 +257,17 @@
{ScalabilityMode::kL3T3_KEY, Create<ScalabilityStructureL3T3Key>,
kConfigL3T3},
{ScalabilityMode::kS2T1, Create<ScalabilityStructureS2T1>, kConfigS2T1},
+ {ScalabilityMode::kS2T1h, CreateH<ScalabilityStructureS2T1>, kConfigS2T1h},
{ScalabilityMode::kS2T2, Create<ScalabilityStructureS2T2>, kConfigS2T2},
+ {ScalabilityMode::kS2T2h, CreateH<ScalabilityStructureS2T2>, kConfigS2T2h},
{ScalabilityMode::kS2T3, Create<ScalabilityStructureS2T3>, kConfigS2T3},
+ {ScalabilityMode::kS2T3h, CreateH<ScalabilityStructureS2T3>, kConfigS2T3h},
{ScalabilityMode::kS3T1, Create<ScalabilityStructureS3T1>, kConfigS3T1},
+ {ScalabilityMode::kS3T1h, CreateH<ScalabilityStructureS3T1>, kConfigS3T1h},
{ScalabilityMode::kS3T2, Create<ScalabilityStructureS3T2>, kConfigS3T2},
+ {ScalabilityMode::kS3T2h, CreateH<ScalabilityStructureS3T2>, kConfigS3T2h},
{ScalabilityMode::kS3T3, Create<ScalabilityStructureS3T3>, kConfigS3T3},
+ {ScalabilityMode::kS3T3h, CreateH<ScalabilityStructureS3T3>, kConfigS3T3h},
};
} // namespace
diff --git a/modules/video_coding/svc/scalability_mode_util.cc b/modules/video_coding/svc/scalability_mode_util.cc
index 070dcfe..39a4f1f 100644
--- a/modules/video_coding/svc/scalability_mode_util.cc
+++ b/modules/video_coding/svc/scalability_mode_util.cc
@@ -71,16 +71,28 @@
if (mode_string == "S2T1")
return ScalabilityMode::kS2T1;
+ if (mode_string == "S2T1h")
+ return ScalabilityMode::kS2T1h;
if (mode_string == "S2T2")
return ScalabilityMode::kS2T2;
+ if (mode_string == "S2T2h")
+ return ScalabilityMode::kS2T2h;
if (mode_string == "S2T3")
return ScalabilityMode::kS2T3;
+ if (mode_string == "S2T3h")
+ return ScalabilityMode::kS2T3h;
if (mode_string == "S3T1")
return ScalabilityMode::kS3T1;
+ if (mode_string == "S3T1h")
+ return ScalabilityMode::kS3T1h;
if (mode_string == "S3T2")
return ScalabilityMode::kS3T2;
+ if (mode_string == "S3T2h")
+ return ScalabilityMode::kS3T2h;
if (mode_string == "S3T3")
return ScalabilityMode::kS3T3;
+ if (mode_string == "S3T3h")
+ return ScalabilityMode::kS3T3h;
return absl::nullopt;
}
@@ -123,11 +135,17 @@
case ScalabilityMode::kL3T3_KEY:
return InterLayerPredMode::kOnKeyPic;
case ScalabilityMode::kS2T1:
+ case ScalabilityMode::kS2T1h:
case ScalabilityMode::kS2T2:
+ case ScalabilityMode::kS2T2h:
case ScalabilityMode::kS2T3:
+ case ScalabilityMode::kS2T3h:
case ScalabilityMode::kS3T1:
+ case ScalabilityMode::kS3T1h:
case ScalabilityMode::kS3T2:
+ case ScalabilityMode::kS3T2h:
case ScalabilityMode::kS3T3:
+ case ScalabilityMode::kS3T3h:
return InterLayerPredMode::kOff;
}
RTC_CHECK_NOTREACHED();
@@ -161,12 +179,18 @@
case ScalabilityMode::kL3T3_KEY:
return 3;
case ScalabilityMode::kS2T1:
+ case ScalabilityMode::kS2T1h:
case ScalabilityMode::kS2T2:
+ case ScalabilityMode::kS2T2h:
case ScalabilityMode::kS2T3:
+ case ScalabilityMode::kS2T3h:
return 2;
case ScalabilityMode::kS3T1:
+ case ScalabilityMode::kS3T1h:
case ScalabilityMode::kS3T2:
+ case ScalabilityMode::kS3T2h:
case ScalabilityMode::kS3T3:
+ case ScalabilityMode::kS3T3h:
return 3;
}
RTC_CHECK_NOTREACHED();
@@ -206,13 +230,19 @@
case ScalabilityMode::kL3T3_KEY:
return 3;
case ScalabilityMode::kS2T1:
+ case ScalabilityMode::kS2T1h:
case ScalabilityMode::kS3T1:
+ case ScalabilityMode::kS3T1h:
return 1;
case ScalabilityMode::kS2T2:
+ case ScalabilityMode::kS2T2h:
case ScalabilityMode::kS3T2:
+ case ScalabilityMode::kS3T2h:
return 2;
case ScalabilityMode::kS2T3:
+ case ScalabilityMode::kS2T3h:
case ScalabilityMode::kS3T3:
+ case ScalabilityMode::kS3T3h:
return 3;
}
RTC_CHECK_NOTREACHED();
@@ -251,6 +281,12 @@
case ScalabilityMode::kL3T1h:
case ScalabilityMode::kL3T2h:
case ScalabilityMode::kL3T3h:
+ case ScalabilityMode::kS2T1h:
+ case ScalabilityMode::kS2T2h:
+ case ScalabilityMode::kS2T3h:
+ case ScalabilityMode::kS3T1h:
+ case ScalabilityMode::kS3T2h:
+ case ScalabilityMode::kS3T3h:
return ScalabilityModeResolutionRatio::kThreeToTwo;
}
RTC_CHECK_NOTREACHED();
diff --git a/modules/video_coding/svc/scalability_structure_simulcast.cc b/modules/video_coding/svc/scalability_structure_simulcast.cc
index 010c461..54e27fd 100644
--- a/modules/video_coding/svc/scalability_structure_simulcast.cc
+++ b/modules/video_coding/svc/scalability_structure_simulcast.cc
@@ -43,9 +43,11 @@
ScalabilityStructureSimulcast::ScalabilityStructureSimulcast(
int num_spatial_layers,
- int num_temporal_layers)
+ int num_temporal_layers,
+ ScalingFactor resolution_factor)
: num_spatial_layers_(num_spatial_layers),
num_temporal_layers_(num_temporal_layers),
+ resolution_factor_(resolution_factor),
active_decode_targets_(
(uint32_t{1} << (num_spatial_layers * num_temporal_layers)) - 1) {
RTC_DCHECK_LE(num_spatial_layers, kMaxNumSpatialLayers);
@@ -62,8 +64,10 @@
result.scaling_factor_num[num_spatial_layers_ - 1] = 1;
result.scaling_factor_den[num_spatial_layers_ - 1] = 1;
for (int sid = num_spatial_layers_ - 1; sid > 0; --sid) {
- result.scaling_factor_num[sid - 1] = 1;
- result.scaling_factor_den[sid - 1] = 2 * result.scaling_factor_den[sid];
+ result.scaling_factor_num[sid - 1] =
+ resolution_factor_.num * result.scaling_factor_num[sid];
+ result.scaling_factor_den[sid - 1] =
+ resolution_factor_.den * result.scaling_factor_den[sid];
}
result.uses_reference_scaling = false;
return result;
diff --git a/modules/video_coding/svc/scalability_structure_simulcast.h b/modules/video_coding/svc/scalability_structure_simulcast.h
index 7438173..99be9f0 100644
--- a/modules/video_coding/svc/scalability_structure_simulcast.h
+++ b/modules/video_coding/svc/scalability_structure_simulcast.h
@@ -23,8 +23,13 @@
// same temporal layering.
class ScalabilityStructureSimulcast : public ScalableVideoController {
public:
+ struct ScalingFactor {
+ int num = 1;
+ int den = 2;
+ };
ScalabilityStructureSimulcast(int num_spatial_layers,
- int num_temporal_layers);
+ int num_temporal_layers,
+ ScalingFactor resolution_factor);
~ScalabilityStructureSimulcast() override;
StreamLayersConfig StreamConfig() const override;
@@ -58,6 +63,7 @@
const int num_spatial_layers_;
const int num_temporal_layers_;
+ const ScalingFactor resolution_factor_;
FramePattern last_pattern_ = kNone;
std::bitset<kMaxNumSpatialLayers> can_reference_t0_frame_for_spatial_id_ = 0;
@@ -70,7 +76,8 @@
// S0 0--0--0-
class ScalabilityStructureS2T1 : public ScalabilityStructureSimulcast {
public:
- ScalabilityStructureS2T1() : ScalabilityStructureSimulcast(2, 1) {}
+ explicit ScalabilityStructureS2T1(ScalingFactor resolution_factor = {})
+ : ScalabilityStructureSimulcast(2, 1, resolution_factor) {}
~ScalabilityStructureS2T1() override = default;
FrameDependencyStructure DependencyStructure() const override;
@@ -78,7 +85,8 @@
class ScalabilityStructureS2T2 : public ScalabilityStructureSimulcast {
public:
- ScalabilityStructureS2T2() : ScalabilityStructureSimulcast(2, 2) {}
+ explicit ScalabilityStructureS2T2(ScalingFactor resolution_factor = {})
+ : ScalabilityStructureSimulcast(2, 2, resolution_factor) {}
~ScalabilityStructureS2T2() override = default;
FrameDependencyStructure DependencyStructure() const override;
@@ -98,7 +106,8 @@
// Time-> 0 1 2 3 4
class ScalabilityStructureS2T3 : public ScalabilityStructureSimulcast {
public:
- ScalabilityStructureS2T3() : ScalabilityStructureSimulcast(2, 3) {}
+ explicit ScalabilityStructureS2T3(ScalingFactor resolution_factor = {})
+ : ScalabilityStructureSimulcast(2, 3, resolution_factor) {}
~ScalabilityStructureS2T3() override = default;
FrameDependencyStructure DependencyStructure() const override;
@@ -106,7 +115,8 @@
class ScalabilityStructureS3T1 : public ScalabilityStructureSimulcast {
public:
- ScalabilityStructureS3T1() : ScalabilityStructureSimulcast(3, 1) {}
+ explicit ScalabilityStructureS3T1(ScalingFactor resolution_factor = {})
+ : ScalabilityStructureSimulcast(3, 1, resolution_factor) {}
~ScalabilityStructureS3T1() override = default;
FrameDependencyStructure DependencyStructure() const override;
@@ -114,7 +124,8 @@
class ScalabilityStructureS3T2 : public ScalabilityStructureSimulcast {
public:
- ScalabilityStructureS3T2() : ScalabilityStructureSimulcast(3, 2) {}
+ explicit ScalabilityStructureS3T2(ScalingFactor resolution_factor = {})
+ : ScalabilityStructureSimulcast(3, 2, resolution_factor) {}
~ScalabilityStructureS3T2() override = default;
FrameDependencyStructure DependencyStructure() const override;
@@ -122,7 +133,8 @@
class ScalabilityStructureS3T3 : public ScalabilityStructureSimulcast {
public:
- ScalabilityStructureS3T3() : ScalabilityStructureSimulcast(3, 3) {}
+ explicit ScalabilityStructureS3T3(ScalingFactor resolution_factor = {})
+ : ScalabilityStructureSimulcast(3, 3, resolution_factor) {}
~ScalabilityStructureS3T3() override = default;
FrameDependencyStructure DependencyStructure() const override;
diff --git a/pc/peer_connection_factory_unittest.cc b/pc/peer_connection_factory_unittest.cc
index fdbc415..2251990 100644
--- a/pc/peer_connection_factory_unittest.cc
+++ b/pc/peer_connection_factory_unittest.cc
@@ -222,11 +222,17 @@
webrtc::ScalabilityMode::kL3T3h,
webrtc::ScalabilityMode::kL3T3_KEY,
webrtc::ScalabilityMode::kS2T1,
+ webrtc::ScalabilityMode::kS2T1h,
webrtc::ScalabilityMode::kS2T2,
+ webrtc::ScalabilityMode::kS2T2h,
webrtc::ScalabilityMode::kS2T3,
+ webrtc::ScalabilityMode::kS2T3h,
webrtc::ScalabilityMode::kS3T1,
+ webrtc::ScalabilityMode::kS3T1h,
webrtc::ScalabilityMode::kS3T2,
- webrtc::ScalabilityMode::kS3T3)
+ webrtc::ScalabilityMode::kS3T2h,
+ webrtc::ScalabilityMode::kS3T3,
+ webrtc::ScalabilityMode::kS3T3h)
// clang-format on
)
<< "Codec: " << codec.name;
diff --git a/video/video_send_stream_tests.cc b/video/video_send_stream_tests.cc
index c948012..f4ba3df 100644
--- a/video/video_send_stream_tests.cc
+++ b/video/video_send_stream_tests.cc
@@ -3392,7 +3392,13 @@
{"L2T2_KEY_SHIFT", 2, 2, InterLayerPredMode::kOnKeyPic},
{"L3T1h", 3, 1, InterLayerPredMode::kOn},
{"L3T2h", 3, 2, InterLayerPredMode::kOn},
- {"L3T3h", 3, 3, InterLayerPredMode::kOn}}),
+ {"L3T3h", 3, 3, InterLayerPredMode::kOn},
+ {"S2T1h", 2, 1, InterLayerPredMode::kOff},
+ {"S2T2h", 2, 2, InterLayerPredMode::kOff},
+ {"S2T3h", 2, 3, InterLayerPredMode::kOff},
+ {"S3T1h", 3, 1, InterLayerPredMode::kOff},
+ {"S3T2h", 3, 2, InterLayerPredMode::kOff},
+ {"S3T3h", 3, 3, InterLayerPredMode::kOff}}),
::testing::Values(true)), // use_scalability_mode_identifier
ParamInfoToStr);