Layering and rate allocation for VP9 screen sharing.
- Two quality layers (same resolution, different bitrate).
- Max bitrate of low layer is limited to 200kbps. The choice of the
limit is driven by VP8 screen sharing which limits max bitrate of low
temporal layer to 200kbps. Using the same value for VP9 guarantees
that there will be no regressions for participants with limited
bandwidth.
- Max bitrate of high layer is limited to 500kbps. According to test
results this value is enough to get up to +5dB higher PSNR than VP8
SS provides on 1.2Mbps (max total bitrate for VP8 SS) link.
- Max total sent bitrate is limited to 700kbps. It is 500kbps lower
than that in VP8 SS (1200kbps).
Bug: webrtc:9261
Change-Id: I7919cc3933064664567c39e380a44cad0c65f1e8
Reviewed-on: https://webrtc-review.googlesource.com/76380
Commit-Queue: Sergey Silkin <ssilkin@webrtc.org>
Reviewed-by: Erik Språng <sprang@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#23226}diff --git a/modules/video_coding/codecs/test/test_config.cc b/modules/video_coding/codecs/test/test_config.cc
index de6beb9..0d02ba7 100644
--- a/modules/video_coding/codecs/test/test_config.cc
+++ b/modules/video_coding/codecs/test/test_config.cc
@@ -56,7 +56,7 @@
const std::vector<SpatialLayer> layers =
GetSvcConfig(codec_settings->width, codec_settings->height,
codec_settings->VP9()->numberOfSpatialLayers,
- codec_settings->VP9()->numberOfTemporalLayers);
+ codec_settings->VP9()->numberOfTemporalLayers, false);
for (size_t i = 0; i < layers.size(); ++i) {
codec_settings->spatialLayers[i] = layers[i];
diff --git a/modules/video_coding/codecs/vp9/svc_config.cc b/modules/video_coding/codecs/vp9/svc_config.cc
index ac6b4e5..ed60a0d 100644
--- a/modules/video_coding/codecs/vp9/svc_config.cc
+++ b/modules/video_coding/codecs/vp9/svc_config.cc
@@ -19,18 +19,38 @@
namespace webrtc {
namespace {
-const int kMinVp9SvcBitrateKbps = 30; // Lowest VP9 video rate in kbps.
+const size_t kMinVp9SvcBitrateKbps = 30;
+
+const size_t kMaxNumLayersForScreenSharing = 2;
+const size_t kMaxScreenSharingLayerBitrateKbps[] = {200, 500};
} // namespace
-std::vector<SpatialLayer> GetSvcConfig(size_t input_width,
- size_t input_height,
- size_t num_spatial_layers,
- size_t num_temporal_layers) {
- RTC_DCHECK_GT(input_width, 0);
- RTC_DCHECK_GT(input_height, 0);
- RTC_DCHECK_GT(num_spatial_layers, 0);
- RTC_DCHECK_GT(num_temporal_layers, 0);
+std::vector<SpatialLayer> ConfigureSvcScreenSharing(size_t input_width,
+ size_t input_height,
+ size_t num_spatial_layers) {
+ num_spatial_layers =
+ std::min(num_spatial_layers, kMaxNumLayersForScreenSharing);
+ std::vector<SpatialLayer> spatial_layers;
+ for (size_t sl_idx = 0; sl_idx < num_spatial_layers; ++sl_idx) {
+ SpatialLayer spatial_layer = {0};
+ spatial_layer.width = input_width;
+ spatial_layer.height = input_height;
+ spatial_layer.numberOfTemporalLayers = 1;
+ spatial_layer.minBitrate = static_cast<int>(kMinVp9SvcBitrateKbps);
+ spatial_layer.maxBitrate =
+ static_cast<int>(kMaxScreenSharingLayerBitrateKbps[sl_idx]);
+ spatial_layer.targetBitrate = spatial_layer.maxBitrate;
+ spatial_layers.push_back(spatial_layer);
+ }
+
+ return spatial_layers;
+}
+
+std::vector<SpatialLayer> ConfigureSvcNormalVideo(size_t input_width,
+ size_t input_height,
+ size_t num_spatial_layers,
+ size_t num_temporal_layers) {
std::vector<SpatialLayer> spatial_layers;
// Limit number of layers for given resolution.
@@ -57,7 +77,7 @@
// TODO(ssilkin): Add to the comment PSNR/SSIM we get at encoding certain
// video to min/max bitrate specified by those formulas.
const size_t num_pixels = spatial_layer.width * spatial_layer.height;
- const int min_bitrate =
+ const size_t min_bitrate =
static_cast<int>((600. * std::sqrt(num_pixels) - 95000.) / 1000.);
spatial_layer.minBitrate = std::max(min_bitrate, kMinVp9SvcBitrateKbps);
spatial_layer.maxBitrate =
@@ -71,4 +91,23 @@
return spatial_layers;
}
+std::vector<SpatialLayer> GetSvcConfig(size_t input_width,
+ size_t input_height,
+ size_t num_spatial_layers,
+ size_t num_temporal_layers,
+ bool is_screen_sharing) {
+ RTC_DCHECK_GT(input_width, 0);
+ RTC_DCHECK_GT(input_height, 0);
+ RTC_DCHECK_GT(num_spatial_layers, 0);
+ RTC_DCHECK_GT(num_temporal_layers, 0);
+
+ if (is_screen_sharing) {
+ return ConfigureSvcScreenSharing(input_width, input_height,
+ num_spatial_layers);
+ } else {
+ return ConfigureSvcNormalVideo(input_width, input_height,
+ num_spatial_layers, num_temporal_layers);
+ }
+}
+
} // namespace webrtc
diff --git a/modules/video_coding/codecs/vp9/svc_config.h b/modules/video_coding/codecs/vp9/svc_config.h
index 78d2bf2..c8561a4 100644
--- a/modules/video_coding/codecs/vp9/svc_config.h
+++ b/modules/video_coding/codecs/vp9/svc_config.h
@@ -19,7 +19,8 @@
std::vector<SpatialLayer> GetSvcConfig(size_t input_width,
size_t input_height,
size_t num_spatial_layers,
- size_t num_temporal_layers);
+ size_t num_temporal_layers,
+ bool is_screen_sharing);
} // namespace webrtc
diff --git a/modules/video_coding/codecs/vp9/svc_config_unittest.cc b/modules/video_coding/codecs/vp9/svc_config_unittest.cc
index ab47a6f..ebefdda 100644
--- a/modules/video_coding/codecs/vp9/svc_config_unittest.cc
+++ b/modules/video_coding/codecs/vp9/svc_config_unittest.cc
@@ -24,7 +24,7 @@
std::vector<SpatialLayer> spatial_layers =
GetSvcConfig(kMinVp9SpatialLayerWidth << (num_spatial_layers - 1),
kMinVp9SpatialLayerHeight << (num_spatial_layers - 1),
- max_num_spatial_layers, 1);
+ max_num_spatial_layers, 1, false);
EXPECT_EQ(spatial_layers.size(), num_spatial_layers);
}
@@ -34,7 +34,7 @@
std::vector<SpatialLayer> spatial_layers =
GetSvcConfig(kMinVp9SpatialLayerWidth << (num_spatial_layers - 1),
kMinVp9SpatialLayerHeight << (num_spatial_layers - 1),
- num_spatial_layers, 1);
+ num_spatial_layers, 1, false);
EXPECT_EQ(spatial_layers.size(), num_spatial_layers);
@@ -44,4 +44,20 @@
EXPECT_LE(layer.targetBitrate, layer.maxBitrate);
}
}
+
+TEST(SvcConfig, ScreenSharing) {
+ std::vector<SpatialLayer> spatial_layers =
+ GetSvcConfig(1920, 1080, 3, 3, true);
+
+ EXPECT_EQ(spatial_layers.size(), 2UL);
+
+ for (const SpatialLayer& layer : spatial_layers) {
+ EXPECT_EQ(layer.width, 1920);
+ EXPECT_EQ(layer.height, 1080);
+ EXPECT_EQ(layer.numberOfTemporalLayers, 1);
+ EXPECT_LE(layer.minBitrate, layer.maxBitrate);
+ EXPECT_LE(layer.minBitrate, layer.targetBitrate);
+ EXPECT_LE(layer.targetBitrate, layer.maxBitrate);
+ }
+}
} // namespace webrtc
diff --git a/modules/video_coding/codecs/vp9/svc_rate_allocator.cc b/modules/video_coding/codecs/vp9/svc_rate_allocator.cc
index 46caef4..61505c0 100644
--- a/modules/video_coding/codecs/vp9/svc_rate_allocator.cc
+++ b/modules/video_coding/codecs/vp9/svc_rate_allocator.cc
@@ -30,72 +30,49 @@
VideoBitrateAllocation SvcRateAllocator::GetAllocation(
uint32_t total_bitrate_bps,
uint32_t framerate_fps) {
- VideoBitrateAllocation bitrate_allocation;
+ if (codec_.maxBitrate != 0) {
+ total_bitrate_bps = std::min(total_bitrate_bps, codec_.maxBitrate * 1000);
+ }
+ if (codec_.spatialLayers[0].targetBitrate == 0) {
+ // Delegate rate distribution to VP9 encoder wrapper if bitrate thresholds
+ // are not initialized.
+ VideoBitrateAllocation bitrate_allocation;
+ bitrate_allocation.SetBitrate(0, 0, total_bitrate_bps);
+ return bitrate_allocation;
+ } else if (codec_.mode == kRealtimeVideo) {
+ return GetAllocationNormalVideo(total_bitrate_bps);
+ } else {
+ return GetAllocationScreenSharing(total_bitrate_bps);
+ }
+}
+
+VideoBitrateAllocation SvcRateAllocator::GetAllocationNormalVideo(
+ uint32_t total_bitrate_bps) const {
size_t num_spatial_layers = codec_.VP9().numberOfSpatialLayers;
RTC_CHECK(num_spatial_layers > 0);
size_t num_temporal_layers = codec_.VP9().numberOfTemporalLayers;
RTC_CHECK(num_temporal_layers > 0);
- if (codec_.maxBitrate != 0) {
- total_bitrate_bps = std::min(total_bitrate_bps, codec_.maxBitrate * 1000);
- }
-
- if (codec_.mode == kScreensharing) {
- // At screen sharing bitrate allocation is handled by VP9 encoder wrapper.
- bitrate_allocation.SetBitrate(0, 0, total_bitrate_bps);
- return bitrate_allocation;
- }
-
std::vector<size_t> spatial_layer_bitrate_bps;
- if (codec_.spatialLayers[0].maxBitrate == 0) {
- // Layers' parameters are not initialized. Do simple split.
+ // Distribute total bitrate across spatial layers. If there is not enough
+ // bitrate to provide all layers with at least minimum required bitrate
+ // then number of layers is reduced by one and distribution is repeated
+ // until that condition is met or if number of layers is reduced to one.
+ for (;; --num_spatial_layers) {
spatial_layer_bitrate_bps =
SplitBitrate(num_spatial_layers, total_bitrate_bps,
kSpatialLayeringRateScalingFactor);
- } else {
- // Distribute total bitrate across spatial layers. If there is not enough
- // bitrate to provide all layers with at least minimum required bitrate
- // then number of layers is reduced by one and distribution is repeated
- // until that condition is met or if number of layers is reduced to one.
- for (;; --num_spatial_layers) {
- spatial_layer_bitrate_bps =
- SplitBitrate(num_spatial_layers, total_bitrate_bps,
- kSpatialLayeringRateScalingFactor);
- bool enough_bitrate = true;
- size_t excess_rate = 0;
- for (size_t sl_idx = 0; sl_idx < num_spatial_layers; ++sl_idx) {
- RTC_DCHECK_GT(codec_.spatialLayers[sl_idx].maxBitrate, 0);
- RTC_DCHECK_GE(codec_.spatialLayers[sl_idx].maxBitrate,
- codec_.spatialLayers[sl_idx].minBitrate);
-
- const size_t min_bitrate_bps =
- codec_.spatialLayers[sl_idx].minBitrate * 1000;
- const size_t max_bitrate_bps =
- codec_.spatialLayers[sl_idx].maxBitrate * 1000;
-
- spatial_layer_bitrate_bps[sl_idx] += excess_rate;
- if (spatial_layer_bitrate_bps[sl_idx] < max_bitrate_bps) {
- excess_rate = 0;
- } else {
- excess_rate = spatial_layer_bitrate_bps[sl_idx] - max_bitrate_bps;
- spatial_layer_bitrate_bps[sl_idx] = max_bitrate_bps;
- }
-
- if (spatial_layer_bitrate_bps[sl_idx] < min_bitrate_bps) {
- enough_bitrate = false;
- break;
- }
- }
-
- if (enough_bitrate || num_spatial_layers == 1) {
- break;
- }
+ const bool enough_bitrate = AdjustAndVerify(&spatial_layer_bitrate_bps);
+ if (enough_bitrate || num_spatial_layers == 1) {
+ break;
}
}
+ VideoBitrateAllocation bitrate_allocation;
+
for (size_t sl_idx = 0; sl_idx < num_spatial_layers; ++sl_idx) {
std::vector<size_t> temporal_layer_bitrate_bps =
SplitBitrate(num_temporal_layers, spatial_layer_bitrate_bps[sl_idx],
@@ -125,13 +102,72 @@
return bitrate_allocation;
}
+bool SvcRateAllocator::AdjustAndVerify(
+ std::vector<size_t>* spatial_layer_bitrate_bps) const {
+ bool enough_bitrate = true;
+ size_t excess_rate = 0;
+ for (size_t sl_idx = 0;
+ sl_idx < spatial_layer_bitrate_bps->size() && enough_bitrate; ++sl_idx) {
+ RTC_DCHECK_GT(codec_.spatialLayers[sl_idx].maxBitrate, 0);
+ RTC_DCHECK_GE(codec_.spatialLayers[sl_idx].maxBitrate,
+ codec_.spatialLayers[sl_idx].minBitrate);
+
+ const size_t min_bitrate_bps =
+ codec_.spatialLayers[sl_idx].minBitrate * 1000;
+ const size_t max_bitrate_bps =
+ codec_.spatialLayers[sl_idx].maxBitrate * 1000;
+
+ spatial_layer_bitrate_bps->at(sl_idx) += excess_rate;
+ if (spatial_layer_bitrate_bps->at(sl_idx) < max_bitrate_bps) {
+ excess_rate = 0;
+ } else {
+ excess_rate = spatial_layer_bitrate_bps->at(sl_idx) - max_bitrate_bps;
+ spatial_layer_bitrate_bps->at(sl_idx) = max_bitrate_bps;
+ }
+
+ enough_bitrate = (spatial_layer_bitrate_bps->at(sl_idx) >= min_bitrate_bps);
+ }
+
+ return enough_bitrate;
+}
+
+VideoBitrateAllocation SvcRateAllocator::GetAllocationScreenSharing(
+ uint32_t total_bitrate_bps) const {
+ const size_t num_spatial_layers = codec_.VP9().numberOfSpatialLayers;
+ RTC_CHECK(num_spatial_layers > 0);
+ RTC_CHECK_EQ(codec_.VP9().numberOfTemporalLayers, 1U);
+
+ VideoBitrateAllocation bitrate_allocation;
+
+ // Add next layer after bitrate of previous layer has reached its maximum.
+ size_t left_bitrate_bps = total_bitrate_bps;
+ for (size_t sl_idx = 0; sl_idx < num_spatial_layers; ++sl_idx) {
+ const size_t min_bitrate_bps =
+ codec_.spatialLayers[sl_idx].minBitrate * 1000;
+ const size_t max_bitrate_bps =
+ codec_.spatialLayers[sl_idx].maxBitrate * 1000;
+
+ const size_t bitrate_bps = std::min(left_bitrate_bps, max_bitrate_bps);
+ if (bitrate_bps >= min_bitrate_bps) {
+ bitrate_allocation.SetBitrate(sl_idx, 0, bitrate_bps);
+ } else {
+ break;
+ }
+
+ left_bitrate_bps -= bitrate_bps;
+ }
+
+ return bitrate_allocation;
+}
+
uint32_t SvcRateAllocator::GetPreferredBitrateBps(uint32_t framerate) {
return GetAllocation(codec_.maxBitrate * 1000, framerate).get_sum_bps();
}
-std::vector<size_t> SvcRateAllocator::SplitBitrate(size_t num_layers,
- size_t total_bitrate,
- float rate_scaling_factor) {
+std::vector<size_t> SvcRateAllocator::SplitBitrate(
+ size_t num_layers,
+ size_t total_bitrate,
+ float rate_scaling_factor) const {
std::vector<size_t> bitrates;
double denominator = 0.0;
diff --git a/modules/video_coding/codecs/vp9/svc_rate_allocator.h b/modules/video_coding/codecs/vp9/svc_rate_allocator.h
index cbf2096..268f18d 100644
--- a/modules/video_coding/codecs/vp9/svc_rate_allocator.h
+++ b/modules/video_coding/codecs/vp9/svc_rate_allocator.h
@@ -28,9 +28,14 @@
uint32_t GetPreferredBitrateBps(uint32_t framerate_fps) override;
private:
+ VideoBitrateAllocation GetAllocationNormalVideo(
+ uint32_t total_bitrate_bps) const;
+ VideoBitrateAllocation GetAllocationScreenSharing(
+ uint32_t total_bitrate_bps) const;
std::vector<size_t> SplitBitrate(size_t num_layers,
size_t total_bitrate,
- float rate_scaling_factor);
+ float rate_scaling_factor) const;
+ bool AdjustAndVerify(std::vector<size_t>* spatial_layer_bitrate_bps) const;
const VideoCodec codec_;
};
diff --git a/modules/video_coding/codecs/vp9/svc_rate_allocator_unittest.cc b/modules/video_coding/codecs/vp9/svc_rate_allocator_unittest.cc
index 536950e..7058baf 100644
--- a/modules/video_coding/codecs/vp9/svc_rate_allocator_unittest.cc
+++ b/modules/video_coding/codecs/vp9/svc_rate_allocator_unittest.cc
@@ -19,14 +19,17 @@
static VideoCodec Configure(size_t width,
size_t height,
size_t num_spatial_layers,
- size_t num_temporal_layers) {
+ size_t num_temporal_layers,
+ bool is_screen_sharing) {
VideoCodec codec;
codec.width = width;
codec.height = height;
codec.codecType = kVideoCodecVP9;
+ codec.mode = is_screen_sharing ? kScreensharing : kRealtimeVideo;
std::vector<SpatialLayer> spatial_layers =
- GetSvcConfig(width, height, num_spatial_layers, num_temporal_layers);
+ GetSvcConfig(width, height, num_spatial_layers, num_temporal_layers,
+ is_screen_sharing);
RTC_CHECK_LE(spatial_layers.size(), kMaxSpatialLayers);
codec.VP9()->numberOfSpatialLayers =
@@ -43,7 +46,7 @@
} // namespace
TEST(SvcRateAllocatorTest, SingleLayerFor320x180Input) {
- VideoCodec codec = Configure(320, 180, 3, 3);
+ VideoCodec codec = Configure(320, 180, 3, 3, false);
SvcRateAllocator allocator = SvcRateAllocator(codec);
VideoBitrateAllocation allocation = allocator.GetAllocation(1000 * 1000, 30);
@@ -53,7 +56,7 @@
}
TEST(SvcRateAllocatorTest, TwoLayersFor640x360Input) {
- VideoCodec codec = Configure(640, 360, 3, 3);
+ VideoCodec codec = Configure(640, 360, 3, 3, false);
SvcRateAllocator allocator = SvcRateAllocator(codec);
VideoBitrateAllocation allocation = allocator.GetAllocation(1000 * 1000, 30);
@@ -64,7 +67,7 @@
}
TEST(SvcRateAllocatorTest, ThreeLayersFor1280x720Input) {
- VideoCodec codec = Configure(1280, 720, 3, 3);
+ VideoCodec codec = Configure(1280, 720, 3, 3, false);
SvcRateAllocator allocator = SvcRateAllocator(codec);
VideoBitrateAllocation allocation = allocator.GetAllocation(1000 * 1000, 30);
@@ -76,7 +79,7 @@
TEST(SvcRateAllocatorTest,
BaseLayerNonZeroBitrateEvenIfTotalIfLessThanMinimum) {
- VideoCodec codec = Configure(1280, 720, 3, 3);
+ VideoCodec codec = Configure(1280, 720, 3, 3, false);
SvcRateAllocator allocator = SvcRateAllocator(codec);
const SpatialLayer* layers = codec.spatialLayers;
@@ -90,7 +93,7 @@
}
TEST(SvcRateAllocatorTest, Disable640x360Layer) {
- VideoCodec codec = Configure(1280, 720, 3, 3);
+ VideoCodec codec = Configure(1280, 720, 3, 3, false);
SvcRateAllocator allocator = SvcRateAllocator(codec);
const SpatialLayer* layers = codec.spatialLayers;
@@ -106,7 +109,7 @@
}
TEST(SvcRateAllocatorTest, Disable1280x720Layer) {
- VideoCodec codec = Configure(1280, 720, 3, 3);
+ VideoCodec codec = Configure(1280, 720, 3, 3, false);
SvcRateAllocator allocator = SvcRateAllocator(codec);
const SpatialLayer* layers = codec.spatialLayers;
@@ -123,7 +126,7 @@
}
TEST(SvcRateAllocatorTest, BitrateIsCapped) {
- VideoCodec codec = Configure(1280, 720, 3, 3);
+ VideoCodec codec = Configure(1280, 720, 3, 3, false);
SvcRateAllocator allocator = SvcRateAllocator(codec);
const SpatialLayer* layers = codec.spatialLayers;
@@ -139,4 +142,23 @@
EXPECT_EQ(allocation.GetSpatialLayerSum(2) / 1000, layers[2].maxBitrate);
}
+TEST(SvcRateAllocatorTest, MinBitrateToGetQualityLayer) {
+ VideoCodec codec = Configure(1280, 720, 3, 1, true);
+ SvcRateAllocator allocator = SvcRateAllocator(codec);
+
+ const SpatialLayer* layers = codec.spatialLayers;
+
+ EXPECT_LE(codec.VP9()->numberOfSpatialLayers, 2U);
+
+ VideoBitrateAllocation allocation =
+ allocator.GetAllocation(layers[0].minBitrate * 1000, 30);
+ EXPECT_EQ(allocation.GetSpatialLayerSum(0) / 1000, layers[0].minBitrate);
+ EXPECT_EQ(allocation.GetSpatialLayerSum(1), 0UL);
+
+ allocation = allocator.GetAllocation(
+ (layers[0].maxBitrate + layers[1].minBitrate) * 1000, 30);
+ EXPECT_EQ(allocation.GetSpatialLayerSum(0) / 1000, layers[0].maxBitrate);
+ EXPECT_EQ(allocation.GetSpatialLayerSum(1) / 1000, layers[1].minBitrate);
+}
+
} // namespace webrtc
diff --git a/modules/video_coding/codecs/vp9/test/vp9_impl_unittest.cc b/modules/video_coding/codecs/vp9/test/vp9_impl_unittest.cc
index 6fa383b..9b1a016 100644
--- a/modules/video_coding/codecs/vp9/test/vp9_impl_unittest.cc
+++ b/modules/video_coding/codecs/vp9/test/vp9_impl_unittest.cc
@@ -53,8 +53,9 @@
codec_settings_.VP9()->numberOfTemporalLayers = 1;
codec_settings_.VP9()->frameDroppingOn = false;
- std::vector<SpatialLayer> layers = GetSvcConfig(
- codec_settings_.width, codec_settings_.height, num_spatial_layers, 1);
+ std::vector<SpatialLayer> layers =
+ GetSvcConfig(codec_settings_.width, codec_settings_.height,
+ num_spatial_layers, 1, false);
for (size_t i = 0; i < layers.size(); ++i) {
codec_settings_.spatialLayers[i] = layers[i];
}
diff --git a/modules/video_coding/video_codec_initializer.cc b/modules/video_coding/video_codec_initializer.cc
index 192f80f..adc7f32 100644
--- a/modules/video_coding/video_codec_initializer.cc
+++ b/modules/video_coding/video_codec_initializer.cc
@@ -214,7 +214,7 @@
spatial_layers =
GetSvcConfig(video_codec.width, video_codec.height,
video_codec.VP9()->numberOfSpatialLayers,
- video_codec.VP9()->numberOfTemporalLayers);
+ video_codec.VP9()->numberOfTemporalLayers, false);
const bool no_spatial_layering = (spatial_layers.size() == 1);
if (no_spatial_layering) {