Refactor of GetSimulcastConfig & EncoderStreamFactory.
The main pieces of this refactor are splitting up the creation of
simulcast layers for screenshare or the normal case, more consistent
naming, renaming streams to layers and trying to be more explicit with
some of the logic. Also added TODOs for future work to put more
application control into creating simulcast streams.
Bug: webrtc:8785
Change-Id: Ibf49fa0cc6d890ff96f8ee11c89d93a2c94119d6
Reviewed-on: https://webrtc-review.googlesource.com/47580
Commit-Queue: Seth Hampson <shampson@webrtc.org>
Reviewed-by: Peter Thatcher <pthatcher@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#21989}diff --git a/media/engine/simulcast.cc b/media/engine/simulcast.cc
index f2909d3..956e31d 100644
--- a/media/engine/simulcast.cc
+++ b/media/engine/simulcast.cc
@@ -51,7 +51,7 @@
{0, 0, 1, 200, 150, 30}
};
-const int kMaxScreenshareSimulcastStreams = 2;
+const int kMaxScreenshareSimulcastLayers = 2;
// Multiway: Number of temporal layers for each simulcast stream, for maximum
// possible number of simulcast streams |kMaxSimulcastStreams|. The array
@@ -124,125 +124,164 @@
<< " height:" << *height;
}
-int GetTotalMaxBitrateBps(const std::vector<webrtc::VideoStream>& streams) {
- int total_max_bitrate_bps = 0;
- for (size_t s = 0; s < streams.size() - 1; ++s) {
- total_max_bitrate_bps += streams[s].target_bitrate_bps;
+void BoostMaxSimulcastLayer(int max_bitrate_bps,
+ std::vector<webrtc::VideoStream>* layers) {
+ // Spend additional bits to boost the max layer.
+ int bitrate_left_bps = max_bitrate_bps - GetTotalMaxBitrateBps(*layers);
+ if (bitrate_left_bps > 0) {
+ layers->back().max_bitrate_bps += bitrate_left_bps;
}
- total_max_bitrate_bps += streams.back().max_bitrate_bps;
+}
+
+int GetTotalMaxBitrateBps(const std::vector<webrtc::VideoStream>& layers) {
+ int total_max_bitrate_bps = 0;
+ for (size_t s = 0; s < layers.size() - 1; ++s) {
+ total_max_bitrate_bps += layers[s].target_bitrate_bps;
+ }
+ total_max_bitrate_bps += layers.back().max_bitrate_bps;
return total_max_bitrate_bps;
}
-std::vector<webrtc::VideoStream> GetSimulcastConfig(size_t max_streams,
+std::vector<webrtc::VideoStream> GetSimulcastConfig(size_t max_layers,
int width,
int height,
int max_bitrate_bps,
double bitrate_priority,
int max_qp,
int max_framerate,
- bool is_screencast) {
- size_t num_simulcast_layers;
- if (is_screencast) {
- if (UseSimulcastScreenshare()) {
- num_simulcast_layers =
- std::min<int>(max_streams, kMaxScreenshareSimulcastStreams);
- } else {
- num_simulcast_layers = 1;
- }
+ bool is_screenshare) {
+ if (is_screenshare) {
+ return GetScreenshareLayers(max_layers, width, height, max_bitrate_bps,
+ bitrate_priority, max_qp, max_framerate,
+ ScreenshareSimulcastFieldTrialEnabled());
} else {
- num_simulcast_layers = FindSimulcastMaxLayers(width, height);
+ return GetNormalSimulcastLayers(max_layers, width, height, max_bitrate_bps,
+ bitrate_priority, max_qp, max_framerate);
}
+}
- if (num_simulcast_layers > max_streams) {
- // If the number of SSRCs in the group differs from our target
- // number of simulcast streams for current resolution, switch down
- // to a resolution that matches our number of SSRCs.
- SlotSimulcastMaxResolution(max_streams, &width, &height);
- num_simulcast_layers = max_streams;
+std::vector<webrtc::VideoStream> GetNormalSimulcastLayers(
+ size_t max_layers,
+ int width,
+ int height,
+ int max_bitrate_bps,
+ double bitrate_priority,
+ int max_qp,
+ int max_framerate) {
+ // TODO(bugs.webrtc.org/8785): Currently if the resolution isn't large enough
+ // (defined in kSimulcastFormats) we scale down the number of simulcast
+ // layers. Consider changing this so that the application can have more
+ // control over exactly how many simulcast layers are used.
+ size_t num_simulcast_layers = FindSimulcastMaxLayers(width, height);
+ if (num_simulcast_layers > max_layers) {
+ // TODO(bugs.webrtc.org/8486): This scales down the resolution if the
+ // number of simulcast layers created by the application isn't sufficient
+ // (defined in kSimulcastFormats). For example if the input frame's
+ // resolution is HD, but there are only 2 simulcast layers, the
+ // resolution gets scaled down to VGA. Consider taking this logic out to
+ // allow the application more control over the resolutions.
+ SlotSimulcastMaxResolution(max_layers, &width, &height);
+ num_simulcast_layers = max_layers;
}
- std::vector<webrtc::VideoStream> streams;
- streams.resize(num_simulcast_layers);
+ std::vector<webrtc::VideoStream> layers(num_simulcast_layers);
- if (is_screencast) {
- ScreenshareLayerConfig config = ScreenshareLayerConfig::GetDefault();
- // For legacy screenshare in conference mode, tl0 and tl1 bitrates are
- // piggybacked on the VideoCodec struct as target and max bitrates,
- // respectively. See eg. webrtc::VP8EncoderImpl::SetRates().
- streams[0].width = width;
- streams[0].height = height;
- streams[0].max_qp = max_qp;
- streams[0].max_framerate = 5;
- streams[0].min_bitrate_bps = kMinVideoBitrateBps;
- streams[0].target_bitrate_bps = config.tl0_bitrate_kbps * 1000;
- streams[0].max_bitrate_bps = config.tl1_bitrate_kbps * 1000;
- streams[0].temporal_layer_thresholds_bps.clear();
- streams[0].temporal_layer_thresholds_bps.push_back(config.tl0_bitrate_kbps *
- 1000);
+ // Format width and height has to be divisible by |2 ^ num_simulcast_layers -
+ // 1|.
+ width = NormalizeSimulcastSize(width, num_simulcast_layers);
+ height = NormalizeSimulcastSize(height, num_simulcast_layers);
+ // Add simulcast streams, from highest resolution (|s| = num_simulcast_layers
+ // -1) to lowest resolution at |s| = 0.
+ for (size_t s = num_simulcast_layers - 1;; --s) {
+ layers[s].width = width;
+ layers[s].height = height;
+ // TODO(pbos): Fill actual temporal-layer bitrate thresholds.
+ layers[s].max_qp = max_qp;
+ layers[s].temporal_layer_thresholds_bps.resize(
+ kDefaultConferenceNumberOfTemporalLayers[s] - 1);
+ layers[s].max_bitrate_bps = FindSimulcastMaxBitrateBps(width, height);
+ layers[s].target_bitrate_bps = FindSimulcastTargetBitrateBps(width, height);
+ layers[s].min_bitrate_bps = FindSimulcastMinBitrateBps(width, height);
+ layers[s].max_framerate = max_framerate;
- // With simulcast enabled, add another spatial layer. This one will have a
- // more normal layout, with the regular 3 temporal layer pattern and no fps
- // restrictions. The base simulcast stream will still use legacy setup.
- if (num_simulcast_layers == kMaxScreenshareSimulcastStreams) {
- // Add optional upper simulcast layer.
- // Lowest temporal layers of a 3 layer setup will have 40% of the total
- // bitrate allocation for that stream. Make sure the gap between the
- // target of the lower stream and first temporal layer of the higher one
- // is at most 2x the bitrate, so that upswitching is not hampered by
- // stalled bitrate estimates.
- int max_bitrate_bps = 2 * ((streams[0].target_bitrate_bps * 10) / 4);
- // Cap max bitrate so it isn't overly high for the given resolution.
- max_bitrate_bps = std::min<int>(
- max_bitrate_bps, FindSimulcastMaxBitrateBps(width, height));
+ width /= 2;
+ height /= 2;
- streams[1].width = width;
- streams[1].height = height;
- streams[1].max_qp = max_qp;
- streams[1].max_framerate = max_framerate;
- // Three temporal layers means two thresholds.
- streams[1].temporal_layer_thresholds_bps.resize(2);
- streams[1].min_bitrate_bps = streams[0].target_bitrate_bps * 2;
- streams[1].target_bitrate_bps = max_bitrate_bps;
- streams[1].max_bitrate_bps = max_bitrate_bps;
- }
- } else {
- // Format width and height has to be divisible by |2 ^ number_streams - 1|.
- width = NormalizeSimulcastSize(width, num_simulcast_layers);
- height = NormalizeSimulcastSize(height, num_simulcast_layers);
-
- // Add simulcast sub-streams from lower resolution to higher resolutions.
- // Add simulcast streams, from highest resolution (|s| = number_streams -1)
- // to lowest resolution at |s| = 0.
- for (size_t s = num_simulcast_layers - 1;; --s) {
- streams[s].width = width;
- streams[s].height = height;
- // TODO(pbos): Fill actual temporal-layer bitrate thresholds.
- streams[s].max_qp = max_qp;
- streams[s].temporal_layer_thresholds_bps.resize(
- kDefaultConferenceNumberOfTemporalLayers[s] - 1);
- streams[s].max_bitrate_bps = FindSimulcastMaxBitrateBps(width, height);
- streams[s].target_bitrate_bps =
- FindSimulcastTargetBitrateBps(width, height);
- streams[s].min_bitrate_bps = FindSimulcastMinBitrateBps(width, height);
- streams[s].max_framerate = max_framerate;
-
- width /= 2;
- height /= 2;
-
- if (s == 0)
- break;
- }
-
- // Spend additional bits to boost the max stream.
- int bitrate_left_bps = max_bitrate_bps - GetTotalMaxBitrateBps(streams);
- if (bitrate_left_bps > 0) {
- streams.back().max_bitrate_bps += bitrate_left_bps;
+ if (s == 0) {
+ break;
}
}
+ // If there is bitrate leftover, give it to the largest layer.
+ BoostMaxSimulcastLayer(max_bitrate_bps, &layers);
+ // Currently the relative bitrate priority of the sender is controlled by
+ // the value of the lowest VideoStream.
+ // TODO(bugs.webrtc.org/8630): The web specification describes being able to
+ // control relative bitrate for each individual simulcast layer, but this
+ // is currently just implemented per rtp sender.
+ layers[0].bitrate_priority = bitrate_priority;
+ return layers;
+}
+std::vector<webrtc::VideoStream> GetScreenshareLayers(
+ size_t max_layers,
+ int width,
+ int height,
+ int max_bitrate_bps,
+ double bitrate_priority,
+ int max_qp,
+ int max_framerate,
+ bool screenshare_simulcast_enabled) {
+ auto max_screenshare_layers =
+ screenshare_simulcast_enabled ? kMaxScreenshareSimulcastLayers : 1;
+ size_t num_simulcast_layers =
+ std::min<int>(max_layers, max_screenshare_layers);
+
+ std::vector<webrtc::VideoStream> layers(num_simulcast_layers);
+ ScreenshareLayerConfig config = ScreenshareLayerConfig::GetDefault();
+ // For legacy screenshare in conference mode, tl0 and tl1 bitrates are
+ // piggybacked on the VideoCodec struct as target and max bitrates,
+ // respectively. See eg. webrtc::VP8EncoderImpl::SetRates().
+ layers[0].width = width;
+ layers[0].height = height;
+ layers[0].max_qp = max_qp;
+ layers[0].max_framerate = 5;
+ layers[0].min_bitrate_bps = kMinVideoBitrateBps;
+ layers[0].target_bitrate_bps = config.tl0_bitrate_kbps * 1000;
+ layers[0].max_bitrate_bps = config.tl1_bitrate_kbps * 1000;
+ layers[0].temporal_layer_thresholds_bps.clear();
+ layers[0].temporal_layer_thresholds_bps.push_back(config.tl0_bitrate_kbps *
+ 1000);
+
+ // With simulcast enabled, add another spatial layer. This one will have a
+ // more normal layout, with the regular 3 temporal layer pattern and no fps
+ // restrictions. The base simulcast layer will still use legacy setup.
+ if (num_simulcast_layers == kMaxScreenshareSimulcastLayers) {
+ // Add optional upper simulcast layer.
+ // Lowest temporal layers of a 3 layer setup will have 40% of the total
+ // bitrate allocation for that simulcast layer. Make sure the gap between
+ // the target of the lower simulcast layer and first temporal layer of the
+ // higher one is at most 2x the bitrate, so that upswitching is not hampered
+ // by stalled bitrate estimates.
+ int max_bitrate_bps = 2 * ((layers[0].target_bitrate_bps * 10) / 4);
+ // Cap max bitrate so it isn't overly high for the given resolution.
+ max_bitrate_bps = std::min<int>(max_bitrate_bps,
+ FindSimulcastMaxBitrateBps(width, height));
+
+ layers[1].width = width;
+ layers[1].height = height;
+ layers[1].max_qp = max_qp;
+ layers[1].max_framerate = max_framerate;
+ // Three temporal layers means two thresholds.
+ layers[1].temporal_layer_thresholds_bps.resize(2);
+ layers[1].min_bitrate_bps = layers[0].target_bitrate_bps * 2;
+ layers[1].target_bitrate_bps = max_bitrate_bps;
+ layers[1].max_bitrate_bps = max_bitrate_bps;
+ }
+
+ BoostMaxSimulcastLayer(max_bitrate_bps, &layers);
// The bitrate priority currently implemented on a per-sender level, so we
- // just set it for the first video stream.
- streams[0].bitrate_priority = bitrate_priority;
- return streams;
+ // just set it for the first simulcast layer.
+ layers[0].bitrate_priority = bitrate_priority;
+ return layers;
}
static const int kScreenshareMinBitrateKbps = 50;
@@ -250,7 +289,7 @@
static const int kScreenshareDefaultTl0BitrateKbps = 200;
static const int kScreenshareDefaultTl1BitrateKbps = 1000;
-static const char* kScreencastLayerFieldTrialName =
+static const char* kScreenshareLayerFieldTrialName =
"WebRTC-ScreenshareLayerRates";
static const char* kSimulcastScreenshareFieldTrialName =
"WebRTC-SimulcastScreenshare";
@@ -261,7 +300,7 @@
ScreenshareLayerConfig ScreenshareLayerConfig::GetDefault() {
std::string group =
- webrtc::field_trial::FindFullName(kScreencastLayerFieldTrialName);
+ webrtc::field_trial::FindFullName(kScreenshareLayerFieldTrialName);
ScreenshareLayerConfig config(kScreenshareDefaultTl0BitrateKbps,
kScreenshareDefaultTl1BitrateKbps);
@@ -297,7 +336,7 @@
return true;
}
-bool UseSimulcastScreenshare() {
+bool ScreenshareSimulcastFieldTrialEnabled() {
return webrtc::field_trial::IsEnabled(kSimulcastScreenshareFieldTrialName);
}
diff --git a/media/engine/simulcast.h b/media/engine/simulcast.h
index a2d27ed..005c80e 100644
--- a/media/engine/simulcast.h
+++ b/media/engine/simulcast.h
@@ -45,16 +45,38 @@
// Get simulcast settings.
// TODO(sprang): Remove default parameter when it's not longer referenced.
-std::vector<webrtc::VideoStream> GetSimulcastConfig(size_t max_streams,
- int width,
- int height,
- int max_bitrate_bps,
- double bitrate_priority,
- int max_qp,
- int max_framerate,
- bool is_screencast = false);
+std::vector<webrtc::VideoStream> GetSimulcastConfig(
+ size_t max_layers,
+ int width,
+ int height,
+ int max_bitrate_bps,
+ double bitrate_priority,
+ int max_qp,
+ int max_framerate,
+ bool is_screenshare = false);
-bool UseSimulcastScreenshare();
+// Gets the simulcast config layers for a non-screensharing case.
+std::vector<webrtc::VideoStream> GetNormalSimulcastLayers(
+ size_t max_layers,
+ int width,
+ int height,
+ int max_bitrate_bps,
+ double bitrate_priority,
+ int max_qp,
+ int max_framerate);
+
+// Get simulcast config layers for screenshare settings.
+std::vector<webrtc::VideoStream> GetScreenshareLayers(
+ size_t max_layers,
+ int width,
+ int height,
+ int max_bitrate_bps,
+ double bitrate_priority,
+ int max_qp,
+ int max_framerate,
+ bool screenshare_simulcast_enabled);
+
+bool ScreenshareSimulcastFieldTrialEnabled();
} // namespace cricket
diff --git a/media/engine/webrtcvideoengine.cc b/media/engine/webrtcvideoengine.cc
index dbf862a..8aa5aa2 100644
--- a/media/engine/webrtcvideoengine.cc
+++ b/media/engine/webrtcvideoengine.cc
@@ -1896,8 +1896,8 @@
// configure a single stream.
encoder_config.number_of_streams = parameters_.config.rtp.ssrcs.size();
if (IsCodecBlacklistedForSimulcast(codec.name) ||
- (is_screencast &&
- (!UseSimulcastScreenshare() || !parameters_.conference_mode))) {
+ (is_screencast && (!ScreenshareSimulcastFieldTrialEnabled() ||
+ !parameters_.conference_mode))) {
encoder_config.number_of_streams = 1;
}
@@ -2614,23 +2614,31 @@
return video_codecs;
}
-EncoderStreamFactory::EncoderStreamFactory(std::string codec_name,
- int max_qp,
- int max_framerate,
- bool is_screencast,
- bool conference_mode)
+// TODO(bugs.webrtc.org/8785): Consider removing max_qp and max_framerate
+// as members of EncoderStreamFactory and instead set these values individually
+// for each stream in the VideoEncoderConfig.simulcast_layers.
+EncoderStreamFactory::EncoderStreamFactory(
+ std::string codec_name,
+ int max_qp,
+ int max_framerate,
+ bool is_screenshare,
+ bool screenshare_config_explicitly_enabled)
+
: codec_name_(codec_name),
max_qp_(max_qp),
max_framerate_(max_framerate),
- is_screencast_(is_screencast),
- conference_mode_(conference_mode) {}
+ is_screenshare_(is_screenshare),
+ screenshare_config_explicitly_enabled_(
+ screenshare_config_explicitly_enabled) {}
std::vector<webrtc::VideoStream> EncoderStreamFactory::CreateEncoderStreams(
int width,
int height,
const webrtc::VideoEncoderConfig& encoder_config) {
- if (is_screencast_ &&
- (!conference_mode_ || !cricket::UseSimulcastScreenshare())) {
+ bool screenshare_simulcast_enabled =
+ screenshare_config_explicitly_enabled_ &&
+ cricket::ScreenshareSimulcastFieldTrialEnabled();
+ if (is_screenshare_ && !screenshare_simulcast_enabled) {
RTC_DCHECK_EQ(1, encoder_config.number_of_streams);
}
RTC_DCHECK_EQ(encoder_config.simulcast_layers.size(),
@@ -2638,12 +2646,12 @@
std::vector<webrtc::VideoStream> layers;
if (encoder_config.number_of_streams > 1 ||
- (CodecNamesEq(codec_name_, kVp8CodecName) && is_screencast_ &&
- conference_mode_)) {
+ (CodecNamesEq(codec_name_, kVp8CodecName) && is_screenshare_ &&
+ screenshare_config_explicitly_enabled_)) {
layers = GetSimulcastConfig(encoder_config.number_of_streams, width, height,
encoder_config.max_bitrate_bps,
encoder_config.bitrate_priority, max_qp_,
- max_framerate_, is_screencast_);
+ max_framerate_, is_screenshare_);
// Update the active simulcast layers.
for (size_t i = 0; i < layers.size(); ++i) {
layers[i].active = encoder_config.simulcast_layers[i].active;
@@ -2666,7 +2674,7 @@
layer.max_qp = max_qp_;
layer.bitrate_priority = encoder_config.bitrate_priority;
- if (CodecNamesEq(codec_name_, kVp9CodecName) && !is_screencast_) {
+ if (CodecNamesEq(codec_name_, kVp9CodecName) && !is_screenshare_) {
layer.temporal_layer_thresholds_bps.resize(GetDefaultVp9TemporalLayers() -
1);
}
diff --git a/media/engine/webrtcvideoengine.h b/media/engine/webrtcvideoengine.h
index 03fa3e8..f126493 100644
--- a/media/engine/webrtcvideoengine.h
+++ b/media/engine/webrtcvideoengine.h
@@ -516,8 +516,8 @@
EncoderStreamFactory(std::string codec_name,
int max_qp,
int max_framerate,
- bool is_screencast,
- bool conference_mode);
+ bool is_screenshare,
+ bool screenshare_config_explicitly_enabled);
private:
std::vector<webrtc::VideoStream> CreateEncoderStreams(
@@ -528,8 +528,10 @@
const std::string codec_name_;
const int max_qp_;
const int max_framerate_;
- const bool is_screencast_;
- const bool conference_mode_;
+ const bool is_screenshare_;
+ // Allows a screenshare specific configuration, which enables temporal
+ // layering and allows simulcast.
+ const bool screenshare_config_explicitly_enabled_;
};
} // namespace cricket