Reland #2 of Issue 2434073003: Extract bitrate allocation ...
This is yet another reland of https://codereview.webrtc.org/2434073003/
including two fixes:
1. SimulcastRateAllocator did not handle the screenshare settings properly for numSimulcastStreams = 1. Additional test case was added for that.
2. In VideoSender, when rate allocation is updated after setting a new VideoCodec config, only update the state of the EncoderParameters, but don't actually run SetRateAllocation on the encoder itself. This caused some problems upstreams.
Please review only the changes after patch set 1.
Original description:
Extract bitrate allocation of spatial/temporal layers out of codec impl.
This CL makes a number of intervowen changes:
* Add BitrateAllocation struct, that contains a codec independent view
of how the target bitrate is distributed over spatial and temporal
layers.
* Adds the BitrateAllocator interface, which takes a bitrate and frame
rate and produces a BitrateAllocation.
* A default (non layered) implementation is added, and
SimulcastRateAllocator is extended to fully handle VP8 allocation.
This includes capturing TemporalLayer instances created by the
encoder.
* ViEEncoder now owns both the bitrate allocator and the temporal layer
factories for VP8. This allows allocation to happen fully outside of
the encoder implementation.
This refactoring will make it possible for ViEEncoder to signal the
full picture of target bitrates to the RTCP module.
BUG=webrtc:6301
R=stefan@webrtc.org
Review URL: https://codereview.webrtc.org/2510583002 .
Cr-Commit-Position: refs/heads/master@{#15105}
diff --git a/webrtc/modules/video_coding/video_codec_initializer.cc b/webrtc/modules/video_coding/video_codec_initializer.cc
new file mode 100644
index 0000000..5d32fd2
--- /dev/null
+++ b/webrtc/modules/video_coding/video_codec_initializer.cc
@@ -0,0 +1,212 @@
+/*
+ * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "webrtc/modules/video_coding/include/video_codec_initializer.h"
+
+#include "webrtc/base/basictypes.h"
+#include "webrtc/common_video/include/video_bitrate_allocator.h"
+#include "webrtc/common_types.h"
+#include "webrtc/modules/video_coding/codecs/vp8/screenshare_layers.h"
+#include "webrtc/modules/video_coding/codecs/vp8/temporal_layers.h"
+#include "webrtc/modules/video_coding/utility/simulcast_rate_allocator.h"
+#include "webrtc/modules/video_coding/utility/default_video_bitrate_allocator.h"
+#include "webrtc/system_wrappers/include/clock.h"
+
+namespace webrtc {
+
+bool VideoCodecInitializer::SetupCodec(
+ const VideoEncoderConfig& config,
+ const VideoSendStream::Config::EncoderSettings settings,
+ const std::vector<VideoStream>& streams,
+ VideoCodec* codec,
+ std::unique_ptr<VideoBitrateAllocator>* bitrate_allocator) {
+ *codec = VideoEncoderConfigToVideoCodec(
+ config, streams, settings.payload_name, settings.payload_type);
+
+ std::unique_ptr<TemporalLayersFactory> tl_factory;
+ switch (codec->codecType) {
+ case kVideoCodecVP8: {
+ if (!codec->VP8()->tl_factory) {
+ if (codec->mode == kScreensharing &&
+ codec->numberOfSimulcastStreams == 1 &&
+ codec->VP8()->numberOfTemporalLayers == 2) {
+ // Conference mode temporal layering for screen content.
+ tl_factory.reset(new ScreenshareTemporalLayersFactory());
+ } else {
+ // Standard video temporal layers.
+ tl_factory.reset(new TemporalLayersFactory());
+ }
+ codec->VP8()->tl_factory = tl_factory.get();
+ }
+ break;
+ }
+ default: {
+ // TODO(sprang): Warn, once we have specific allocators for all supported
+ // codec types.
+ break;
+ }
+ }
+ *bitrate_allocator = CreateBitrateAllocator(*codec, std::move(tl_factory));
+
+ return true;
+}
+
+std::unique_ptr<VideoBitrateAllocator>
+VideoCodecInitializer::CreateBitrateAllocator(
+ const VideoCodec& codec,
+ std::unique_ptr<TemporalLayersFactory> tl_factory) {
+ std::unique_ptr<VideoBitrateAllocator> rate_allocator;
+
+ switch (codec.codecType) {
+ case kVideoCodecVP8: {
+ // Set up default VP8 temporal layer factory, if not provided.
+ rate_allocator.reset(
+ new SimulcastRateAllocator(codec, std::move(tl_factory)));
+ } break;
+ default:
+ rate_allocator.reset(new DefaultVideoBitrateAllocator(codec));
+ }
+
+ return rate_allocator;
+}
+
+// TODO(sprang): Split this up and separate the codec specific parts.
+VideoCodec VideoCodecInitializer::VideoEncoderConfigToVideoCodec(
+ const VideoEncoderConfig& config,
+ const std::vector<VideoStream>& streams,
+ const std::string& payload_name,
+ int payload_type) {
+ static const int kEncoderMinBitrateKbps = 30;
+ RTC_DCHECK(!streams.empty());
+ RTC_DCHECK_GE(config.min_transmit_bitrate_bps, 0);
+
+ VideoCodec video_codec;
+ memset(&video_codec, 0, sizeof(video_codec));
+ video_codec.codecType = PayloadNameToCodecType(payload_name)
+ .value_or(VideoCodecType::kVideoCodecGeneric);
+
+ switch (config.content_type) {
+ case VideoEncoderConfig::ContentType::kRealtimeVideo:
+ video_codec.mode = kRealtimeVideo;
+ break;
+ case VideoEncoderConfig::ContentType::kScreen:
+ video_codec.mode = kScreensharing;
+ if (streams.size() == 1 &&
+ streams[0].temporal_layer_thresholds_bps.size() == 1) {
+ video_codec.targetBitrate =
+ streams[0].temporal_layer_thresholds_bps[0] / 1000;
+ }
+ break;
+ }
+
+ if (config.encoder_specific_settings)
+ config.encoder_specific_settings->FillEncoderSpecificSettings(&video_codec);
+
+ switch (video_codec.codecType) {
+ case kVideoCodecVP8: {
+ if (!config.encoder_specific_settings)
+ *video_codec.VP8() = VideoEncoder::GetDefaultVp8Settings();
+ video_codec.VP8()->numberOfTemporalLayers = static_cast<unsigned char>(
+ streams.back().temporal_layer_thresholds_bps.size() + 1);
+ break;
+ }
+ case kVideoCodecVP9: {
+ if (!config.encoder_specific_settings)
+ *video_codec.VP9() = VideoEncoder::GetDefaultVp9Settings();
+ if (video_codec.mode == kScreensharing &&
+ config.encoder_specific_settings) {
+ video_codec.VP9()->flexibleMode = true;
+ // For now VP9 screensharing use 1 temporal and 2 spatial layers.
+ RTC_DCHECK_EQ(1, video_codec.VP9()->numberOfTemporalLayers);
+ RTC_DCHECK_EQ(2, video_codec.VP9()->numberOfSpatialLayers);
+ }
+ video_codec.VP9()->numberOfTemporalLayers = static_cast<unsigned char>(
+ streams.back().temporal_layer_thresholds_bps.size() + 1);
+ break;
+ }
+ case kVideoCodecH264: {
+ if (!config.encoder_specific_settings)
+ *video_codec.H264() = VideoEncoder::GetDefaultH264Settings();
+ break;
+ }
+ default:
+ // TODO(pbos): Support encoder_settings codec-agnostically.
+ RTC_DCHECK(!config.encoder_specific_settings)
+ << "Encoder-specific settings for codec type not wired up.";
+ break;
+ }
+
+ strncpy(video_codec.plName, payload_name.c_str(), kPayloadNameSize - 1);
+ video_codec.plName[kPayloadNameSize - 1] = '\0';
+ video_codec.plType = payload_type;
+ video_codec.numberOfSimulcastStreams =
+ static_cast<unsigned char>(streams.size());
+ video_codec.minBitrate = streams[0].min_bitrate_bps / 1000;
+ if (video_codec.minBitrate < kEncoderMinBitrateKbps)
+ video_codec.minBitrate = kEncoderMinBitrateKbps;
+ RTC_DCHECK_LE(streams.size(), static_cast<size_t>(kMaxSimulcastStreams));
+ if (video_codec.codecType == kVideoCodecVP9) {
+ // If the vector is empty, bitrates will be configured automatically.
+ RTC_DCHECK(config.spatial_layers.empty() ||
+ config.spatial_layers.size() ==
+ video_codec.VP9()->numberOfSpatialLayers);
+ RTC_DCHECK_LE(video_codec.VP9()->numberOfSpatialLayers,
+ kMaxSimulcastStreams);
+ for (size_t i = 0; i < config.spatial_layers.size(); ++i)
+ video_codec.spatialLayers[i] = config.spatial_layers[i];
+ }
+ for (size_t i = 0; i < streams.size(); ++i) {
+ SimulcastStream* sim_stream = &video_codec.simulcastStream[i];
+ RTC_DCHECK_GT(streams[i].width, 0u);
+ RTC_DCHECK_GT(streams[i].height, 0u);
+ RTC_DCHECK_GT(streams[i].max_framerate, 0);
+ // Different framerates not supported per stream at the moment.
+ RTC_DCHECK_EQ(streams[i].max_framerate, streams[0].max_framerate);
+ RTC_DCHECK_GE(streams[i].min_bitrate_bps, 0);
+ RTC_DCHECK_GE(streams[i].target_bitrate_bps, streams[i].min_bitrate_bps);
+ RTC_DCHECK_GE(streams[i].max_bitrate_bps, streams[i].target_bitrate_bps);
+ RTC_DCHECK_GE(streams[i].max_qp, 0);
+
+ sim_stream->width = static_cast<uint16_t>(streams[i].width);
+ sim_stream->height = static_cast<uint16_t>(streams[i].height);
+ sim_stream->minBitrate = streams[i].min_bitrate_bps / 1000;
+ sim_stream->targetBitrate = streams[i].target_bitrate_bps / 1000;
+ sim_stream->maxBitrate = streams[i].max_bitrate_bps / 1000;
+ sim_stream->qpMax = streams[i].max_qp;
+ sim_stream->numberOfTemporalLayers = static_cast<unsigned char>(
+ streams[i].temporal_layer_thresholds_bps.size() + 1);
+
+ video_codec.width =
+ std::max(video_codec.width, static_cast<uint16_t>(streams[i].width));
+ video_codec.height =
+ std::max(video_codec.height, static_cast<uint16_t>(streams[i].height));
+ video_codec.minBitrate =
+ std::min(static_cast<uint16_t>(video_codec.minBitrate),
+ static_cast<uint16_t>(streams[i].min_bitrate_bps / 1000));
+ video_codec.maxBitrate += streams[i].max_bitrate_bps / 1000;
+ video_codec.qpMax = std::max(video_codec.qpMax,
+ static_cast<unsigned int>(streams[i].max_qp));
+ }
+
+ if (video_codec.maxBitrate == 0) {
+ // Unset max bitrate -> cap to one bit per pixel.
+ video_codec.maxBitrate =
+ (video_codec.width * video_codec.height * video_codec.maxFramerate) /
+ 1000;
+ }
+ if (video_codec.maxBitrate < kEncoderMinBitrateKbps)
+ video_codec.maxBitrate = kEncoderMinBitrateKbps;
+
+ RTC_DCHECK_GT(streams[0].max_framerate, 0);
+ video_codec.maxFramerate = streams[0].max_framerate;
+ return video_codec;
+}
+
+} // namespace webrtc