H.264 temporal layers w/frame marking (PART 2/3)
Bug: None
Change-Id: Id1381d895377d39c3969635e1a59591214aabb71
Reviewed-on: https://webrtc-review.googlesource.com/c/86140
Commit-Queue: Sergey Silkin <ssilkin@webrtc.org>
Reviewed-by: Stefan Holmer <stefan@webrtc.org>
Reviewed-by: Sergey Silkin <ssilkin@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#26624}diff --git a/modules/video_coding/codecs/h264/h264_encoder_impl.cc b/modules/video_coding/codecs/h264/h264_encoder_impl.cc
index 7aeac66..7e9464c 100644
--- a/modules/video_coding/codecs/h264/h264_encoder_impl.cc
+++ b/modules/video_coding/codecs/h264/h264_encoder_impl.cc
@@ -167,7 +167,9 @@
number_of_cores_(0),
encoded_image_callback_(nullptr),
has_reported_init_(false),
- has_reported_error_(false) {
+ has_reported_error_(false),
+ num_temporal_layers_(1),
+ tl0sync_limit_(0) {
RTC_CHECK(absl::EqualsIgnoreCase(codec.name, cricket::kH264CodecName));
std::string packetization_mode_string;
if (codec.GetParam(cricket::kH264FmtpPacketizationMode,
@@ -236,13 +238,10 @@
codec_.simulcastStream[0].height = codec_.height;
}
+ num_temporal_layers_ = codec_.H264()->numberOfTemporalLayers;
+
for (int i = 0, idx = number_of_streams - 1; i < number_of_streams;
++i, --idx) {
- // Temporal layers still not supported.
- if (inst->simulcastStream[i].numberOfTemporalLayers > 1) {
- Release();
- return WEBRTC_VIDEO_CODEC_ERR_SIMULCAST_PARAMETERS_NOT_SUPPORTED;
- }
ISVCEncoder* openh264_encoder;
// Create encoder.
if (WelsCreateSVCEncoder(&openh264_encoder) != 0) {
@@ -530,6 +529,22 @@
codec_specific.codecType = kVideoCodecH264;
codec_specific.codecSpecific.H264.packetization_mode =
packetization_mode_;
+ codec_specific.codecSpecific.H264.temporal_idx = kNoTemporalIdx;
+ codec_specific.codecSpecific.H264.idr_frame =
+ info.eFrameType == videoFrameTypeIDR;
+ codec_specific.codecSpecific.H264.base_layer_sync = false;
+ if (num_temporal_layers_ > 1) {
+ const uint8_t tid = info.sLayerInfo[0].uiTemporalId;
+ codec_specific.codecSpecific.H264.temporal_idx = tid;
+ codec_specific.codecSpecific.H264.base_layer_sync =
+ tid > 0 && tid < tl0sync_limit_;
+ if (codec_specific.codecSpecific.H264.base_layer_sync) {
+ tl0sync_limit_ = tid;
+ }
+ if (tid == 0) {
+ tl0sync_limit_ = num_temporal_layers_;
+ }
+ }
encoded_image_callback_->OnEncodedImage(encoded_images_[i],
&codec_specific, &frag_header);
}
@@ -581,6 +596,10 @@
encoder_params.iTargetBitrate;
encoder_params.sSpatialLayers[0].iMaxSpatialBitrate =
encoder_params.iMaxBitrate;
+ encoder_params.iTemporalLayerNum = num_temporal_layers_;
+ if (encoder_params.iTemporalLayerNum > 1) {
+ encoder_params.iNumRefFrame = 1;
+ }
RTC_LOG(INFO) << "OpenH264 version is " << OPENH264_MAJOR << "."
<< OPENH264_MINOR;
switch (packetization_mode_) {
diff --git a/modules/video_coding/codecs/h264/h264_encoder_impl.h b/modules/video_coding/codecs/h264/h264_encoder_impl.h
index da32563..044b6eb 100644
--- a/modules/video_coding/codecs/h264/h264_encoder_impl.h
+++ b/modules/video_coding/codecs/h264/h264_encoder_impl.h
@@ -100,6 +100,9 @@
bool has_reported_init_;
bool has_reported_error_;
+
+ int num_temporal_layers_;
+ uint8_t tl0sync_limit_;
};
} // namespace webrtc
diff --git a/modules/video_coding/codecs/h264/include/h264_globals.h b/modules/video_coding/codecs/h264/include/h264_globals.h
index e321500..321a6b7 100644
--- a/modules/video_coding/codecs/h264/include/h264_globals.h
+++ b/modules/video_coding/codecs/h264/include/h264_globals.h
@@ -15,6 +15,8 @@
#define MODULES_VIDEO_CODING_CODECS_H264_INCLUDE_H264_GLOBALS_H_
#include <string>
+#include "modules/video_coding/codecs/interface/common_constants.h"
+
#include "rtc_base/checks.h"
diff --git a/modules/video_coding/encoded_frame.cc b/modules/video_coding/encoded_frame.cc
index c18ef13..6e8e342 100644
--- a/modules/video_coding/encoded_frame.cc
+++ b/modules/video_coding/encoded_frame.cc
@@ -135,6 +135,20 @@
}
case kVideoCodecH264: {
_codecSpecificInfo.codecType = kVideoCodecH264;
+
+ // The following H264 codec specific data are not used elsewhere.
+ // Instead they are read directly from the frame marking extension.
+ // These codec specific data structures should be removed
+ // when frame marking is used.
+ _codecSpecificInfo.codecSpecific.H264.temporal_idx = kNoTemporalIdx;
+ if (header->frame_marking.temporal_id != kNoTemporalIdx) {
+ _codecSpecificInfo.codecSpecific.H264.temporal_idx =
+ header->frame_marking.temporal_id;
+ _codecSpecificInfo.codecSpecific.H264.base_layer_sync =
+ header->frame_marking.base_layer_sync;
+ _codecSpecificInfo.codecSpecific.H264.idr_frame =
+ header->frame_marking.independent_frame;
+ }
break;
}
default: {
diff --git a/modules/video_coding/include/video_codec_interface.h b/modules/video_coding/include/video_codec_interface.h
index 052963c..0a6c12d 100644
--- a/modules/video_coding/include/video_codec_interface.h
+++ b/modules/video_coding/include/video_codec_interface.h
@@ -83,6 +83,9 @@
// Hack alert - the code assumes that thisstruct is memset when constructed.
struct CodecSpecificInfoH264 {
H264PacketizationMode packetization_mode;
+ uint8_t temporal_idx;
+ bool base_layer_sync;
+ bool idr_frame;
};
static_assert(std::is_pod<CodecSpecificInfoH264>::value, "");
diff --git a/modules/video_coding/session_info.cc b/modules/video_coding/session_info.cc
index d6b59be..f77f6d6 100644
--- a/modules/video_coding/session_info.cc
+++ b/modules/video_coding/session_info.cc
@@ -95,6 +95,8 @@
return absl::get<RTPVideoHeaderVP9>(
packets_.front().video_header.video_type_header)
.temporal_idx;
+ } else if (packets_.front().video_header.codec == kVideoCodecH264) {
+ return packets_.front().video_header.frame_marking.temporal_id;
} else {
return kNoTemporalIdx;
}
@@ -111,6 +113,8 @@
return absl::get<RTPVideoHeaderVP9>(
packets_.front().video_header.video_type_header)
.temporal_up_switch;
+ } else if (packets_.front().video_header.codec == kVideoCodecH264) {
+ return packets_.front().video_header.frame_marking.base_layer_sync;
} else {
return false;
}
@@ -127,6 +131,8 @@
return absl::get<RTPVideoHeaderVP9>(
packets_.front().video_header.video_type_header)
.tl0_pic_idx;
+ } else if (packets_.front().video_header.codec == kVideoCodecH264) {
+ return packets_.front().video_header.frame_marking.tl0_pic_idx;
} else {
return kNoTl0PicIdx;
}
diff --git a/modules/video_coding/video_codec_initializer.cc b/modules/video_coding/video_codec_initializer.cc
index d67cf52..a2bc27a 100644
--- a/modules/video_coding/video_codec_initializer.cc
+++ b/modules/video_coding/video_codec_initializer.cc
@@ -225,6 +225,12 @@
case kVideoCodecH264: {
if (!config.encoder_specific_settings)
*video_codec.H264() = VideoEncoder::GetDefaultH264Settings();
+ video_codec.H264()->numberOfTemporalLayers = static_cast<unsigned char>(
+ streams.back().num_temporal_layers.value_or(
+ video_codec.H264()->numberOfTemporalLayers));
+ RTC_DCHECK_GE(video_codec.H264()->numberOfTemporalLayers, 1);
+ RTC_DCHECK_LE(video_codec.H264()->numberOfTemporalLayers,
+ kMaxTemporalStreams);
break;
}
default: