Extract color space from H264 decoder
Makes use of ColorSpace class to extract info from H264 stream.
Bug: webrtc:9522
Change-Id: I651d16707260bb2867b1eda95dd4956d62c47279
Reviewed-on: https://webrtc-review.googlesource.com/90180
Reviewed-by: Erik Språng <sprang@webrtc.org>
Reviewed-by: Niklas Enbom <niklas.enbom@webrtc.org>
Commit-Queue: Emircan Uysaler <emircan@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#24085}diff --git a/modules/video_coding/BUILD.gn b/modules/video_coding/BUILD.gn
index 7437603..9e11cce 100644
--- a/modules/video_coding/BUILD.gn
+++ b/modules/video_coding/BUILD.gn
@@ -296,6 +296,7 @@
deps = [
":video_codec_interface",
":video_coding_utility",
+ "../../api/video:video_frame",
"../../api/video:video_frame_i420",
"../../api/video_codecs:video_codecs_api",
"../../media:rtc_h264_profile_id",
@@ -310,6 +311,8 @@
if (rtc_use_h264) {
defines += [ "WEBRTC_USE_H264" ]
sources += [
+ "codecs/h264/h264_color_space.cc",
+ "codecs/h264/h264_color_space.h",
"codecs/h264/h264_decoder_impl.cc",
"codecs/h264/h264_decoder_impl.h",
"codecs/h264/h264_encoder_impl.cc",
diff --git a/modules/video_coding/codecs/h264/h264_color_space.cc b/modules/video_coding/codecs/h264/h264_color_space.cc
new file mode 100644
index 0000000..b48ccfb
--- /dev/null
+++ b/modules/video_coding/codecs/h264/h264_color_space.cc
@@ -0,0 +1,171 @@
+/*
+ * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "modules/video_coding/codecs/h264/h264_color_space.h"
+
+namespace webrtc {
+
+ColorSpace ExtractH264ColorSpace(AVCodecContext* codec) {
+ ColorSpace::PrimaryID primaries = ColorSpace::PrimaryID::kInvalid;
+ switch (codec->color_primaries) {
+ case AVCOL_PRI_BT709:
+ primaries = ColorSpace::PrimaryID::kBT709;
+ break;
+ case AVCOL_PRI_BT470M:
+ primaries = ColorSpace::PrimaryID::kBT470M;
+ break;
+ case AVCOL_PRI_BT470BG:
+ primaries = ColorSpace::PrimaryID::kBT470BG;
+ break;
+ case AVCOL_PRI_SMPTE170M:
+ primaries = ColorSpace::PrimaryID::kSMPTE170M;
+ break;
+ case AVCOL_PRI_SMPTE240M:
+ primaries = ColorSpace::PrimaryID::kSMPTE240M;
+ break;
+ case AVCOL_PRI_FILM:
+ primaries = ColorSpace::PrimaryID::kFILM;
+ break;
+ case AVCOL_PRI_BT2020:
+ primaries = ColorSpace::PrimaryID::kBT2020;
+ break;
+ case AVCOL_PRI_SMPTE428:
+ primaries = ColorSpace::PrimaryID::kSMPTEST428;
+ break;
+ case AVCOL_PRI_SMPTE431:
+ primaries = ColorSpace::PrimaryID::kSMPTEST431;
+ break;
+ case AVCOL_PRI_SMPTE432:
+ primaries = ColorSpace::PrimaryID::kSMPTEST432;
+ break;
+ case AVCOL_PRI_JEDEC_P22:
+ primaries = ColorSpace::PrimaryID::kJEDECP22;
+ break;
+ case AVCOL_PRI_RESERVED0:
+ case AVCOL_PRI_UNSPECIFIED:
+ case AVCOL_PRI_RESERVED:
+ default:
+ break;
+ }
+
+ ColorSpace::TransferID transfer = ColorSpace::TransferID::kInvalid;
+ switch (codec->color_trc) {
+ case AVCOL_TRC_BT709:
+ transfer = ColorSpace::TransferID::kBT709;
+ break;
+ case AVCOL_TRC_GAMMA22:
+ transfer = ColorSpace::TransferID::kGAMMA22;
+ break;
+ case AVCOL_TRC_GAMMA28:
+ transfer = ColorSpace::TransferID::kGAMMA28;
+ break;
+ case AVCOL_TRC_SMPTE170M:
+ transfer = ColorSpace::TransferID::kSMPTE170M;
+ break;
+ case AVCOL_TRC_SMPTE240M:
+ transfer = ColorSpace::TransferID::kSMPTE240M;
+ break;
+ case AVCOL_TRC_LINEAR:
+ transfer = ColorSpace::TransferID::kLINEAR;
+ break;
+ case AVCOL_TRC_LOG:
+ transfer = ColorSpace::TransferID::kLOG;
+ break;
+ case AVCOL_TRC_LOG_SQRT:
+ transfer = ColorSpace::TransferID::kLOG_SQRT;
+ break;
+ case AVCOL_TRC_IEC61966_2_4:
+ transfer = ColorSpace::TransferID::kIEC61966_2_4;
+ break;
+ case AVCOL_TRC_BT1361_ECG:
+ transfer = ColorSpace::TransferID::kBT1361_ECG;
+ break;
+ case AVCOL_TRC_IEC61966_2_1:
+ transfer = ColorSpace::TransferID::kIEC61966_2_1;
+ break;
+ case AVCOL_TRC_BT2020_10:
+ transfer = ColorSpace::TransferID::kBT2020_10;
+ break;
+ case AVCOL_TRC_BT2020_12:
+ transfer = ColorSpace::TransferID::kBT2020_12;
+ break;
+ case AVCOL_TRC_SMPTE2084:
+ transfer = ColorSpace::TransferID::kSMPTEST2084;
+ break;
+ case AVCOL_TRC_SMPTE428:
+ transfer = ColorSpace::TransferID::kSMPTEST428;
+ break;
+ case AVCOL_TRC_ARIB_STD_B67:
+ transfer = ColorSpace::TransferID::kARIB_STD_B67;
+ break;
+ case AVCOL_TRC_RESERVED0:
+ case AVCOL_TRC_UNSPECIFIED:
+ case AVCOL_TRC_RESERVED:
+ default:
+ break;
+ }
+
+ ColorSpace::MatrixID matrix = ColorSpace::MatrixID::kInvalid;
+ switch (codec->colorspace) {
+ case AVCOL_SPC_RGB:
+ matrix = ColorSpace::MatrixID::kRGB;
+ break;
+ case AVCOL_SPC_BT709:
+ matrix = ColorSpace::MatrixID::kBT709;
+ break;
+ case AVCOL_SPC_FCC:
+ matrix = ColorSpace::MatrixID::kFCC;
+ break;
+ case AVCOL_SPC_BT470BG:
+ matrix = ColorSpace::MatrixID::kBT470BG;
+ break;
+ case AVCOL_SPC_SMPTE170M:
+ matrix = ColorSpace::MatrixID::kSMPTE170M;
+ break;
+ case AVCOL_SPC_SMPTE240M:
+ matrix = ColorSpace::MatrixID::kSMPTE240M;
+ break;
+ case AVCOL_SPC_YCGCO:
+ matrix = ColorSpace::MatrixID::kYCOCG;
+ break;
+ case AVCOL_SPC_BT2020_NCL:
+ matrix = ColorSpace::MatrixID::kBT2020_NCL;
+ break;
+ case AVCOL_SPC_BT2020_CL:
+ matrix = ColorSpace::MatrixID::kBT2020_CL;
+ break;
+ case AVCOL_SPC_SMPTE2085:
+ matrix = ColorSpace::MatrixID::kSMPTE2085;
+ break;
+ case AVCOL_SPC_CHROMA_DERIVED_NCL:
+ case AVCOL_SPC_CHROMA_DERIVED_CL:
+ case AVCOL_SPC_ICTCP:
+ case AVCOL_SPC_UNSPECIFIED:
+ case AVCOL_SPC_RESERVED:
+ default:
+ break;
+ }
+
+ ColorSpace::RangeID range = ColorSpace::RangeID::kInvalid;
+ switch (codec->color_range) {
+ case AVCOL_RANGE_MPEG:
+ range = ColorSpace::RangeID::kLimited;
+ break;
+ case AVCOL_RANGE_JPEG:
+ range = ColorSpace::RangeID::kFull;
+ break;
+ case AVCOL_RANGE_UNSPECIFIED:
+ default:
+ break;
+ }
+ return ColorSpace(primaries, transfer, matrix, range);
+}
+
+} // namespace webrtc
diff --git a/modules/video_coding/codecs/h264/h264_color_space.h b/modules/video_coding/codecs/h264/h264_color_space.h
new file mode 100644
index 0000000..5608ab1
--- /dev/null
+++ b/modules/video_coding/codecs/h264/h264_color_space.h
@@ -0,0 +1,27 @@
+/*
+ * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef MODULES_VIDEO_CODING_CODECS_H264_H264_COLOR_SPACE_H_
+#define MODULES_VIDEO_CODING_CODECS_H264_H264_COLOR_SPACE_H_
+
+#include "api/video/color_space.h"
+
+extern "C" {
+#include "third_party/ffmpeg/libavcodec/avcodec.h"
+} // extern "C"
+
+namespace webrtc {
+
+// Helper class for extracting color space information from H264 stream.
+ColorSpace ExtractH264ColorSpace(AVCodecContext* codec);
+
+} // namespace webrtc
+
+#endif // MODULES_VIDEO_CODING_CODECS_H264_H264_COLOR_SPACE_H_
diff --git a/modules/video_coding/codecs/h264/h264_decoder_impl.cc b/modules/video_coding/codecs/h264/h264_decoder_impl.cc
index e91def4..ae3de06 100644
--- a/modules/video_coding/codecs/h264/h264_decoder_impl.cc
+++ b/modules/video_coding/codecs/h264/h264_decoder_impl.cc
@@ -20,8 +20,10 @@
#include "third_party/ffmpeg/libavutil/imgutils.h"
} // extern "C"
+#include "api/video/color_space.h"
#include "api/video/i420_buffer.h"
#include "common_video/include/video_frame_buffer.h"
+#include "modules/video_coding/codecs/h264/h264_color_space.h"
#include "rtc_base/checks.h"
#include "rtc_base/criticalsection.h"
#include "rtc_base/keep_ref_until_done.h"
@@ -288,15 +290,24 @@
RTC_DCHECK_EQ(av_frame_->reordered_opaque, frame_timestamp_us);
// Obtain the |video_frame| containing the decoded image.
- VideoFrame* video_frame = static_cast<VideoFrame*>(
- av_buffer_get_opaque(av_frame_->buf[0]));
- RTC_DCHECK(video_frame);
+ VideoFrame* input_frame =
+ static_cast<VideoFrame*>(av_buffer_get_opaque(av_frame_->buf[0]));
+ RTC_DCHECK(input_frame);
rtc::scoped_refptr<webrtc::I420BufferInterface> i420_buffer =
- video_frame->video_frame_buffer()->GetI420();
+ input_frame->video_frame_buffer()->GetI420();
RTC_CHECK_EQ(av_frame_->data[kYPlaneIndex], i420_buffer->DataY());
RTC_CHECK_EQ(av_frame_->data[kUPlaneIndex], i420_buffer->DataU());
RTC_CHECK_EQ(av_frame_->data[kVPlaneIndex], i420_buffer->DataV());
- video_frame->set_timestamp(input_image._timeStamp);
+
+ const ColorSpace& color_space = ExtractH264ColorSpace(av_context_.get());
+ VideoFrame decoded_frame =
+ VideoFrame::Builder()
+ .set_video_frame_buffer(input_frame->video_frame_buffer())
+ .set_timestamp_us(input_frame->timestamp_us())
+ .set_timestamp_rtp(input_image._timeStamp)
+ .set_rotation(input_frame->rotation())
+ .set_color_space(color_space)
+ .build();
absl::optional<uint8_t> qp;
// TODO(sakal): Maybe it is possible to get QP directly from FFmpeg.
@@ -319,19 +330,24 @@
i420_buffer->DataU(), i420_buffer->StrideU(),
i420_buffer->DataV(), i420_buffer->StrideV(),
rtc::KeepRefUntilDone(i420_buffer)));
- VideoFrame cropped_frame(
- cropped_buf, video_frame->timestamp(), video_frame->render_time_ms(),
- video_frame->rotation());
+ VideoFrame cropped_frame =
+ VideoFrame::Builder()
+ .set_video_frame_buffer(cropped_buf)
+ .set_timestamp_ms(decoded_frame.render_time_ms())
+ .set_timestamp_rtp(decoded_frame.timestamp())
+ .set_rotation(decoded_frame.rotation())
+ .set_color_space(color_space)
+ .build();
// TODO(nisse): Timestamp and rotation are all zero here. Change decoder
// interface to pass a VideoFrameBuffer instead of a VideoFrame?
decoded_image_callback_->Decoded(cropped_frame, absl::nullopt, qp);
} else {
// Return decoded frame.
- decoded_image_callback_->Decoded(*video_frame, absl::nullopt, qp);
+ decoded_image_callback_->Decoded(decoded_frame, absl::nullopt, qp);
}
- // Stop referencing it, possibly freeing |video_frame|.
+ // Stop referencing it, possibly freeing |input_frame|.
av_frame_unref(av_frame_.get());
- video_frame = nullptr;
+ input_frame = nullptr;
return WEBRTC_VIDEO_CODEC_OK;
}
diff --git a/modules/video_coding/codecs/h264/test/h264_impl_unittest.cc b/modules/video_coding/codecs/h264/test/h264_impl_unittest.cc
index 88ef5f2..14bb6bc 100644
--- a/modules/video_coding/codecs/h264/test/h264_impl_unittest.cc
+++ b/modules/video_coding/codecs/h264/test/h264_impl_unittest.cc
@@ -8,6 +8,7 @@
* be found in the AUTHORS file in the root of the source tree.
*/
+#include "api/video/color_space.h"
#include "common_video/libyuv/include/webrtc_libyuv.h"
#include "modules/video_coding/codecs/h264/include/h264.h"
#include "modules/video_coding/codecs/test/video_codec_unittest.h"
@@ -54,6 +55,12 @@
ASSERT_TRUE(WaitForDecodedFrame(&decoded_frame, &decoded_qp));
ASSERT_TRUE(decoded_frame);
EXPECT_GT(I420PSNR(input_frame, decoded_frame.get()), 36);
+
+ const ColorSpace color_space = decoded_frame->color_space().value();
+ EXPECT_EQ(ColorSpace::PrimaryID::kInvalid, color_space.primaries());
+ EXPECT_EQ(ColorSpace::TransferID::kInvalid, color_space.transfer());
+ EXPECT_EQ(ColorSpace::MatrixID::kInvalid, color_space.matrix());
+ EXPECT_EQ(ColorSpace::RangeID::kLimited, color_space.range());
}
TEST_F(TestH264Impl, MAYBE_DecodedQpEqualsEncodedQp) {