VideoFrame: Store a reference to an encoded frame
Enable webrtc::VideoFrame to store a reference to an encoded frame.
Bug: chromium:1013590
Change-Id: Id5a06f1c7249f104dfd328f08677cf8001958f0d
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/158788
Reviewed-by: Stefan Holmer <stefan@webrtc.org>
Reviewed-by: Philip Eliasson <philipel@webrtc.org>
Reviewed-by: Ilya Nikolaevskiy <ilnik@webrtc.org>
Reviewed-by: Niels Moller <nisse@webrtc.org>
Commit-Queue: Markus Handell <handellm@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#29809}
diff --git a/api/video/BUILD.gn b/api/video/BUILD.gn
index 1473229..41698f0 100644
--- a/api/video/BUILD.gn
+++ b/api/video/BUILD.gn
@@ -47,6 +47,7 @@
deps = [
":video_rtp_headers",
"..:array_view",
+ "..:refcountedbase",
"..:rtp_packet_info",
"..:scoped_refptr",
"../../rtc_base:checks",
diff --git a/api/video/DEPS b/api/video/DEPS
index 3f5df95..555f7e1 100644
--- a/api/video/DEPS
+++ b/api/video/DEPS
@@ -19,6 +19,7 @@
],
"video_frame\.h": [
+ "+rtc_base/ref_count.h",
],
"video_frame_buffer\.h": [
diff --git a/api/video/video_frame.cc b/api/video/video_frame.cc
index 0e6a611..63902af 100644
--- a/api/video/video_frame.cc
+++ b/api/video/video_frame.cc
@@ -68,7 +68,7 @@
RTC_CHECK(video_frame_buffer_ != nullptr);
return VideoFrame(id_, video_frame_buffer_, timestamp_us_, timestamp_rtp_,
ntp_time_ms_, rotation_, color_space_, update_rect_,
- packet_infos_);
+ packet_infos_, encoded_frame_buffer_);
}
VideoFrame::Builder& VideoFrame::Builder::set_video_frame_buffer(
@@ -135,6 +135,13 @@
return *this;
}
+VideoFrame::Builder& VideoFrame::Builder::set_encoded_video_frame_buffer(
+ rtc::scoped_refptr<VideoFrame::EncodedVideoFrameBuffer>
+ encoded_frame_buffer) {
+ encoded_frame_buffer_ = std::move(encoded_frame_buffer);
+ return *this;
+}
+
VideoFrame::VideoFrame(const rtc::scoped_refptr<VideoFrameBuffer>& buffer,
webrtc::VideoRotation rotation,
int64_t timestamp_us)
@@ -156,17 +163,20 @@
RTC_DCHECK(buffer);
}
-VideoFrame::VideoFrame(uint16_t id,
- const rtc::scoped_refptr<VideoFrameBuffer>& buffer,
- int64_t timestamp_us,
- uint32_t timestamp_rtp,
- int64_t ntp_time_ms,
- VideoRotation rotation,
- const absl::optional<ColorSpace>& color_space,
- const absl::optional<UpdateRect>& update_rect,
- RtpPacketInfos packet_infos)
+VideoFrame::VideoFrame(
+ uint16_t id,
+ const rtc::scoped_refptr<VideoFrameBuffer>& buffer,
+ int64_t timestamp_us,
+ uint32_t timestamp_rtp,
+ int64_t ntp_time_ms,
+ VideoRotation rotation,
+ const absl::optional<ColorSpace>& color_space,
+ const absl::optional<UpdateRect>& update_rect,
+ RtpPacketInfos packet_infos,
+ const rtc::scoped_refptr<EncodedVideoFrameBuffer>& encoded_frame_buffer)
: id_(id),
video_frame_buffer_(buffer),
+ encoded_frame_buffer_(encoded_frame_buffer),
timestamp_rtp_(timestamp_rtp),
ntp_time_ms_(ntp_time_ms),
timestamp_us_(timestamp_us),
@@ -215,4 +225,14 @@
return timestamp_us() / rtc::kNumMicrosecsPerMillisec;
}
+void VideoFrame::set_encoded_video_frame_buffer(
+ rtc::scoped_refptr<EncodedVideoFrameBuffer> encoded_frame_buffer) {
+ encoded_frame_buffer_ = std::move(encoded_frame_buffer);
+}
+
+rtc::scoped_refptr<VideoFrame::EncodedVideoFrameBuffer>
+VideoFrame::encoded_video_frame_buffer() const {
+ return encoded_frame_buffer_;
+}
+
} // namespace webrtc
diff --git a/api/video/video_frame.h b/api/video/video_frame.h
index 7c512ca..338e2fd 100644
--- a/api/video/video_frame.h
+++ b/api/video/video_frame.h
@@ -13,16 +13,20 @@
#include <stdint.h>
+#include <memory>
#include <utility>
#include "absl/types/optional.h"
+#include "api/array_view.h"
#include "api/rtp_packet_infos.h"
#include "api/scoped_refptr.h"
#include "api/video/color_space.h"
#include "api/video/hdr_metadata.h"
+#include "api/video/video_codec_type.h"
#include "api/video/video_frame_buffer.h"
#include "api/video/video_rotation.h"
#include "rtc_base/checks.h"
+#include "rtc_base/ref_count.h"
#include "rtc_base/system/rtc_export.h"
namespace webrtc {
@@ -47,6 +51,23 @@
bool IsEmpty() const;
};
+ // Interface for accessing elements of the encoded frame that was the base for
+ // the rest of the VideoFrame.
+ class EncodedVideoFrameBuffer : public rtc::RefCountInterface {
+ public:
+ // Returns a span of the bitstream data.
+ virtual rtc::ArrayView<const uint8_t> data() const = 0;
+
+ // Returns the colorspace of the encoded frame, or nullptr if not present
+ virtual const webrtc::ColorSpace* color_space() const = 0;
+
+ // Returns the codec of the encoded frame
+ virtual VideoCodecType codec() const = 0;
+
+ // Returns wether the encoded frame is a keyframe
+ virtual bool is_key_frame() const = 0;
+ };
+
// Preferred way of building VideoFrame objects.
class RTC_EXPORT Builder {
public:
@@ -66,6 +87,8 @@
Builder& set_id(uint16_t id);
Builder& set_update_rect(const UpdateRect& update_rect);
Builder& set_packet_infos(RtpPacketInfos packet_infos);
+ Builder& set_encoded_video_frame_buffer(
+ rtc::scoped_refptr<EncodedVideoFrameBuffer> encoded_frame_buffer);
private:
uint16_t id_ = 0;
@@ -77,6 +100,7 @@
absl::optional<ColorSpace> color_space_;
absl::optional<UpdateRect> update_rect_;
RtpPacketInfos packet_infos_;
+ rtc::scoped_refptr<EncodedVideoFrameBuffer> encoded_frame_buffer_;
};
// To be deprecated. Migrate all use to Builder.
@@ -166,6 +190,12 @@
void set_video_frame_buffer(
const rtc::scoped_refptr<VideoFrameBuffer>& buffer);
+ void set_encoded_video_frame_buffer(
+ rtc::scoped_refptr<EncodedVideoFrameBuffer> encoded_frame_buffer);
+
+ rtc::scoped_refptr<EncodedVideoFrameBuffer> encoded_video_frame_buffer()
+ const;
+
// TODO(nisse): Deprecated.
// Return true if the frame is stored in a texture.
bool is_texture() const {
@@ -207,11 +237,14 @@
VideoRotation rotation,
const absl::optional<ColorSpace>& color_space,
const absl::optional<UpdateRect>& update_rect,
- RtpPacketInfos packet_infos);
+ RtpPacketInfos packet_infos,
+ const rtc::scoped_refptr<EncodedVideoFrameBuffer>& encoded_frame);
uint16_t id_;
- // An opaque reference counted handle that stores the pixel data.
+ // A reference counted handle that stores the pixel data.
rtc::scoped_refptr<webrtc::VideoFrameBuffer> video_frame_buffer_;
+ // A reference counted handle that points to an encoded frame
+ rtc::scoped_refptr<EncodedVideoFrameBuffer> encoded_frame_buffer_;
uint32_t timestamp_rtp_;
int64_t ntp_time_ms_;
int64_t timestamp_us_;
diff --git a/common_video/BUILD.gn b/common_video/BUILD.gn
index d22e4fd..bd440ef 100644
--- a/common_video/BUILD.gn
+++ b/common_video/BUILD.gn
@@ -99,6 +99,7 @@
"../:webrtc_common",
"../api:scoped_refptr",
"../api/units:time_delta",
+ "../api/video:encoded_frame",
"../api/video:video_frame",
"../api/video:video_frame_i010",
"../api/video:video_frame_i420",
diff --git a/common_video/video_frame_unittest.cc b/common_video/video_frame_unittest.cc
index 57298d1..f7a27be7 100644
--- a/common_video/video_frame_unittest.cc
+++ b/common_video/video_frame_unittest.cc
@@ -362,6 +362,39 @@
EXPECT_EQ(20, frame.timestamp_us());
}
+class TestEncodedFrame : public VideoFrame::EncodedVideoFrameBuffer {
+ public:
+ rtc::ArrayView<const uint8_t> data() const override {
+ return rtc::ArrayView<const uint8_t>();
+ }
+ webrtc::ColorSpace* color_space() const override { return nullptr; }
+ VideoCodecType codec() const override { return kVideoCodecGeneric; }
+ bool is_key_frame() const { return false; }
+};
+
+TEST(TestVideoFrame, AcceptsEncodedFrameSource) {
+ VideoFrame frame =
+ VideoFrame::Builder()
+ .set_video_frame_buffer(I420Buffer::Create(10, 10, 10, 14, 90))
+ .build();
+ EXPECT_EQ(frame.encoded_video_frame_buffer(), nullptr);
+ auto encoded_frame = new rtc::RefCountedObject<TestEncodedFrame>();
+ frame.set_encoded_video_frame_buffer(encoded_frame);
+ EXPECT_EQ(frame.encoded_video_frame_buffer(), encoded_frame);
+}
+
+TEST(TestVideoFrame, CopiesWithSameEncodedFrameSource) {
+ VideoFrame frame =
+ VideoFrame::Builder()
+ .set_video_frame_buffer(I420Buffer::Create(10, 10, 10, 14, 90))
+ .set_encoded_video_frame_buffer(
+ new rtc::RefCountedObject<TestEncodedFrame>())
+ .build();
+ VideoFrame frame2 = frame;
+ EXPECT_EQ(frame.encoded_video_frame_buffer().get(),
+ frame2.encoded_video_frame_buffer().get());
+}
+
class TestPlanarYuvBuffer
: public ::testing::TestWithParam<VideoFrameBuffer::Type> {};