Injectable software video codecs in Obj-C.
When injecting video codec factories in the Obj-C SDK, use the new
peer connection API that uses webrtc::Video{De,En}CoderFactory classes
and does not automatically add internal software codecs. Instead the
injected factory can support internal VP8 and VP9 codecs through the
included Obj-C classes RTCVideo{De,En}coderVP{8,9}.
When not explicitly injecting any video codec factory, the old code
path is still used and injects only H264 as an external codec and
the internal codec factory is used.
Bug: webrtc:7925
Change-Id: I657d30dfde71da9c0be341e213ab9f97a04caa58
Reviewed-on: https://webrtc-review.googlesource.com/3620
Commit-Queue: Anders Carlsson <andersc@webrtc.org>
Reviewed-by: Magnus Jedvert <magjed@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#20175}diff --git a/sdk/objc/Framework/Classes/VideoToolbox/RTCVideoEncoderH264.mm b/sdk/objc/Framework/Classes/VideoToolbox/RTCVideoEncoderH264.mm
index a953ce0..7a6629e 100644
--- a/sdk/objc/Framework/Classes/VideoToolbox/RTCVideoEncoderH264.mm
+++ b/sdk/objc/Framework/Classes/VideoToolbox/RTCVideoEncoderH264.mm
@@ -169,9 +169,9 @@
// specific VideoToolbox profile for the specified level, AutoLevel will be
// returned. The user must initialize the encoder with a resolution and
// framerate conforming to the selected H264 level regardless.
-CFStringRef ExtractProfile(const cricket::VideoCodec &codec) {
+CFStringRef ExtractProfile(webrtc::SdpVideoFormat videoFormat) {
const rtc::Optional<webrtc::H264::ProfileLevelId> profile_level_id =
- webrtc::H264::ParseSdpProfileLevelId(codec.params);
+ webrtc::H264::ParseSdpProfileLevelId(videoFormat.parameters);
RTC_DCHECK(profile_level_id);
switch (profile_level_id->profile) {
case webrtc::H264::kProfileConstrainedBaseline:
@@ -302,7 +302,7 @@
_bitrateAdjuster.reset(new webrtc::BitrateAdjuster(
webrtc::Clock::GetRealTimeClock(), .5, .95));
_packetizationMode = RTCH264PacketizationModeNonInterleaved;
- _profile = ExtractProfile([codecInfo nativeVideoCodec]);
+ _profile = ExtractProfile([codecInfo nativeSdpVideoFormat]);
LOG(LS_INFO) << "Using profile " << CFStringToString(_profile);
RTC_CHECK([codecInfo.name isEqualToString:@"H264"]);
diff --git a/sdk/objc/Framework/Classes/VideoToolbox/objc_video_decoder_factory.h b/sdk/objc/Framework/Classes/VideoToolbox/objc_video_decoder_factory.h
index 93510e7..a0106ff 100644
--- a/sdk/objc/Framework/Classes/VideoToolbox/objc_video_decoder_factory.h
+++ b/sdk/objc/Framework/Classes/VideoToolbox/objc_video_decoder_factory.h
@@ -11,6 +11,7 @@
#ifndef SDK_OBJC_FRAMEWORK_CLASSES_PEERCONNECTION_OBJC_VIDEO_DECODER_FACTORY_H_
#define SDK_OBJC_FRAMEWORK_CLASSES_PEERCONNECTION_OBJC_VIDEO_DECODER_FACTORY_H_
+#include "api/video_codecs/video_decoder_factory.h"
#include "media/base/codec.h"
#include "media/engine/webrtcvideodecoderfactory.h"
@@ -18,22 +19,30 @@
namespace webrtc {
-class ObjCVideoDecoderFactory : public cricket::WebRtcVideoDecoderFactory {
+// TODO(andersc): Remove the inheritance from cricket::WebRtcVideoDecoderFactory
+// when the legacy path in [RTCPeerConnectionFactory init] is no longer needed.
+class ObjCVideoDecoderFactory : public VideoDecoderFactory,
+ public cricket::WebRtcVideoDecoderFactory {
public:
explicit ObjCVideoDecoderFactory(id<RTCVideoDecoderFactory>);
~ObjCVideoDecoderFactory();
id<RTCVideoDecoderFactory> wrapped_decoder_factory() const;
- VideoDecoder* CreateVideoDecoderWithParams(
+ std::vector<SdpVideoFormat> GetSupportedFormats() const override;
+ std::unique_ptr<VideoDecoder> CreateVideoDecoder(
+ const SdpVideoFormat& format) override;
+
+ // Needed for WebRtcVideoDecoderFactory interface.
+ webrtc::VideoDecoder* CreateVideoDecoderWithParams(
const cricket::VideoCodec& codec,
cricket::VideoDecoderParams params) override;
-
+ webrtc::VideoDecoder* CreateVideoDecoder(
+ webrtc::VideoCodecType type) override;
void DestroyVideoDecoder(webrtc::VideoDecoder* decoder) override;
private:
id<RTCVideoDecoderFactory> decoder_factory_;
- std::vector<cricket::VideoCodec> supported_codecs_;
};
} // namespace webrtc
diff --git a/sdk/objc/Framework/Classes/VideoToolbox/objc_video_decoder_factory.mm b/sdk/objc/Framework/Classes/VideoToolbox/objc_video_decoder_factory.mm
index e20f1ae..ab615a3 100644
--- a/sdk/objc/Framework/Classes/VideoToolbox/objc_video_decoder_factory.mm
+++ b/sdk/objc/Framework/Classes/VideoToolbox/objc_video_decoder_factory.mm
@@ -12,12 +12,14 @@
#import "NSString+StdString.h"
#import "RTCVideoCodec+Private.h"
+#import "RTCWrappedNativeVideoDecoder.h"
#import "WebRTC/RTCVideoCodec.h"
#import "WebRTC/RTCVideoCodecFactory.h"
#import "WebRTC/RTCVideoCodecH264.h"
#import "WebRTC/RTCVideoFrame.h"
#import "WebRTC/RTCVideoFrameBuffer.h"
+#include "api/video_codecs/sdp_video_format.h"
#include "api/video_codecs/video_decoder.h"
#include "modules/include/module_common_types.h"
#include "modules/video_coding/include/video_codec_interface.h"
@@ -103,19 +105,47 @@
return decoder_factory_;
}
-VideoDecoder *ObjCVideoDecoderFactory::CreateVideoDecoderWithParams(
- const cricket::VideoCodec &codec, cricket::VideoDecoderParams params) {
- NSString *codecName = [NSString stringWithUTF8String:codec.name.c_str()];
+std::unique_ptr<VideoDecoder> ObjCVideoDecoderFactory::CreateVideoDecoder(
+ const SdpVideoFormat &format) {
+ NSString *codecName = [NSString stringWithUTF8String:format.name.c_str()];
for (RTCVideoCodecInfo *codecInfo in decoder_factory_.supportedCodecs) {
if ([codecName isEqualToString:codecInfo.name]) {
id<RTCVideoDecoder> decoder = [decoder_factory_ createDecoder:codecInfo];
- return new ObjCVideoDecoder(decoder);
+
+ if ([decoder isKindOfClass:[RTCWrappedNativeVideoDecoder class]]) {
+ return [(RTCWrappedNativeVideoDecoder *)decoder releaseWrappedDecoder];
+ } else {
+ return std::unique_ptr<ObjCVideoDecoder>(new ObjCVideoDecoder(decoder));
+ }
}
}
return nullptr;
}
+std::vector<SdpVideoFormat> ObjCVideoDecoderFactory::GetSupportedFormats() const {
+ std::vector<SdpVideoFormat> supported_formats;
+ for (RTCVideoCodecInfo *supportedCodec in decoder_factory_.supportedCodecs) {
+ SdpVideoFormat format = [supportedCodec nativeSdpVideoFormat];
+ supported_formats.push_back(format);
+ }
+
+ return supported_formats;
+}
+
+// WebRtcVideoDecoderFactory
+
+VideoDecoder *ObjCVideoDecoderFactory::CreateVideoDecoderWithParams(
+ const cricket::VideoCodec &codec, cricket::VideoDecoderParams params) {
+ return CreateVideoDecoder(SdpVideoFormat(codec.name, codec.params)).release();
+}
+
+VideoDecoder *ObjCVideoDecoderFactory::CreateVideoDecoder(VideoCodecType type) {
+ // This is implemented to avoid hiding an overloaded virtual function
+ RTC_NOTREACHED();
+ return nullptr;
+}
+
void ObjCVideoDecoderFactory::DestroyVideoDecoder(VideoDecoder *decoder) {
delete decoder;
decoder = nullptr;
diff --git a/sdk/objc/Framework/Classes/VideoToolbox/objc_video_encoder_factory.h b/sdk/objc/Framework/Classes/VideoToolbox/objc_video_encoder_factory.h
index cb8dda6..6cdfc56 100644
--- a/sdk/objc/Framework/Classes/VideoToolbox/objc_video_encoder_factory.h
+++ b/sdk/objc/Framework/Classes/VideoToolbox/objc_video_encoder_factory.h
@@ -13,19 +13,29 @@
#import <Foundation/Foundation.h>
+#include "api/video_codecs/video_encoder_factory.h"
#include "media/engine/webrtcvideoencoderfactory.h"
@protocol RTCVideoEncoderFactory;
namespace webrtc {
-class ObjCVideoEncoderFactory : public cricket::WebRtcVideoEncoderFactory {
+// TODO(andersc): Remove the inheritance from cricket::WebRtcVideoEncoderFactory
+// when the legacy path in [RTCPeerConnectionFactory init] is no longer needed.
+class ObjCVideoEncoderFactory : public VideoEncoderFactory,
+ public cricket::WebRtcVideoEncoderFactory {
public:
explicit ObjCVideoEncoderFactory(id<RTCVideoEncoderFactory>);
~ObjCVideoEncoderFactory();
id<RTCVideoEncoderFactory> wrapped_encoder_factory() const;
+ std::vector<SdpVideoFormat> GetSupportedFormats() const override;
+ std::unique_ptr<VideoEncoder> CreateVideoEncoder(
+ const SdpVideoFormat& format) override;
+ CodecInfo QueryVideoEncoder(const SdpVideoFormat& format) const override;
+
+ // Needed for WebRtcVideoEncoderFactory interface.
webrtc::VideoEncoder* CreateVideoEncoder(
const cricket::VideoCodec& codec) override;
const std::vector<cricket::VideoCodec>& supported_codecs() const override;
@@ -33,6 +43,8 @@
private:
id<RTCVideoEncoderFactory> encoder_factory_;
+
+ // Needed for WebRtcVideoEncoderFactory interface.
mutable std::vector<cricket::VideoCodec> supported_codecs_;
};
diff --git a/sdk/objc/Framework/Classes/VideoToolbox/objc_video_encoder_factory.mm b/sdk/objc/Framework/Classes/VideoToolbox/objc_video_encoder_factory.mm
index 230719e..f78b74b 100644
--- a/sdk/objc/Framework/Classes/VideoToolbox/objc_video_encoder_factory.mm
+++ b/sdk/objc/Framework/Classes/VideoToolbox/objc_video_encoder_factory.mm
@@ -15,6 +15,8 @@
#import "NSString+StdString.h"
#import "RTCI420Buffer+Private.h"
#import "RTCVideoCodec+Private.h"
+#import "RTCVideoFrame+Private.h"
+#import "RTCWrappedNativeVideoEncoder.h"
#import "WebRTC/RTCVideoCodec.h"
#import "WebRTC/RTCVideoCodecFactory.h"
#import "WebRTC/RTCVideoCodecH264.h"
@@ -22,12 +24,12 @@
#import "WebRTC/RTCVideoFrameBuffer.h"
#include "api/video/video_frame.h"
+#include "api/video_codecs/sdp_video_format.h"
#include "api/video_codecs/video_encoder.h"
#include "modules/include/module_common_types.h"
#include "modules/video_coding/include/video_codec_interface.h"
#include "modules/video_coding/include/video_error_codes.h"
#include "rtc_base/logging.h"
-#include "rtc_base/timeutils.h"
#include "sdk/objc/Framework/Classes/Common/helpers.h"
#include "sdk/objc/Framework/Classes/Video/objc_frame_buffer.h"
@@ -35,21 +37,6 @@
namespace {
-id<RTCVideoFrameBuffer> nativeToRtcFrameBuffer(const rtc::scoped_refptr<VideoFrameBuffer> &buffer) {
- return buffer->type() == VideoFrameBuffer::Type::kNative ?
- static_cast<ObjCFrameBuffer *>(buffer.get())->wrapped_frame_buffer() :
- [[RTCI420Buffer alloc] initWithFrameBuffer:buffer->ToI420()];
-}
-
-RTCVideoFrame *nativeToRtcFrame(const VideoFrame &frame) {
- RTCVideoFrame *rtcFrame =
- [[RTCVideoFrame alloc] initWithBuffer:nativeToRtcFrameBuffer(frame.video_frame_buffer())
- rotation:RTCVideoRotation(frame.rotation())
- timeStampNs:frame.timestamp_us() * rtc::kNumNanosecsPerMicrosec];
- rtcFrame.timeStamp = frame.timestamp();
- return rtcFrame;
-}
-
class ObjCVideoEncoder : public VideoEncoder {
public:
ObjCVideoEncoder(id<RTCVideoEncoder> encoder)
@@ -69,7 +56,7 @@
RTCRtpFragmentationHeader *_Nonnull header) {
EncodedImage encodedImage = [frame nativeEncodedImage];
- // Handle types than can be converted into one of CodecSpecificInfo's hard coded cases.
+ // Handle types that can be converted into one of CodecSpecificInfo's hard coded cases.
CodecSpecificInfo codecSpecificInfo;
if ([info isKindOfClass:[RTCCodecSpecificInfoH264 class]]) {
codecSpecificInfo = [(RTCCodecSpecificInfoH264 *)info nativeCodecSpecificInfo];
@@ -106,7 +93,7 @@
[rtcFrameTypes addObject:@(RTCFrameType(frame_types->at(i)))];
}
- return [encoder_ encode:nativeToRtcFrame(frame)
+ return [encoder_ encode:[[RTCVideoFrame alloc] initWithNativeVideoFrame:frame]
codecSpecificInfo:rtcCodecSpecificInfo
frameTypes:rtcFrameTypes];
}
@@ -143,8 +130,44 @@
return encoder_factory_;
}
+std::vector<SdpVideoFormat> ObjCVideoEncoderFactory::GetSupportedFormats() const {
+ std::vector<SdpVideoFormat> supported_formats;
+ for (RTCVideoCodecInfo *supportedCodec in encoder_factory_.supportedCodecs) {
+ SdpVideoFormat format = [supportedCodec nativeSdpVideoFormat];
+ supported_formats.push_back(format);
+ }
+
+ return supported_formats;
+}
+
+VideoEncoderFactory::CodecInfo ObjCVideoEncoderFactory::QueryVideoEncoder(
+ const SdpVideoFormat &format) const {
+ // TODO(andersc): This is a hack until we figure out how this should be done properly.
+ NSString *formatName = [NSString stringForStdString:format.name];
+ NSSet *wrappedSoftwareFormats = [NSSet setWithObjects:@"VP8", @"VP9", nil];
+
+ CodecInfo codec_info;
+ codec_info.is_hardware_accelerated = ![wrappedSoftwareFormats containsObject:formatName];
+ codec_info.has_internal_source = false;
+ return codec_info;
+}
+
+std::unique_ptr<VideoEncoder> ObjCVideoEncoderFactory::CreateVideoEncoder(
+ const SdpVideoFormat &format) {
+ RTCVideoCodecInfo *info = [[RTCVideoCodecInfo alloc] initWithNativeSdpVideoFormat:format];
+ id<RTCVideoEncoder> encoder = [encoder_factory_ createEncoder:info];
+ if ([encoder isKindOfClass:[RTCWrappedNativeVideoEncoder class]]) {
+ return [(RTCWrappedNativeVideoEncoder *)encoder releaseWrappedEncoder];
+ } else {
+ return std::unique_ptr<ObjCVideoEncoder>(new ObjCVideoEncoder(encoder));
+ }
+}
+
+// WebRtcVideoEncoderFactory
+
VideoEncoder *ObjCVideoEncoderFactory::CreateVideoEncoder(const cricket::VideoCodec &codec) {
- RTCVideoCodecInfo *info = [[RTCVideoCodecInfo alloc] initWithNativeVideoCodec:codec];
+ RTCVideoCodecInfo *info = [[RTCVideoCodecInfo alloc]
+ initWithNativeSdpVideoFormat:SdpVideoFormat(codec.name, codec.params)];
id<RTCVideoEncoder> encoder = [encoder_factory_ createEncoder:info];
return new ObjCVideoEncoder(encoder);
}
@@ -152,8 +175,8 @@
const std::vector<cricket::VideoCodec> &ObjCVideoEncoderFactory::supported_codecs() const {
supported_codecs_.clear();
for (RTCVideoCodecInfo *supportedCodec in encoder_factory_.supportedCodecs) {
- cricket::VideoCodec codec = [supportedCodec nativeVideoCodec];
- supported_codecs_.push_back(codec);
+ SdpVideoFormat format = [supportedCodec nativeSdpVideoFormat];
+ supported_codecs_.push_back(cricket::VideoCodec(format));
}
return supported_codecs_;