Adding a Metal RGB renderer.
The new RTCMTLRGBRenderer dynamically handles both the kCVPixelFormatType_32BGRA
and the kCVPixelFormatType_32ARGB pixel formats.
Change-Id: I935532f762eff74c4b84fea9b855191f4c321fb7
Bug: webrtc:9200
Reviewed-on: https://webrtc-review.googlesource.com/72482
Reviewed-by: Kári Helgason <kthelgason@webrtc.org>
Reviewed-by: Anders Carlsson <andersc@webrtc.org>
Commit-Queue: Peter Hanspers <peterhanspers@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#23100}diff --git a/sdk/objc/Framework/Classes/Metal/RTCMTLI420Renderer.mm b/sdk/objc/Framework/Classes/Metal/RTCMTLI420Renderer.mm
index 027db05..ae55258 100644
--- a/sdk/objc/Framework/Classes/Metal/RTCMTLI420Renderer.mm
+++ b/sdk/objc/Framework/Classes/Metal/RTCMTLI420Renderer.mm
@@ -19,10 +19,10 @@
#import "RTCMTLRenderer+Private.h"
-#define MTL_STRINGIFY(s) @ #s
-
static NSString *const shaderSource = MTL_STRINGIFY(
- using namespace metal; typedef struct {
+ using namespace metal;
+
+ typedef struct {
packed_float2 position;
packed_float2 texcoord;
} Vertex;
diff --git a/sdk/objc/Framework/Classes/Metal/RTCMTLNV12Renderer.mm b/sdk/objc/Framework/Classes/Metal/RTCMTLNV12Renderer.mm
index 6de9c34..d8dd7e7 100644
--- a/sdk/objc/Framework/Classes/Metal/RTCMTLNV12Renderer.mm
+++ b/sdk/objc/Framework/Classes/Metal/RTCMTLNV12Renderer.mm
@@ -20,10 +20,10 @@
#import "RTCMTLRenderer+Private.h"
#include "rtc_base/checks.h"
-#define MTL_STRINGIFY(s) @ #s
-
static NSString *const shaderSource = MTL_STRINGIFY(
- using namespace metal; typedef struct {
+ using namespace metal;
+
+ typedef struct {
packed_float2 position;
packed_float2 texcoord;
} Vertex;
@@ -67,18 +67,20 @@
- (BOOL)addRenderingDestination:(__kindof MTKView *)view {
if ([super addRenderingDestination:view]) {
- [self initializeTextureCache];
- return YES;
+ return [self initializeTextureCache];
}
return NO;
}
-- (void)initializeTextureCache {
+- (BOOL)initializeTextureCache {
CVReturn status = CVMetalTextureCacheCreate(kCFAllocatorDefault, nil, [self currentMetalDevice],
nil, &_textureCache);
if (status != kCVReturnSuccess) {
RTCLogError(@"Metal: Failed to initialize metal texture cache. Return status is %d", status);
+ return NO;
}
+
+ return YES;
}
- (NSString *)shaderSource {
diff --git a/sdk/objc/Framework/Classes/Metal/RTCMTLRGBRenderer.h b/sdk/objc/Framework/Classes/Metal/RTCMTLRGBRenderer.h
new file mode 100644
index 0000000..de27055
--- /dev/null
+++ b/sdk/objc/Framework/Classes/Metal/RTCMTLRGBRenderer.h
@@ -0,0 +1,21 @@
+/*
+ * Copyright 2018 The WebRTC Project Authors. All rights reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#import <Foundation/Foundation.h>
+
+#import "RTCMTLRenderer.h"
+
+/** @abstract RGB/BGR renderer.
+ * @discussion This renderer handles both kCVPixelFormatType_32BGRA and kCVPixelFormatType_32ARGB.
+ */
+NS_AVAILABLE(10_11, 9_0)
+@interface RTCMTLRGBRenderer : RTCMTLRenderer
+
+@end
diff --git a/sdk/objc/Framework/Classes/Metal/RTCMTLRGBRenderer.mm b/sdk/objc/Framework/Classes/Metal/RTCMTLRGBRenderer.mm
new file mode 100644
index 0000000..e735879
--- /dev/null
+++ b/sdk/objc/Framework/Classes/Metal/RTCMTLRGBRenderer.mm
@@ -0,0 +1,145 @@
+/*
+ * Copyright 2018 The WebRTC Project Authors. All rights reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#import "RTCMTLRGBRenderer.h"
+
+#import <Metal/Metal.h>
+#import <MetalKit/MetalKit.h>
+
+#import "WebRTC/RTCLogging.h"
+#import "WebRTC/RTCVideoFrame.h"
+#import "WebRTC/RTCVideoFrameBuffer.h"
+
+#import "RTCMTLRenderer+Private.h"
+#include "rtc_base/checks.h"
+
+static NSString *const shaderSource = MTL_STRINGIFY(
+ using namespace metal;
+
+ typedef struct {
+ packed_float2 position;
+ packed_float2 texcoord;
+ } Vertex;
+
+ typedef struct {
+ float4 position[[position]];
+ float2 texcoord;
+ } VertexIO;
+
+ vertex VertexIO vertexPassthrough(device Vertex * verticies[[buffer(0)]],
+ uint vid[[vertex_id]]) {
+ VertexIO out;
+ device Vertex &v = verticies[vid];
+ out.position = float4(float2(v.position), 0.0, 1.0);
+ out.texcoord = v.texcoord;
+ return out;
+ }
+
+ fragment half4 fragmentColorConversion(
+ VertexIO in[[stage_in]], texture2d<half, access::sample> texture[[texture(0)]],
+ constant bool &isARGB[[buffer(0)]]) {
+ constexpr sampler s(address::clamp_to_edge, filter::linear);
+
+ half4 out = texture.sample(s, in.texcoord);
+ if (isARGB) {
+ out = half4(out.g, out.b, out.a, out.r);
+ }
+
+ return out;
+ });
+
+@implementation RTCMTLRGBRenderer {
+ // Textures.
+ CVMetalTextureCacheRef _textureCache;
+ id<MTLTexture> _texture;
+
+ // Uniforms.
+ id<MTLBuffer> _uniformsBuffer;
+}
+
+- (BOOL)addRenderingDestination:(__kindof MTKView *)view {
+ if ([super addRenderingDestination:view]) {
+ return [self initializeTextureCache];
+ }
+ return NO;
+}
+
+- (BOOL)initializeTextureCache {
+ CVReturn status = CVMetalTextureCacheCreate(kCFAllocatorDefault, nil, [self currentMetalDevice],
+ nil, &_textureCache);
+ if (status != kCVReturnSuccess) {
+ RTCLogError(@"Metal: Failed to initialize metal texture cache. Return status is %d", status);
+ return NO;
+ }
+
+ return YES;
+}
+
+- (NSString *)shaderSource {
+ return shaderSource;
+}
+
+- (BOOL)setupTexturesForFrame:(nonnull RTCVideoFrame *)frame {
+ RTC_DCHECK([frame.buffer isKindOfClass:[RTCCVPixelBuffer class]]);
+ [super setupTexturesForFrame:frame];
+ CVPixelBufferRef pixelBuffer = ((RTCCVPixelBuffer *)frame.buffer).pixelBuffer;
+
+ id<MTLTexture> gpuTexture = nil;
+ CVMetalTextureRef textureOut = nullptr;
+ bool isARGB;
+
+ int width = CVPixelBufferGetWidth(pixelBuffer);
+ int height = CVPixelBufferGetHeight(pixelBuffer);
+ OSType pixelFormat = CVPixelBufferGetPixelFormatType(pixelBuffer);
+
+ MTLPixelFormat mtlPixelFormat;
+ if (pixelFormat == kCVPixelFormatType_32BGRA) {
+ mtlPixelFormat = MTLPixelFormatBGRA8Unorm;
+ isARGB = false;
+ } else if (pixelFormat == kCVPixelFormatType_32ARGB) {
+ mtlPixelFormat = MTLPixelFormatRGBA8Unorm;
+ isARGB = true;
+ } else {
+ RTC_NOTREACHED();
+ return NO;
+ }
+
+ CVReturn result = CVMetalTextureCacheCreateTextureFromImage(
+ kCFAllocatorDefault, _textureCache, pixelBuffer, nil, mtlPixelFormat,
+ width, height, 0, &textureOut);
+ if (result == kCVReturnSuccess) {
+ gpuTexture = CVMetalTextureGetTexture(textureOut);
+ }
+ CVBufferRelease(textureOut);
+
+ if (gpuTexture != nil) {
+ _texture = gpuTexture;
+ _uniformsBuffer =
+ [[self currentMetalDevice] newBufferWithBytes:&isARGB
+ length:sizeof(isARGB)
+ options:MTLResourceCPUCacheModeDefaultCache];
+ return YES;
+ }
+
+ return NO;
+}
+
+- (void)uploadTexturesToRenderEncoder:(id<MTLRenderCommandEncoder>)renderEncoder {
+ [renderEncoder setFragmentTexture:_texture atIndex:0];
+ [renderEncoder setFragmentBuffer:_uniformsBuffer offset:0 atIndex:0];
+}
+
+- (void)dealloc {
+ if (_textureCache) {
+ CFRelease(_textureCache);
+ }
+}
+
+@end
diff --git a/sdk/objc/Framework/Classes/Metal/RTCMTLRenderer+Private.h b/sdk/objc/Framework/Classes/Metal/RTCMTLRenderer+Private.h
index dfeb87b..c9ea3cc 100644
--- a/sdk/objc/Framework/Classes/Metal/RTCMTLRenderer+Private.h
+++ b/sdk/objc/Framework/Classes/Metal/RTCMTLRenderer+Private.h
@@ -11,6 +11,8 @@
#import <Metal/Metal.h>
#import "RTCMTLRenderer.h"
+#define MTL_STRINGIFY(s) @ #s
+
NS_ASSUME_NONNULL_BEGIN
@interface RTCMTLRenderer (Private)
- (nullable id<MTLDevice>)currentMetalDevice;
diff --git a/sdk/objc/Framework/Classes/Metal/RTCMTLRenderer.mm b/sdk/objc/Framework/Classes/Metal/RTCMTLRenderer.mm
index 7fcba49..68486dc 100644
--- a/sdk/objc/Framework/Classes/Metal/RTCMTLRenderer.mm
+++ b/sdk/objc/Framework/Classes/Metal/RTCMTLRenderer.mm
@@ -139,9 +139,10 @@
// Load metal library from source.
NSError *libraryError = nil;
+ NSString *shaderSource = [self shaderSource];
id<MTLLibrary> sourceLibrary =
- [_device newLibraryWithSource:[self shaderSource] options:NULL error:&libraryError];
+ [_device newLibraryWithSource:shaderSource options:NULL error:&libraryError];
if (libraryError) {
RTCLogError(@"Metal: Library with source failed\n%@", libraryError);
diff --git a/sdk/objc/Framework/Classes/Metal/RTCMTLVideoView.m b/sdk/objc/Framework/Classes/Metal/RTCMTLVideoView.m
index 9ec2583..08594c8 100644
--- a/sdk/objc/Framework/Classes/Metal/RTCMTLVideoView.m
+++ b/sdk/objc/Framework/Classes/Metal/RTCMTLVideoView.m
@@ -19,16 +19,19 @@
#import "RTCMTLI420Renderer.h"
#import "RTCMTLNV12Renderer.h"
+#import "RTCMTLRGBRenderer.h"
// To avoid unreconized symbol linker errors, we're taking advantage of the objc runtime.
// Linking errors occur when compiling for architectures that don't support Metal.
#define MTKViewClass NSClassFromString(@"MTKView")
#define RTCMTLNV12RendererClass NSClassFromString(@"RTCMTLNV12Renderer")
#define RTCMTLI420RendererClass NSClassFromString(@"RTCMTLI420Renderer")
+#define RTCMTLRGBRendererClass NSClassFromString(@"RTCMTLRGBRenderer")
@interface RTCMTLVideoView () <MTKViewDelegate>
@property(nonatomic, strong) RTCMTLI420Renderer *rendererI420;
@property(nonatomic, strong) RTCMTLNV12Renderer *rendererNV12;
+@property(nonatomic, strong) RTCMTLRGBRenderer *rendererRGB;
@property(nonatomic, strong) MTKView *metalView;
@property(atomic, strong) RTCVideoFrame *videoFrame;
@end
@@ -41,6 +44,7 @@
@synthesize delegate = _delegate;
@synthesize rendererI420 = _rendererI420;
@synthesize rendererNV12 = _rendererNV12;
+@synthesize rendererRGB = _rendererRGB;
@synthesize metalView = _metalView;
@synthesize videoFrame = _videoFrame;
@@ -83,6 +87,10 @@
return [[RTCMTLI420RendererClass alloc] init];
}
++ (RTCMTLRGBRenderer *)createRGBRenderer {
+ return [[RTCMTLRGBRenderer alloc] init];
+}
+
- (void)configure {
NSAssert([RTCMTLVideoView isMetalAvailable], @"Metal not availiable on this device");
@@ -127,15 +135,29 @@
}
if ([videoFrame.buffer isKindOfClass:[RTCCVPixelBuffer class]]) {
- if (!self.rendererNV12) {
- self.rendererNV12 = [RTCMTLVideoView createNV12Renderer];
- if (![self.rendererNV12 addRenderingDestination:self.metalView]) {
- self.rendererNV12 = nil;
- RTCLogError(@"Failed to create NV12 renderer");
- return;
+ RTCCVPixelBuffer *buffer = (RTCCVPixelBuffer*)videoFrame.buffer;
+ const OSType pixelFormat = CVPixelBufferGetPixelFormatType(buffer.pixelBuffer);
+ if (pixelFormat == kCVPixelFormatType_32BGRA || pixelFormat == kCVPixelFormatType_32ARGB) {
+ if (!self.rendererRGB) {
+ self.rendererRGB = [RTCMTLVideoView createRGBRenderer];
+ if (![self.rendererRGB addRenderingDestination:self.metalView]) {
+ self.rendererRGB = nil;
+ RTCLogError(@"Failed to create RGB renderer");
+ return;
+ }
}
+ [self.rendererRGB drawFrame:videoFrame];
+ } else {
+ if (!self.rendererNV12) {
+ self.rendererNV12 = [RTCMTLVideoView createNV12Renderer];
+ if (![self.rendererNV12 addRenderingDestination:self.metalView]) {
+ self.rendererNV12 = nil;
+ RTCLogError(@"Failed to create NV12 renderer");
+ return;
+ }
+ }
+ [self.rendererNV12 drawFrame:videoFrame];
}
- [self.rendererNV12 drawFrame:videoFrame];
} else {
if (!self.rendererI420) {
self.rendererI420 = [RTCMTLVideoView createI420Renderer];