iOS: Add support for rendering native CVPixelBuffers directly
This CL adds support in RTCEAGLVideoView for rendering CVPixelBuffers as
OpenGL ES textures directly, compared to the current code that first
converts the CVPixelBuffers to I420, and then reuploads them as
textures. This is only supported on iOS with the use of a
CVOpenGLESTextureCache.
The I420 rendering and native rendering are separated in two different
implementations of a simple shader interface:
@protocol Shader
- (BOOL)drawFrame:(RTCVideoFrame*)frame;
@end
GL resources are allocated when the shader is instantiated and released
when the shader is destroyed. RTCEAGLVideoView will lazily instantiate
the necessary shader when it receives the first frame of that kind. This
is primarily done to avoid allocating GL resources for both I420 and
native rendering.
Some other changes are:
- Print GL shader compilation errors.
- Remove updateTextureSizesForFrame() function. The textures will
resize automatically anyway when the texture data is uploaded with
glTexImage2D().
patch from issue 2154243002 at patchset 140001 (http://crrev.com/2154243002#ps140001)
Continuing magjed@'s work since he is OOO this week.
BUG=
Review-Url: https://codereview.webrtc.org/2202823004
Cr-Commit-Position: refs/heads/master@{#13668}
diff --git a/webrtc/sdk/objc/Framework/Classes/RTCI420Shader.mm b/webrtc/sdk/objc/Framework/Classes/RTCI420Shader.mm
new file mode 100644
index 0000000..0618c1a
--- /dev/null
+++ b/webrtc/sdk/objc/Framework/Classes/RTCI420Shader.mm
@@ -0,0 +1,211 @@
+/*
+ * Copyright 2016 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#import "RTCShader.h"
+
+#include <vector>
+
+#import "RTCShader+Private.h"
+#import "WebRTC/RTCVideoFrame.h"
+
+// |kNumTextures| must not exceed 8, which is the limit in OpenGLES2. Two sets
+// of 3 textures are used here, one for each of the Y, U and V planes. Having
+// two sets alleviates CPU blockage in the event that the GPU is asked to render
+// to a texture that is already in use.
+static const GLsizei kNumTextureSets = 2;
+static const GLsizei kNumTexturesPerSet = 3;
+static const GLsizei kNumTextures = kNumTexturesPerSet * kNumTextureSets;
+
+// Fragment shader converts YUV values from input textures into a final RGB
+// pixel. The conversion formula is from http://www.fourcc.org/fccyvrgb.php.
+static const char kI420FragmentShaderSource[] =
+ SHADER_VERSION
+ "precision highp float;"
+ FRAGMENT_SHADER_IN " vec2 v_texcoord;\n"
+ "uniform lowp sampler2D s_textureY;\n"
+ "uniform lowp sampler2D s_textureU;\n"
+ "uniform lowp sampler2D s_textureV;\n"
+ FRAGMENT_SHADER_OUT
+ "void main() {\n"
+ " float y, u, v, r, g, b;\n"
+ " y = " FRAGMENT_SHADER_TEXTURE "(s_textureY, v_texcoord).r;\n"
+ " u = " FRAGMENT_SHADER_TEXTURE "(s_textureU, v_texcoord).r;\n"
+ " v = " FRAGMENT_SHADER_TEXTURE "(s_textureV, v_texcoord).r;\n"
+ " u = u - 0.5;\n"
+ " v = v - 0.5;\n"
+ " r = y + 1.403 * v;\n"
+ " g = y - 0.344 * u - 0.714 * v;\n"
+ " b = y + 1.770 * u;\n"
+ " " FRAGMENT_SHADER_COLOR " = vec4(r, g, b, 1.0);\n"
+ " }\n";
+
+@implementation RTCI420Shader {
+ BOOL _hasUnpackRowLength;
+ GLint _currentTextureSet;
+ // Handles for OpenGL constructs.
+ GLuint _textures[kNumTextures];
+ GLuint _i420Program;
+ GLuint _vertexArray;
+ GLuint _vertexBuffer;
+ GLint _ySampler;
+ GLint _uSampler;
+ GLint _vSampler;
+ // Used to create a non-padded plane for GPU upload when we receive padded
+ // frames.
+ std::vector<uint8_t> _planeBuffer;
+}
+
+- (instancetype)initWithContext:(GlContextType *)context {
+ if (self = [super init]) {
+#if TARGET_OS_IPHONE
+ _hasUnpackRowLength = (context.API == kEAGLRenderingAPIOpenGLES3);
+#else
+ _hasUnpackRowLength = YES;
+#endif
+ glPixelStorei(GL_UNPACK_ALIGNMENT, 1);
+ if (![self setupI420Program] || ![self setupTextures] ||
+ !RTCSetupVerticesForProgram(_i420Program, &_vertexBuffer, &_vertexArray)) {
+ self = nil;
+ }
+ }
+ return self;
+}
+
+- (void)dealloc {
+ glDeleteProgram(_i420Program);
+ glDeleteTextures(kNumTextures, _textures);
+ glDeleteBuffers(1, &_vertexBuffer);
+ glDeleteVertexArrays(1, &_vertexArray);
+}
+
+- (BOOL)setupI420Program {
+ _i420Program = RTCCreateProgramFromFragmentSource(kI420FragmentShaderSource);
+ if (!_i420Program) {
+ return NO;
+ }
+ _ySampler = glGetUniformLocation(_i420Program, "s_textureY");
+ _uSampler = glGetUniformLocation(_i420Program, "s_textureU");
+ _vSampler = glGetUniformLocation(_i420Program, "s_textureV");
+
+ return (_ySampler >= 0 && _uSampler >= 0 && _vSampler >= 0);
+}
+
+- (BOOL)setupTextures {
+ glGenTextures(kNumTextures, _textures);
+ // Set parameters for each of the textures we created.
+ for (GLsizei i = 0; i < kNumTextures; i++) {
+ glBindTexture(GL_TEXTURE_2D, _textures[i]);
+ glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
+ glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
+ glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
+ glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
+ }
+ return YES;
+}
+
+- (BOOL)drawFrame:(RTCVideoFrame*)frame {
+ glUseProgram(_i420Program);
+ if (![self updateTextureDataForFrame:frame]) {
+ return NO;
+ }
+#if !TARGET_OS_IPHONE
+ glBindVertexArray(_vertexArray);
+#endif
+ glBindBuffer(GL_ARRAY_BUFFER, _vertexBuffer);
+ glDrawArrays(GL_TRIANGLE_FAN, 0, 4);
+
+ return YES;
+}
+
+- (void)uploadPlane:(const uint8_t *)plane
+ sampler:(GLint)sampler
+ offset:(GLint)offset
+ width:(size_t)width
+ height:(size_t)height
+ stride:(int32_t)stride {
+ glActiveTexture(static_cast<GLenum>(GL_TEXTURE0 + offset));
+ glBindTexture(GL_TEXTURE_2D, _textures[offset]);
+
+ // When setting texture sampler uniforms, the texture index is used not
+ // the texture handle.
+ glUniform1i(sampler, offset);
+ const uint8_t *uploadPlane = plane;
+ if ((size_t)stride != width) {
+ if (_hasUnpackRowLength) {
+ // GLES3 allows us to specify stride.
+ glPixelStorei(GL_UNPACK_ROW_LENGTH, stride);
+ glTexImage2D(GL_TEXTURE_2D,
+ 0,
+ RTC_PIXEL_FORMAT,
+ static_cast<GLsizei>(width),
+ static_cast<GLsizei>(height),
+ 0,
+ RTC_PIXEL_FORMAT,
+ GL_UNSIGNED_BYTE,
+ uploadPlane);
+ glPixelStorei(GL_UNPACK_ROW_LENGTH, 0);
+ return;
+ } else {
+ // Make an unpadded copy and upload that instead. Quick profiling showed
+ // that this is faster than uploading row by row using glTexSubImage2D.
+ uint8_t *unpaddedPlane = _planeBuffer.data();
+ for (size_t y = 0; y < height; ++y) {
+ memcpy(unpaddedPlane + y * width, plane + y * stride, width);
+ }
+ uploadPlane = unpaddedPlane;
+ }
+ }
+ glTexImage2D(GL_TEXTURE_2D,
+ 0,
+ RTC_PIXEL_FORMAT,
+ static_cast<GLsizei>(width),
+ static_cast<GLsizei>(height),
+ 0,
+ RTC_PIXEL_FORMAT,
+ GL_UNSIGNED_BYTE,
+ uploadPlane);
+}
+
+- (BOOL)updateTextureDataForFrame:(RTCVideoFrame *)frame {
+ GLint textureOffset = _currentTextureSet * 3;
+ NSAssert(textureOffset + 3 <= kNumTextures, @"invalid offset");
+
+ if (frame.yPitch != static_cast<int32_t>(frame.width) ||
+ frame.uPitch != static_cast<int32_t>(frame.chromaWidth) ||
+ frame.vPitch != static_cast<int32_t>(frame.chromaWidth)) {
+ _planeBuffer.resize(frame.width * frame.height);
+ }
+
+ [self uploadPlane:frame.yPlane
+ sampler:_ySampler
+ offset:textureOffset
+ width:frame.width
+ height:frame.height
+ stride:frame.yPitch];
+
+ [self uploadPlane:frame.uPlane
+ sampler:_uSampler
+ offset:textureOffset + 1
+ width:frame.chromaWidth
+ height:frame.chromaHeight
+ stride:frame.uPitch];
+
+ [self uploadPlane:frame.vPlane
+ sampler:_vSampler
+ offset:textureOffset + 2
+ width:frame.chromaWidth
+ height:frame.chromaHeight
+ stride:frame.vPitch];
+
+ _currentTextureSet = (_currentTextureSet + 1) % kNumTextureSets;
+ return YES;
+}
+
+@end