Support cropping and rotation override in Metal renderers.
Bug: webrtc:9301
Change-Id: Ic761f0fd6ad6fee74021b84903f1653878453533
Reviewed-on: https://webrtc-review.googlesource.com/80460
Reviewed-by: Anders Carlsson <andersc@webrtc.org>
Commit-Queue: Peter Hanspers <peterhanspers@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#23522}diff --git a/sdk/objc/Framework/Classes/Metal/RTCMTLVideoView.m b/sdk/objc/Framework/Classes/Metal/RTCMTLVideoView.m
index 08594c8..66194db 100644
--- a/sdk/objc/Framework/Classes/Metal/RTCMTLVideoView.m
+++ b/sdk/objc/Framework/Classes/Metal/RTCMTLVideoView.m
@@ -29,17 +29,16 @@
#define RTCMTLRGBRendererClass NSClassFromString(@"RTCMTLRGBRenderer")
@interface RTCMTLVideoView () <MTKViewDelegate>
-@property(nonatomic, strong) RTCMTLI420Renderer *rendererI420;
-@property(nonatomic, strong) RTCMTLNV12Renderer *rendererNV12;
-@property(nonatomic, strong) RTCMTLRGBRenderer *rendererRGB;
-@property(nonatomic, strong) MTKView *metalView;
-@property(atomic, strong) RTCVideoFrame *videoFrame;
+@property(nonatomic) RTCMTLI420Renderer *rendererI420;
+@property(nonatomic) RTCMTLNV12Renderer *rendererNV12;
+@property(nonatomic) RTCMTLRGBRenderer *rendererRGB;
+@property(nonatomic) MTKView *metalView;
+@property(atomic) RTCVideoFrame *videoFrame;
+@property(nonatomic) CGSize videoFrameSize;
+@property(nonatomic) int64_t lastFrameTimeNs;
@end
-@implementation RTCMTLVideoView {
- int64_t _lastFrameTimeNs;
- CGSize _videoFrameSize;
-}
+@implementation RTCMTLVideoView
@synthesize delegate = _delegate;
@synthesize rendererI420 = _rendererI420;
@@ -47,6 +46,9 @@
@synthesize rendererRGB = _rendererRGB;
@synthesize metalView = _metalView;
@synthesize videoFrame = _videoFrame;
+@synthesize videoFrameSize = _videoFrameSize;
+@synthesize lastFrameTimeNs = _lastFrameTimeNs;
+@synthesize rotationOverride = _rotationOverride;
- (instancetype)initWithFrame:(CGRect)frameRect {
self = [super initWithFrame:frameRect];
@@ -64,6 +66,22 @@
return self;
}
+- (BOOL)isEnabled {
+ return !self.metalView.paused;
+}
+
+- (void)setEnabled:(BOOL)enabled {
+ self.metalView.paused = !enabled;
+}
+
+- (UIViewContentMode)videoContentMode {
+ return self.metalView.contentMode;
+}
+
+- (void)setVideoContentMode:(UIViewContentMode)mode {
+ self.metalView.contentMode = mode;
+}
+
#pragma mark - Private
+ (BOOL)isMetalAvailable {
@@ -74,11 +92,6 @@
#endif
}
-+ (MTKView *)createMetalView:(CGRect)frame {
- MTKView *view = [[MTKViewClass alloc] initWithFrame:frame];
- return view;
-}
-
+ (RTCMTLNV12Renderer *)createNV12Renderer {
return [[RTCMTLNV12RendererClass alloc] init];
}
@@ -94,33 +107,22 @@
- (void)configure {
NSAssert([RTCMTLVideoView isMetalAvailable], @"Metal not availiable on this device");
- _metalView = [RTCMTLVideoView createMetalView:self.bounds];
- [self configureMetalView];
+ self.metalView = [[MTKViewClass alloc] initWithFrame:self.bounds];
+ self.metalView.delegate = self;
+ self.metalView.contentMode = UIViewContentModeScaleAspectFill;
+ [self addSubview:self.metalView];
+ self.videoFrameSize = CGSizeZero;
}
-- (void)configureMetalView {
- if (_metalView) {
- _metalView.delegate = self;
- [self addSubview:_metalView];
- _metalView.contentMode = UIViewContentModeScaleAspectFit;
- _videoFrameSize = CGSizeZero;
- }
-}
-
-- (void)setVideoContentMode:(UIViewContentMode)mode {
- _metalView.contentMode = mode;
-}
-
-#pragma mark - Private
-
- (void)layoutSubviews {
[super layoutSubviews];
+
CGRect bounds = self.bounds;
- _metalView.frame = bounds;
- if (!CGSizeEqualToSize(_videoFrameSize, CGSizeZero)) {
- _metalView.drawableSize = _videoFrameSize;
+ self.metalView.frame = bounds;
+ if (!CGSizeEqualToSize(self.videoFrameSize, CGSizeZero)) {
+ self.metalView.drawableSize = [self drawableSize];
} else {
- _metalView.drawableSize = bounds.size;
+ self.metalView.drawableSize = bounds.size;
}
}
@@ -130,10 +132,11 @@
NSAssert(view == self.metalView, @"Receiving draw callbacks from foreign instance.");
RTCVideoFrame *videoFrame = self.videoFrame;
// Skip rendering if we've already rendered this frame.
- if (!videoFrame || videoFrame.timeStampNs == _lastFrameTimeNs) {
+ if (!videoFrame || videoFrame.timeStampNs == self.lastFrameTimeNs) {
return;
}
+ RTCMTLRenderer *renderer;
if ([videoFrame.buffer isKindOfClass:[RTCCVPixelBuffer class]]) {
RTCCVPixelBuffer *buffer = (RTCCVPixelBuffer*)videoFrame.buffer;
const OSType pixelFormat = CVPixelBufferGetPixelFormatType(buffer.pixelBuffer);
@@ -146,7 +149,7 @@
return;
}
}
- [self.rendererRGB drawFrame:videoFrame];
+ renderer = self.rendererRGB;
} else {
if (!self.rendererNV12) {
self.rendererNV12 = [RTCMTLVideoView createNV12Renderer];
@@ -156,7 +159,7 @@
return;
}
}
- [self.rendererNV12 drawFrame:videoFrame];
+ renderer = self.rendererNV12;
}
} else {
if (!self.rendererI420) {
@@ -167,25 +170,73 @@
return;
}
}
- [self.rendererI420 drawFrame:videoFrame];
+ renderer = self.rendererI420;
}
- _lastFrameTimeNs = videoFrame.timeStampNs;
+
+ renderer.rotationOverride = self.rotationOverride;
+
+ [renderer drawFrame:videoFrame];
+ self.lastFrameTimeNs = videoFrame.timeStampNs;
}
- (void)mtkView:(MTKView *)view drawableSizeWillChange:(CGSize)size {
}
+- (RTCVideoRotation)frameRotation {
+ if (self.rotationOverride) {
+ RTCVideoRotation rotation;
+#if defined(__IPHONE_11_0) && (__IPHONE_OS_VERSION_MAX_ALLOWED >= __IPHONE_11_0)
+ if (@available(iOS 11, *)) {
+ [self.rotationOverride getValue:&rotation size:sizeof(rotation)];
+ } else
+#endif
+ {
+ [self.rotationOverride getValue:&rotation];
+ }
+ return rotation;
+ }
+
+ return self.videoFrame.rotation;
+}
+
+- (CGSize)drawableSize {
+ // Flip width/height if the rotations are not the same.
+ CGSize videoFrameSize = self.videoFrameSize;
+ RTCVideoRotation frameRotation = [self frameRotation];
+
+ BOOL useLandscape =
+ (frameRotation == RTCVideoRotation_0) || (frameRotation == RTCVideoRotation_180);
+ BOOL sizeIsLandscape = (self.videoFrame.rotation == RTCVideoRotation_0) ||
+ (self.videoFrame.rotation == RTCVideoRotation_180);
+
+ if (useLandscape == sizeIsLandscape) {
+ return videoFrameSize;
+ } else {
+ return CGSizeMake(videoFrameSize.height, videoFrameSize.width);
+ }
+}
+
#pragma mark - RTCVideoRenderer
- (void)setSize:(CGSize)size {
- self.metalView.drawableSize = size;
+ __weak RTCMTLVideoView *weakSelf = self;
dispatch_async(dispatch_get_main_queue(), ^{
- _videoFrameSize = size;
- [self.delegate videoView:self didChangeVideoSize:size];
+ RTCMTLVideoView *strongSelf = weakSelf;
+
+ strongSelf.videoFrameSize = size;
+ CGSize drawableSize = [strongSelf drawableSize];
+
+ strongSelf.metalView.drawableSize = drawableSize;
+ [strongSelf setNeedsLayout];
+ [strongSelf.delegate videoView:self didChangeVideoSize:size];
});
}
- (void)renderFrame:(nullable RTCVideoFrame *)frame {
+ if (!self.isEnabled) {
+ return;
+ }
+
if (frame == nil) {
RTCLogInfo(@"Incoming frame is nil. Exiting render callback.");
return;