henrike@webrtc.org | 28e2075 | 2013-07-10 00:45:36 +0000 | [diff] [blame] | 1 | /* |
kjellander | b24317b | 2016-02-10 07:54:43 -0800 | [diff] [blame] | 2 | * Copyright 2013 The WebRTC project authors. All Rights Reserved. |
henrike@webrtc.org | 28e2075 | 2013-07-10 00:45:36 +0000 | [diff] [blame] | 3 | * |
kjellander | b24317b | 2016-02-10 07:54:43 -0800 | [diff] [blame] | 4 | * Use of this source code is governed by a BSD-style license |
| 5 | * that can be found in the LICENSE file in the root of the source |
| 6 | * tree. An additional intellectual property rights grant can be found |
| 7 | * in the file PATENTS. All contributing project authors may |
| 8 | * be found in the AUTHORS file in the root of the source tree. |
henrike@webrtc.org | 28e2075 | 2013-07-10 00:45:36 +0000 | [diff] [blame] | 9 | */ |
| 10 | |
| 11 | package org.webrtc; |
| 12 | |
| 13 | import java.nio.ByteBuffer; |
Magnus Jedvert | 9060eb1 | 2017-12-12 12:52:54 +0100 | [diff] [blame] | 14 | import org.webrtc.VideoFrame; |
henrike@webrtc.org | 28e2075 | 2013-07-10 00:45:36 +0000 | [diff] [blame] | 15 | |
| 16 | /** |
Niels Möller | 8f59762 | 2016-03-23 10:33:07 +0100 | [diff] [blame] | 17 | * Java version of VideoSinkInterface. In addition to allowing clients to |
henrike@webrtc.org | 28e2075 | 2013-07-10 00:45:36 +0000 | [diff] [blame] | 18 | * define their own rendering behavior (by passing in a Callbacks object), this |
| 19 | * class also provides a createGui() method for creating a GUI-rendering window |
| 20 | * on various platforms. |
| 21 | */ |
| 22 | public class VideoRenderer { |
Magnus Jedvert | e76fb36 | 2015-10-08 19:05:24 +0200 | [diff] [blame] | 23 | /** |
nisse | acd935b | 2016-11-11 03:55:13 -0800 | [diff] [blame] | 24 | * Java version of webrtc::VideoFrame. Frames are only constructed from native code and test |
Magnus Jedvert | e76fb36 | 2015-10-08 19:05:24 +0200 | [diff] [blame] | 25 | * code. |
| 26 | */ |
henrike@webrtc.org | 28e2075 | 2013-07-10 00:45:36 +0000 | [diff] [blame] | 27 | public static class I420Frame { |
| 28 | public final int width; |
| 29 | public final int height; |
| 30 | public final int[] yuvStrides; |
Magnus Jedvert | a6cba3a | 2015-08-29 15:57:43 +0200 | [diff] [blame] | 31 | public ByteBuffer[] yuvPlanes; |
glaznev@webrtc.org | 9967845 | 2014-09-15 17:52:42 +0000 | [diff] [blame] | 32 | public final boolean yuvFrame; |
Per | 488e75f | 2015-11-19 10:43:36 +0100 | [diff] [blame] | 33 | // Matrix that transforms standard coordinates to their proper sampling locations in |
| 34 | // the texture. This transform compensates for any properties of the video source that |
| 35 | // cause it to appear different from a normalized texture. This matrix does not take |
| 36 | // |rotationDegree| into account. |
| 37 | public final float[] samplingMatrix; |
glaznev@webrtc.org | 9967845 | 2014-09-15 17:52:42 +0000 | [diff] [blame] | 38 | public int textureId; |
Magnus Jedvert | 7afc12f | 2015-09-03 12:40:38 +0200 | [diff] [blame] | 39 | // Frame pointer in C++. |
Magnus Jedvert | a6cba3a | 2015-08-29 15:57:43 +0200 | [diff] [blame] | 40 | private long nativeFramePointer; |
henrike@webrtc.org | 28e2075 | 2013-07-10 00:45:36 +0000 | [diff] [blame] | 41 | |
guoweis@webrtc.org | 840da7b | 2015-03-18 16:58:13 +0000 | [diff] [blame] | 42 | // rotationDegree is the degree that the frame must be rotated clockwisely |
| 43 | // to be rendered correctly. |
| 44 | public int rotationDegree; |
| 45 | |
Sami Kalliomäki | 7d1f493 | 2017-09-14 12:31:53 +0200 | [diff] [blame] | 46 | // If this I420Frame was constructed from VideoFrame.Buffer, this points to |
| 47 | // the backing buffer. |
| 48 | private final VideoFrame.Buffer backingBuffer; |
| 49 | |
henrike@webrtc.org | 28e2075 | 2013-07-10 00:45:36 +0000 | [diff] [blame] | 50 | /** |
Magnus Jedvert | 7afc12f | 2015-09-03 12:40:38 +0200 | [diff] [blame] | 51 | * Construct a frame of the given dimensions with the specified planar data. |
henrike@webrtc.org | 28e2075 | 2013-07-10 00:45:36 +0000 | [diff] [blame] | 52 | */ |
skvlad | ed6e077 | 2016-11-30 14:40:18 -0800 | [diff] [blame] | 53 | public I420Frame(int width, int height, int rotationDegree, int[] yuvStrides, |
| 54 | ByteBuffer[] yuvPlanes, long nativeFramePointer) { |
henrike@webrtc.org | 28e2075 | 2013-07-10 00:45:36 +0000 | [diff] [blame] | 55 | this.width = width; |
| 56 | this.height = height; |
| 57 | this.yuvStrides = yuvStrides; |
henrike@webrtc.org | 28e2075 | 2013-07-10 00:45:36 +0000 | [diff] [blame] | 58 | this.yuvPlanes = yuvPlanes; |
glaznev@webrtc.org | 9967845 | 2014-09-15 17:52:42 +0000 | [diff] [blame] | 59 | this.yuvFrame = true; |
guoweis@webrtc.org | 840da7b | 2015-03-18 16:58:13 +0000 | [diff] [blame] | 60 | this.rotationDegree = rotationDegree; |
Magnus Jedvert | a6cba3a | 2015-08-29 15:57:43 +0200 | [diff] [blame] | 61 | this.nativeFramePointer = nativeFramePointer; |
Sami Kalliomäki | 7d1f493 | 2017-09-14 12:31:53 +0200 | [diff] [blame] | 62 | backingBuffer = null; |
Magnus Jedvert | 80cf97c | 2015-06-11 10:08:59 +0200 | [diff] [blame] | 63 | if (rotationDegree % 90 != 0) { |
| 64 | throw new IllegalArgumentException("Rotation degree not multiple of 90: " + rotationDegree); |
| 65 | } |
Per | 488e75f | 2015-11-19 10:43:36 +0100 | [diff] [blame] | 66 | // The convention in WebRTC is that the first element in a ByteBuffer corresponds to the |
| 67 | // top-left corner of the image, but in glTexImage2D() the first element corresponds to the |
| 68 | // bottom-left corner. This discrepancy is corrected by setting a vertical flip as sampling |
| 69 | // matrix. |
Sami Kalliomäki | 372e587 | 2017-06-27 17:00:21 +0200 | [diff] [blame] | 70 | samplingMatrix = RendererCommon.verticalFlipMatrix(); |
glaznev@webrtc.org | 9967845 | 2014-09-15 17:52:42 +0000 | [diff] [blame] | 71 | } |
| 72 | |
| 73 | /** |
| 74 | * Construct a texture frame of the given dimensions with data in SurfaceTexture |
| 75 | */ |
skvlad | ed6e077 | 2016-11-30 14:40:18 -0800 | [diff] [blame] | 76 | public I420Frame(int width, int height, int rotationDegree, int textureId, |
| 77 | float[] samplingMatrix, long nativeFramePointer) { |
glaznev@webrtc.org | 9967845 | 2014-09-15 17:52:42 +0000 | [diff] [blame] | 78 | this.width = width; |
| 79 | this.height = height; |
| 80 | this.yuvStrides = null; |
| 81 | this.yuvPlanes = null; |
Per | 488e75f | 2015-11-19 10:43:36 +0100 | [diff] [blame] | 82 | this.samplingMatrix = samplingMatrix; |
glaznev@webrtc.org | 9967845 | 2014-09-15 17:52:42 +0000 | [diff] [blame] | 83 | this.textureId = textureId; |
| 84 | this.yuvFrame = false; |
guoweis@webrtc.org | 840da7b | 2015-03-18 16:58:13 +0000 | [diff] [blame] | 85 | this.rotationDegree = rotationDegree; |
Magnus Jedvert | a6cba3a | 2015-08-29 15:57:43 +0200 | [diff] [blame] | 86 | this.nativeFramePointer = nativeFramePointer; |
Sami Kalliomäki | 7d1f493 | 2017-09-14 12:31:53 +0200 | [diff] [blame] | 87 | backingBuffer = null; |
Magnus Jedvert | 80cf97c | 2015-06-11 10:08:59 +0200 | [diff] [blame] | 88 | if (rotationDegree % 90 != 0) { |
| 89 | throw new IllegalArgumentException("Rotation degree not multiple of 90: " + rotationDegree); |
| 90 | } |
| 91 | } |
| 92 | |
Sami Kalliomäki | bc061b4 | 2017-06-16 10:08:23 +0200 | [diff] [blame] | 93 | /** |
sakal | 836f60c | 2017-07-28 07:12:23 -0700 | [diff] [blame] | 94 | * Construct a frame from VideoFrame.Buffer. |
Sami Kalliomäki | bc061b4 | 2017-06-16 10:08:23 +0200 | [diff] [blame] | 95 | */ |
Magnus Jedvert | 9060eb1 | 2017-12-12 12:52:54 +0100 | [diff] [blame] | 96 | @CalledByNative("I420Frame") |
sakal | 836f60c | 2017-07-28 07:12:23 -0700 | [diff] [blame] | 97 | public I420Frame(int rotationDegree, VideoFrame.Buffer buffer, long nativeFramePointer) { |
| 98 | this.width = buffer.getWidth(); |
| 99 | this.height = buffer.getHeight(); |
Sami Kalliomäki | bc061b4 | 2017-06-16 10:08:23 +0200 | [diff] [blame] | 100 | this.rotationDegree = rotationDegree; |
| 101 | if (rotationDegree % 90 != 0) { |
| 102 | throw new IllegalArgumentException("Rotation degree not multiple of 90: " + rotationDegree); |
| 103 | } |
magjed | 2ab9879 | 2017-09-13 05:20:45 -0700 | [diff] [blame] | 104 | if (buffer instanceof VideoFrame.TextureBuffer |
| 105 | && ((VideoFrame.TextureBuffer) buffer).getType() == VideoFrame.TextureBuffer.Type.OES) { |
Sami Kalliomäki | bc061b4 | 2017-06-16 10:08:23 +0200 | [diff] [blame] | 106 | VideoFrame.TextureBuffer textureBuffer = (VideoFrame.TextureBuffer) buffer; |
| 107 | this.yuvFrame = false; |
| 108 | this.textureId = textureBuffer.getTextureId(); |
sakal | 836f60c | 2017-07-28 07:12:23 -0700 | [diff] [blame] | 109 | this.samplingMatrix = RendererCommon.convertMatrixFromAndroidGraphicsMatrix( |
| 110 | textureBuffer.getTransformMatrix()); |
Sami Kalliomäki | bc061b4 | 2017-06-16 10:08:23 +0200 | [diff] [blame] | 111 | |
| 112 | this.yuvStrides = null; |
| 113 | this.yuvPlanes = null; |
Sami Kalliomäki | 7d1f493 | 2017-09-14 12:31:53 +0200 | [diff] [blame] | 114 | } else if (buffer instanceof VideoFrame.I420Buffer) { |
| 115 | VideoFrame.I420Buffer i420Buffer = (VideoFrame.I420Buffer) buffer; |
Sami Kalliomäki | bc061b4 | 2017-06-16 10:08:23 +0200 | [diff] [blame] | 116 | this.yuvFrame = true; |
| 117 | this.yuvStrides = |
| 118 | new int[] {i420Buffer.getStrideY(), i420Buffer.getStrideU(), i420Buffer.getStrideV()}; |
| 119 | this.yuvPlanes = |
| 120 | new ByteBuffer[] {i420Buffer.getDataY(), i420Buffer.getDataU(), i420Buffer.getDataV()}; |
Sami Kalliomäki | 372e587 | 2017-06-27 17:00:21 +0200 | [diff] [blame] | 121 | // The convention in WebRTC is that the first element in a ByteBuffer corresponds to the |
| 122 | // top-left corner of the image, but in glTexImage2D() the first element corresponds to the |
| 123 | // bottom-left corner. This discrepancy is corrected by multiplying the sampling matrix with |
| 124 | // a vertical flip matrix. |
sakal | 836f60c | 2017-07-28 07:12:23 -0700 | [diff] [blame] | 125 | this.samplingMatrix = RendererCommon.verticalFlipMatrix(); |
Sami Kalliomäki | bc061b4 | 2017-06-16 10:08:23 +0200 | [diff] [blame] | 126 | |
| 127 | this.textureId = 0; |
Sami Kalliomäki | 7d1f493 | 2017-09-14 12:31:53 +0200 | [diff] [blame] | 128 | } else { |
| 129 | this.yuvFrame = false; |
| 130 | this.textureId = 0; |
| 131 | this.samplingMatrix = null; |
| 132 | this.yuvStrides = null; |
| 133 | this.yuvPlanes = null; |
Sami Kalliomäki | bc061b4 | 2017-06-16 10:08:23 +0200 | [diff] [blame] | 134 | } |
| 135 | this.nativeFramePointer = nativeFramePointer; |
Sami Kalliomäki | 7d1f493 | 2017-09-14 12:31:53 +0200 | [diff] [blame] | 136 | backingBuffer = buffer; |
Sami Kalliomäki | bc061b4 | 2017-06-16 10:08:23 +0200 | [diff] [blame] | 137 | } |
| 138 | |
Magnus Jedvert | 80cf97c | 2015-06-11 10:08:59 +0200 | [diff] [blame] | 139 | public int rotatedWidth() { |
| 140 | return (rotationDegree % 180 == 0) ? width : height; |
| 141 | } |
| 142 | |
| 143 | public int rotatedHeight() { |
| 144 | return (rotationDegree % 180 == 0) ? height : width; |
henrike@webrtc.org | 28e2075 | 2013-07-10 00:45:36 +0000 | [diff] [blame] | 145 | } |
| 146 | |
henrike@webrtc.org | 28e2075 | 2013-07-10 00:45:36 +0000 | [diff] [blame] | 147 | @Override |
| 148 | public String toString() { |
sakal | 71a62b9 | 2017-08-10 02:12:24 -0700 | [diff] [blame] | 149 | final String type = yuvFrame |
| 150 | ? "Y: " + yuvStrides[0] + ", U: " + yuvStrides[1] + ", V: " + yuvStrides[2] |
| 151 | : "Texture: " + textureId; |
| 152 | return width + "x" + height + ", " + type; |
henrike@webrtc.org | 28e2075 | 2013-07-10 00:45:36 +0000 | [diff] [blame] | 153 | } |
sakal | 6bdcefc | 2017-08-15 01:56:02 -0700 | [diff] [blame] | 154 | |
Sami Kalliomäki | 7d1f493 | 2017-09-14 12:31:53 +0200 | [diff] [blame] | 155 | /** |
| 156 | * Convert the frame to VideoFrame. It is no longer safe to use the I420Frame after calling |
| 157 | * this. |
| 158 | */ |
sakal | 6bdcefc | 2017-08-15 01:56:02 -0700 | [diff] [blame] | 159 | VideoFrame toVideoFrame() { |
| 160 | final VideoFrame.Buffer buffer; |
Sami Kalliomäki | 7d1f493 | 2017-09-14 12:31:53 +0200 | [diff] [blame] | 161 | if (backingBuffer != null) { |
| 162 | // We were construted from a VideoFrame.Buffer, just return it. |
| 163 | // Make sure webrtc::VideoFrame object is released. |
| 164 | backingBuffer.retain(); |
| 165 | VideoRenderer.renderFrameDone(this); |
| 166 | buffer = backingBuffer; |
| 167 | } else if (yuvFrame) { |
Sami Kalliomäki | 48b3c02 | 2017-10-04 17:01:00 +0200 | [diff] [blame] | 168 | buffer = JavaI420Buffer.wrap(width, height, yuvPlanes[0], yuvStrides[0], yuvPlanes[1], |
sakal | 6bdcefc | 2017-08-15 01:56:02 -0700 | [diff] [blame] | 169 | yuvStrides[1], yuvPlanes[2], yuvStrides[2], |
| 170 | () -> { VideoRenderer.renderFrameDone(this); }); |
| 171 | } else { |
| 172 | // Note: surfaceTextureHelper being null means calling toI420 will crash. |
| 173 | buffer = new TextureBufferImpl(width, height, VideoFrame.TextureBuffer.Type.OES, textureId, |
| 174 | RendererCommon.convertMatrixToAndroidGraphicsMatrix(samplingMatrix), |
| 175 | null /* surfaceTextureHelper */, () -> { VideoRenderer.renderFrameDone(this); }); |
| 176 | } |
| 177 | return new VideoFrame(buffer, rotationDegree, 0 /* timestampNs */); |
| 178 | } |
Magnus Jedvert | 9060eb1 | 2017-12-12 12:52:54 +0100 | [diff] [blame] | 179 | |
| 180 | @CalledByNative("I420Frame") |
| 181 | static I420Frame createI420Frame(int width, int height, int rotationDegree, int y_stride, |
| 182 | ByteBuffer y_buffer, int u_stride, ByteBuffer u_buffer, int v_stride, ByteBuffer v_buffer, |
| 183 | long nativeFramePointer) { |
| 184 | return new I420Frame(width, height, rotationDegree, new int[] {y_stride, u_stride, v_stride}, |
| 185 | new ByteBuffer[] {y_buffer, u_buffer, v_buffer}, nativeFramePointer); |
| 186 | } |
| 187 | |
| 188 | @CalledByNative("I420Frame") |
| 189 | static I420Frame createTextureFrame(int width, int height, int rotationDegree, int textureId, |
| 190 | float[] samplingMatrix, long nativeFramePointer) { |
| 191 | return new I420Frame( |
| 192 | width, height, rotationDegree, textureId, samplingMatrix, nativeFramePointer); |
| 193 | } |
glaznev@webrtc.org | f693229 | 2015-02-05 17:29:59 +0000 | [diff] [blame] | 194 | } |
henrike@webrtc.org | 28e2075 | 2013-07-10 00:45:36 +0000 | [diff] [blame] | 195 | |
glaznev@webrtc.org | f693229 | 2015-02-05 17:29:59 +0000 | [diff] [blame] | 196 | // Helper native function to do a video frame plane copying. |
Magnus Jedvert | 9060eb1 | 2017-12-12 12:52:54 +0100 | [diff] [blame] | 197 | static native void copyPlaneNative( |
sakal | b6760f9 | 2016-09-29 04:12:44 -0700 | [diff] [blame] | 198 | ByteBuffer src, int width, int height, int srcStride, ByteBuffer dst, int dstStride); |
henrike@webrtc.org | 28e2075 | 2013-07-10 00:45:36 +0000 | [diff] [blame] | 199 | |
Niels Möller | 8f59762 | 2016-03-23 10:33:07 +0100 | [diff] [blame] | 200 | /** The real meat of VideoSinkInterface. */ |
henrike@webrtc.org | 28e2075 | 2013-07-10 00:45:36 +0000 | [diff] [blame] | 201 | public static interface Callbacks { |
guoweis@webrtc.org | 00c509a | 2015-03-12 21:37:26 +0000 | [diff] [blame] | 202 | // |frame| might have pending rotation and implementation of Callbacks |
Magnus Jedvert | a6cba3a | 2015-08-29 15:57:43 +0200 | [diff] [blame] | 203 | // should handle that by applying rotation during rendering. The callee |
| 204 | // is responsible for signaling when it is done with |frame| by calling |
| 205 | // renderFrameDone(frame). |
Magnus Jedvert | 9060eb1 | 2017-12-12 12:52:54 +0100 | [diff] [blame] | 206 | @CalledByNative("Callbacks") void renderFrame(I420Frame frame); |
henrike@webrtc.org | 28e2075 | 2013-07-10 00:45:36 +0000 | [diff] [blame] | 207 | } |
| 208 | |
sakal | b6760f9 | 2016-09-29 04:12:44 -0700 | [diff] [blame] | 209 | /** |
| 210 | * This must be called after every renderFrame() to release the frame. |
| 211 | */ |
| 212 | public static void renderFrameDone(I420Frame frame) { |
| 213 | frame.yuvPlanes = null; |
| 214 | frame.textureId = 0; |
| 215 | if (frame.nativeFramePointer != 0) { |
| 216 | releaseNativeFrame(frame.nativeFramePointer); |
| 217 | frame.nativeFramePointer = 0; |
| 218 | } |
| 219 | } |
Magnus Jedvert | a6cba3a | 2015-08-29 15:57:43 +0200 | [diff] [blame] | 220 | |
magjed | d5031fc | 2015-08-14 03:13:05 -0700 | [diff] [blame] | 221 | long nativeVideoRenderer; |
henrike@webrtc.org | 28e2075 | 2013-07-10 00:45:36 +0000 | [diff] [blame] | 222 | |
| 223 | public VideoRenderer(Callbacks callbacks) { |
Magnus Jedvert | 9060eb1 | 2017-12-12 12:52:54 +0100 | [diff] [blame] | 224 | nativeVideoRenderer = createNativeVideoRenderer(callbacks); |
henrike@webrtc.org | 28e2075 | 2013-07-10 00:45:36 +0000 | [diff] [blame] | 225 | } |
| 226 | |
henrike@webrtc.org | 28e2075 | 2013-07-10 00:45:36 +0000 | [diff] [blame] | 227 | public void dispose() { |
magjed | d5031fc | 2015-08-14 03:13:05 -0700 | [diff] [blame] | 228 | if (nativeVideoRenderer == 0) { |
| 229 | // Already disposed. |
| 230 | return; |
| 231 | } |
perkj | 47b6263 | 2016-02-08 01:07:19 -0800 | [diff] [blame] | 232 | |
| 233 | freeWrappedVideoRenderer(nativeVideoRenderer); |
magjed | d5031fc | 2015-08-14 03:13:05 -0700 | [diff] [blame] | 234 | nativeVideoRenderer = 0; |
henrike@webrtc.org | 28e2075 | 2013-07-10 00:45:36 +0000 | [diff] [blame] | 235 | } |
| 236 | |
Magnus Jedvert | 9060eb1 | 2017-12-12 12:52:54 +0100 | [diff] [blame] | 237 | private static native long createNativeVideoRenderer(Callbacks callbacks); |
glaznev@webrtc.org | a59c501 | 2014-09-17 03:26:59 +0000 | [diff] [blame] | 238 | private static native void freeWrappedVideoRenderer(long nativeVideoRenderer); |
Magnus Jedvert | a6cba3a | 2015-08-29 15:57:43 +0200 | [diff] [blame] | 239 | private static native void releaseNativeFrame(long nativeFramePointer); |
henrike@webrtc.org | 28e2075 | 2013-07-10 00:45:36 +0000 | [diff] [blame] | 240 | } |