Add VideoSink interface to VideoFileRenderer.
Bug: webrtc:8776
Change-Id: I1782b0c197abf6f82a200a2808ddc87d1f250326
Reviewed-on: https://webrtc-review.googlesource.com/41320
Reviewed-by: Anders Carlsson <andersc@webrtc.org>
Commit-Queue: Sami Kalliomäki <sakal@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#21719}diff --git a/sdk/android/api/org/webrtc/VideoFileRenderer.java b/sdk/android/api/org/webrtc/VideoFileRenderer.java
index 20ca839..995c562 100644
--- a/sdk/android/api/org/webrtc/VideoFileRenderer.java
+++ b/sdk/android/api/org/webrtc/VideoFileRenderer.java
@@ -17,17 +17,18 @@
import java.nio.ByteBuffer;
import java.nio.charset.Charset;
import java.util.ArrayList;
+import java.util.concurrent.BlockingQueue;
import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.LinkedBlockingQueue;
/**
* Can be used to save the video frames to file.
*/
@JNINamespace("webrtc::jni")
-public class VideoFileRenderer implements VideoRenderer.Callbacks {
+public class VideoFileRenderer implements VideoRenderer.Callbacks, VideoSink {
private static final String TAG = "VideoFileRenderer";
private final HandlerThread renderThread;
- private final Object handlerLock = new Object();
private final Handler renderThreadHandler;
private final FileOutputStream videoOutFile;
private final String outputFileName;
@@ -73,61 +74,56 @@
}
@Override
- public void renderFrame(final VideoRenderer.I420Frame frame) {
- renderThreadHandler.post(new Runnable() {
- @Override
- public void run() {
- renderFrameOnRenderThread(frame);
- }
- });
+ public void renderFrame(final VideoRenderer.I420Frame i420Frame) {
+ final VideoFrame frame = i420Frame.toVideoFrame();
+ onFrame(frame);
+ frame.release();
}
- // TODO(sakal): yuvConverter.convert is deprecated. This will be removed once this file is updated
- // to implement VideoSink instead of VideoRenderer.Callbacks.
- @SuppressWarnings("deprecation")
- private void renderFrameOnRenderThread(VideoRenderer.I420Frame frame) {
- final float frameAspectRatio = (float) frame.rotatedWidth() / (float) frame.rotatedHeight();
+ @Override
+ public void onFrame(VideoFrame frame) {
+ frame.retain();
+ renderThreadHandler.post(() -> renderFrameOnRenderThread(frame));
+ }
- final float[] rotatedSamplingMatrix =
- RendererCommon.rotateTextureMatrix(frame.samplingMatrix, frame.rotationDegree);
- final float[] layoutMatrix = RendererCommon.getLayoutMatrix(
- false, frameAspectRatio, (float) outputFileWidth / outputFileHeight);
- final float[] texMatrix = RendererCommon.multiplyMatrices(rotatedSamplingMatrix, layoutMatrix);
+ private void renderFrameOnRenderThread(VideoFrame frame) {
+ final VideoFrame.Buffer buffer = frame.getBuffer();
- try {
- ByteBuffer buffer = JniCommon.nativeAllocateByteBuffer(outputFrameSize);
- if (!frame.yuvFrame) {
- yuvConverter.convert(outputFrameBuffer, outputFileWidth, outputFileHeight, outputFileWidth,
- frame.textureId, texMatrix);
+ // If the frame is rotated, it will be applied after cropAndScale. Therefore, if the frame is
+ // rotated by 90 degrees, swap width and height.
+ final int targetWidth = frame.getRotation() % 180 == 0 ? outputFileWidth : outputFileHeight;
+ final int targetHeight = frame.getRotation() % 180 == 0 ? outputFileHeight : outputFileWidth;
- int stride = outputFileWidth;
- byte[] data = outputFrameBuffer.array();
- int offset = outputFrameBuffer.arrayOffset();
+ final float frameAspectRatio = (float) buffer.getWidth() / (float) buffer.getHeight();
+ final float fileAspectRatio = (float) targetWidth / (float) targetHeight;
- // Write Y
- buffer.put(data, offset, outputFileWidth * outputFileHeight);
-
- // Write U
- for (int r = outputFileHeight; r < outputFileHeight * 3 / 2; ++r) {
- buffer.put(data, offset + r * stride, stride / 2);
- }
-
- // Write V
- for (int r = outputFileHeight; r < outputFileHeight * 3 / 2; ++r) {
- buffer.put(data, offset + r * stride + stride / 2, stride / 2);
- }
- } else {
- nativeI420Scale(frame.yuvPlanes[0], frame.yuvStrides[0], frame.yuvPlanes[1],
- frame.yuvStrides[1], frame.yuvPlanes[2], frame.yuvStrides[2], frame.width, frame.height,
- outputFrameBuffer, outputFileWidth, outputFileHeight);
-
- buffer.put(outputFrameBuffer.array(), outputFrameBuffer.arrayOffset(), outputFrameSize);
- }
- buffer.rewind();
- rawFrames.add(buffer);
- } finally {
- VideoRenderer.renderFrameDone(frame);
+ // Calculate cropping to equalize the aspect ratio.
+ int cropWidth = buffer.getWidth();
+ int cropHeight = buffer.getHeight();
+ if (fileAspectRatio > frameAspectRatio) {
+ cropHeight *= frameAspectRatio / fileAspectRatio;
+ } else {
+ cropWidth *= fileAspectRatio / frameAspectRatio;
}
+
+ final int cropX = (buffer.getWidth() - cropWidth) / 2;
+ final int cropY = (buffer.getHeight() - cropHeight) / 2;
+
+ final VideoFrame.Buffer scaledBuffer =
+ buffer.cropAndScale(cropX, cropY, cropWidth, cropHeight, targetWidth, targetHeight);
+ frame.release();
+
+ final VideoFrame.I420Buffer i420 = scaledBuffer.toI420();
+ scaledBuffer.release();
+
+ ByteBuffer byteBuffer = JniCommon.nativeAllocateByteBuffer(outputFrameSize);
+ YuvHelper.I420Rotate(i420.getDataY(), i420.getStrideY(), i420.getDataU(), i420.getStrideU(),
+ i420.getDataV(), i420.getStrideV(), byteBuffer, i420.getWidth(), i420.getHeight(),
+ frame.getRotation());
+ i420.release();
+
+ byteBuffer.rewind();
+ rawFrames.add(byteBuffer);
}
/**
@@ -135,14 +131,11 @@
*/
public void release() {
final CountDownLatch cleanupBarrier = new CountDownLatch(1);
- renderThreadHandler.post(new Runnable() {
- @Override
- public void run() {
- yuvConverter.release();
- eglBase.release();
- renderThread.quit();
- cleanupBarrier.countDown();
- }
+ renderThreadHandler.post(() -> {
+ yuvConverter.release();
+ eglBase.release();
+ renderThread.quit();
+ cleanupBarrier.countDown();
});
ThreadUtils.awaitUninterruptibly(cleanupBarrier);
try {
@@ -157,15 +150,12 @@
JniCommon.nativeFreeByteBuffer(buffer);
}
videoOutFile.close();
- Logging.d(TAG, "Video written to disk as " + outputFileName + ". Number frames are "
- + rawFrames.size() + " and the dimension of the frames are " + outputFileWidth + "x"
- + outputFileHeight + ".");
+ Logging.d(TAG,
+ "Video written to disk as " + outputFileName + ". Number frames are " + rawFrames.size()
+ + " and the dimension of the frames are " + outputFileWidth + "x" + outputFileHeight
+ + ".");
} catch (IOException e) {
Logging.e(TAG, "Error writing video to disk", e);
}
}
-
- public static native void nativeI420Scale(ByteBuffer srcY, int strideY, ByteBuffer srcU,
- int strideU, ByteBuffer srcV, int strideV, int width, int height, ByteBuffer dst,
- int dstWidth, int dstHeight);
}
diff --git a/sdk/android/api/org/webrtc/YuvHelper.java b/sdk/android/api/org/webrtc/YuvHelper.java
index 2d6877b..105584c 100644
--- a/sdk/android/api/org/webrtc/YuvHelper.java
+++ b/sdk/android/api/org/webrtc/YuvHelper.java
@@ -66,6 +66,37 @@
chromaWidth * 2, width, height);
}
+ /** Helper method for rotating I420 to tightly packed destination buffer. */
+ public static void I420Rotate(ByteBuffer srcY, int srcStrideY, ByteBuffer srcU, int srcStrideU,
+ ByteBuffer srcV, int srcStrideV, ByteBuffer dst, int srcWidth, int srcHeight,
+ int rotationMode) {
+ final int dstWidth = rotationMode % 180 == 0 ? srcWidth : srcHeight;
+ final int dstHeight = rotationMode % 180 == 0 ? srcHeight : srcWidth;
+
+ final int dstChromaHeight = (dstHeight + 1) / 2;
+ final int dstChromaWidth = (dstWidth + 1) / 2;
+
+ final int minSize = dstWidth * dstHeight + dstChromaWidth * dstChromaHeight * 2;
+ if (dst.capacity() < minSize) {
+ throw new IllegalArgumentException("Expected destination buffer capacity to be at least "
+ + minSize + " was " + dst.capacity());
+ }
+
+ final int startY = 0;
+ final int startU = dstHeight * dstWidth;
+ final int startV = startU + dstChromaHeight * dstChromaWidth;
+
+ dst.position(startY);
+ final ByteBuffer dstY = dst.slice();
+ dst.position(startU);
+ final ByteBuffer dstU = dst.slice();
+ dst.position(startV);
+ final ByteBuffer dstV = dst.slice();
+
+ nativeI420Rotate(srcY, srcStrideY, srcU, srcStrideU, srcV, srcStrideV, dstY, dstWidth, dstU,
+ dstChromaWidth, dstV, dstChromaWidth, srcWidth, srcHeight, rotationMode);
+ }
+
public static void I420Copy(ByteBuffer srcY, int srcStrideY, ByteBuffer srcU, int srcStrideU,
ByteBuffer srcV, int srcStrideV, ByteBuffer dstY, int dstStrideY, ByteBuffer dstU,
int dstStrideU, ByteBuffer dstV, int dstStrideV, int width, int height) {
@@ -80,10 +111,22 @@
dstStrideUV, width, height);
}
+ public static void I420Rotate(ByteBuffer srcY, int srcStrideY, ByteBuffer srcU, int srcStrideU,
+ ByteBuffer srcV, int srcStrideV, ByteBuffer dstY, int dstStrideY, ByteBuffer dstU,
+ int dstStrideU, ByteBuffer dstV, int dstStrideV, int srcWidth, int srcHeight,
+ int rotationMode) {
+ nativeI420Rotate(srcY, srcStrideY, srcU, srcStrideU, srcV, srcStrideV, dstY, dstStrideY, dstU,
+ dstStrideU, dstV, dstStrideV, srcWidth, srcHeight, rotationMode);
+ }
+
private static native void nativeI420Copy(ByteBuffer srcY, int srcStrideY, ByteBuffer srcU,
int srcStrideU, ByteBuffer srcV, int srcStrideV, ByteBuffer dstY, int dstStrideY,
ByteBuffer dstU, int dstStrideU, ByteBuffer dstV, int dstStrideV, int width, int height);
private static native void nativeI420ToNV12(ByteBuffer srcY, int srcStrideY, ByteBuffer srcU,
int srcStrideU, ByteBuffer srcV, int srcStrideV, ByteBuffer dstY, int dstStrideY,
ByteBuffer dstUV, int dstStrideUV, int width, int height);
+ private static native void nativeI420Rotate(ByteBuffer srcY, int srcStrideY, ByteBuffer srcU,
+ int srcStrideU, ByteBuffer srcV, int srcStrideV, ByteBuffer dstY, int dstStrideY,
+ ByteBuffer dstU, int dstStrideU, ByteBuffer dstV, int dstStrideV, int srcWidth, int srcHeight,
+ int rotationMode);
}