Use the underlying type of Java Video Buffer on Java -> C++ Frame Buffer
Just like the C++ API, add a method in Java VideoFrame.Buffer that
describes the underlying implementation.
Use this method to properly select AndroidVideoBuffer
or AndroidVideoI420Buffer in Java -> C++ Video Frame Conversion.
Also, add a test case for WrappedNativeI420Buffer
in VideoFrameBufferTest for consistency.
Bug: webrtc:12602
Change-Id: I4c0444e8af6f6a1109bc514e7ab6c2214f1f6d60
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/223080
Commit-Queue: Byoungchan Lee <daniel.l@hpcnt.com>
Reviewed-by: Niels Moller <nisse@webrtc.org>
Reviewed-by: Xavier Lepaul‎ <xalep@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#34545}
diff --git a/sdk/android/BUILD.gn b/sdk/android/BUILD.gn
index 57b9d62..d4e8d18 100644
--- a/sdk/android/BUILD.gn
+++ b/sdk/android/BUILD.gn
@@ -202,7 +202,9 @@
":base_java",
"//rtc_base:base_java",
"//third_party/android_deps:com_android_support_support_annotations_java",
+ "//third_party/androidx:androidx_annotation_annotation_java",
]
+ srcjar_deps = [ "//api/video:video_frame_enums" ]
}
rtc_android_library("video_java") {
@@ -1486,11 +1488,16 @@
rtc_library("instrumentationtests_jni") {
testonly = true
- sources = [ "instrumentationtests/loggable_test.cc" ]
+ sources = [
+ "instrumentationtests/loggable_test.cc",
+ "instrumentationtests/video_frame_buffer_test.cc",
+ ]
deps = [
":base_jni",
":native_api_jni",
+ ":videoframe_jni",
+ "../../api/video:video_frame",
"../../rtc_base:rtc_base_approved",
]
}
diff --git a/sdk/android/api/org/webrtc/VideoFrame.java b/sdk/android/api/org/webrtc/VideoFrame.java
index a0a0d4e..bb30069 100644
--- a/sdk/android/api/org/webrtc/VideoFrame.java
+++ b/sdk/android/api/org/webrtc/VideoFrame.java
@@ -35,6 +35,15 @@
*/
public interface Buffer extends RefCounted {
/**
+ * Representation of the underlying buffer. Currently, only NATIVE and I420 are supported.
+ */
+ @CalledByNative("Buffer")
+ @VideoFrameBufferType
+ default int getBufferType() {
+ return VideoFrameBufferType.NATIVE;
+ }
+
+ /**
* Resolution of the buffer in pixels.
*/
@CalledByNative("Buffer") int getWidth();
@@ -63,6 +72,11 @@
* Interface for I420 buffers.
*/
public interface I420Buffer extends Buffer {
+ @Override
+ default int getBufferType() {
+ return VideoFrameBufferType.I420;
+ }
+
/**
* Returns a direct ByteBuffer containing Y-plane data. The buffer capacity is at least
* getStrideY() * getHeight() bytes. The position of the returned buffer is ignored and must
diff --git a/sdk/android/instrumentationtests/src/org/webrtc/VideoFrameBufferTest.java b/sdk/android/instrumentationtests/src/org/webrtc/VideoFrameBufferTest.java
index 53b2c58..4388e4a 100644
--- a/sdk/android/instrumentationtests/src/org/webrtc/VideoFrameBufferTest.java
+++ b/sdk/android/instrumentationtests/src/org/webrtc/VideoFrameBufferTest.java
@@ -32,6 +32,7 @@
import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.runner.RunWith;
+import org.webrtc.VideoFrame;
/**
* Test VideoFrame buffers of different kind of formats: I420, RGB, OES, NV12, NV21, and verify
@@ -45,7 +46,7 @@
/**
* These tests are parameterized on this enum which represents the different VideoFrame.Buffers.
*/
- private static enum BufferType { I420, RGB_TEXTURE, OES_TEXTURE, NV21, NV12 }
+ private static enum BufferType { I420_JAVA, I420_NATIVE, RGB_TEXTURE, OES_TEXTURE, NV21, NV12 }
@ClassParameter private static List<ParameterSet> CLASS_PARAMS = new ArrayList<>();
@@ -75,17 +76,39 @@
*/
private static VideoFrame.Buffer createBufferWithType(
BufferType bufferType, VideoFrame.I420Buffer i420Buffer) {
+ VideoFrame.Buffer buffer;
switch (bufferType) {
- case I420:
- return i420Buffer.toI420();
+ case I420_JAVA:
+ buffer = i420Buffer;
+ buffer.retain();
+ assertEquals(VideoFrameBufferType.I420, buffer.getBufferType());
+ assertEquals(VideoFrameBufferType.I420, nativeGetBufferType(buffer));
+ return buffer;
+ case I420_NATIVE:
+ buffer = nativeGetNativeI420Buffer(i420Buffer);
+ assertEquals(VideoFrameBufferType.I420, buffer.getBufferType());
+ assertEquals(VideoFrameBufferType.I420, nativeGetBufferType(buffer));
+ return buffer;
case RGB_TEXTURE:
- return createRgbTextureBuffer(/* eglContext= */ null, i420Buffer);
+ buffer = createRgbTextureBuffer(/* eglContext= */ null, i420Buffer);
+ assertEquals(VideoFrameBufferType.NATIVE, buffer.getBufferType());
+ assertEquals(VideoFrameBufferType.NATIVE, nativeGetBufferType(buffer));
+ return buffer;
case OES_TEXTURE:
- return createOesTextureBuffer(/* eglContext= */ null, i420Buffer);
+ buffer = createOesTextureBuffer(/* eglContext= */ null, i420Buffer);
+ assertEquals(VideoFrameBufferType.NATIVE, buffer.getBufferType());
+ assertEquals(VideoFrameBufferType.NATIVE, nativeGetBufferType(buffer));
+ return buffer;
case NV21:
- return createNV21Buffer(i420Buffer);
+ buffer = createNV21Buffer(i420Buffer);
+ assertEquals(VideoFrameBufferType.NATIVE, buffer.getBufferType());
+ assertEquals(VideoFrameBufferType.NATIVE, nativeGetBufferType(buffer));
+ return buffer;
case NV12:
- return createNV12Buffer(i420Buffer);
+ buffer = createNV12Buffer(i420Buffer);
+ assertEquals(VideoFrameBufferType.NATIVE, buffer.getBufferType());
+ assertEquals(VideoFrameBufferType.NATIVE, nativeGetBufferType(buffer));
+ return buffer;
default:
throw new IllegalArgumentException("Unknown buffer type: " + bufferType);
}
@@ -452,6 +475,7 @@
final VideoFrame.I420Buffer outputI420Buffer = bufferToTest.toI420();
bufferToTest.release();
+ assertEquals(VideoFrameBufferType.I420, nativeGetBufferType(outputI420Buffer));
assertAlmostEqualI420Buffers(referenceI420Buffer, outputI420Buffer);
referenceI420Buffer.release();
outputI420Buffer.release();
@@ -504,4 +528,10 @@
testCropAndScale(4 /* cropX= */, 4 /* cropY= */, /* cropWidth= */ 12, /* cropHeight= */ 12,
/* scaleWidth= */ 8, /* scaleHeight= */ 8);
}
+
+ @VideoFrameBufferType private static native int nativeGetBufferType(VideoFrame.Buffer buffer);
+
+ /** Returns the copy of I420Buffer using WrappedNativeI420Buffer. */
+ private static native VideoFrame.Buffer nativeGetNativeI420Buffer(
+ VideoFrame.I420Buffer i420Buffer);
}
diff --git a/sdk/android/instrumentationtests/video_frame_buffer_test.cc b/sdk/android/instrumentationtests/video_frame_buffer_test.cc
new file mode 100644
index 0000000..686b232
--- /dev/null
+++ b/sdk/android/instrumentationtests/video_frame_buffer_test.cc
@@ -0,0 +1,45 @@
+/*
+ * Copyright 2021 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "api/video/i420_buffer.h"
+#include "sdk/android/src/jni/jni_helpers.h"
+#include "sdk/android/src/jni/video_frame.h"
+#include "sdk/android/src/jni/wrapped_native_i420_buffer.h"
+
+namespace webrtc {
+namespace jni {
+
+JNI_FUNCTION_DECLARATION(jint,
+ VideoFrameBufferTest_nativeGetBufferType,
+ JNIEnv* jni,
+ jclass,
+ jobject video_frame_buffer) {
+ const JavaParamRef<jobject> j_video_frame_buffer(video_frame_buffer);
+ rtc::scoped_refptr<VideoFrameBuffer> buffer =
+ JavaToNativeFrameBuffer(jni, j_video_frame_buffer);
+ return static_cast<jint>(buffer->type());
+}
+
+JNI_FUNCTION_DECLARATION(jobject,
+ VideoFrameBufferTest_nativeGetNativeI420Buffer,
+ JNIEnv* jni,
+ jclass,
+ jobject i420_buffer) {
+ const JavaParamRef<jobject> j_i420_buffer(i420_buffer);
+ rtc::scoped_refptr<VideoFrameBuffer> buffer =
+ JavaToNativeFrameBuffer(jni, j_i420_buffer);
+ const I420BufferInterface* inputBuffer = buffer->GetI420();
+ RTC_DCHECK(inputBuffer != nullptr);
+ rtc::scoped_refptr<I420Buffer> outputBuffer = I420Buffer::Copy(*inputBuffer);
+ return WrapI420Buffer(jni, outputBuffer).Release();
+}
+
+} // namespace jni
+} // namespace webrtc
diff --git a/sdk/android/src/jni/video_frame.cc b/sdk/android/src/jni/video_frame.cc
index e27a5c8..35d04d8 100644
--- a/sdk/android/src/jni/video_frame.cc
+++ b/sdk/android/src/jni/video_frame.cc
@@ -70,6 +70,13 @@
class AndroidVideoI420Buffer : public I420BufferInterface {
public:
+ // Creates a native VideoFrameBuffer from a Java VideoFrame.I420Buffer.
+ static rtc::scoped_refptr<AndroidVideoI420Buffer> Create(
+ JNIEnv* jni,
+ int width,
+ int height,
+ const JavaRef<jobject>& j_video_frame_buffer);
+
// Adopts and takes ownership of the Java VideoFrame.Buffer. I.e. retain()
// will not be called, but release() will be called when the returned
// AndroidVideoBuffer is destroyed.
@@ -113,11 +120,24 @@
int stride_v_;
};
+rtc::scoped_refptr<AndroidVideoI420Buffer> AndroidVideoI420Buffer::Create(
+ JNIEnv* jni,
+ int width,
+ int height,
+ const JavaRef<jobject>& j_video_frame_buffer) {
+ Java_Buffer_retain(jni, j_video_frame_buffer);
+ return AndroidVideoI420Buffer::Adopt(jni, width, height,
+ j_video_frame_buffer);
+}
+
rtc::scoped_refptr<AndroidVideoI420Buffer> AndroidVideoI420Buffer::Adopt(
JNIEnv* jni,
int width,
int height,
const JavaRef<jobject>& j_video_frame_buffer) {
+ RTC_DCHECK_EQ(
+ static_cast<Type>(Java_Buffer_getBufferType(jni, j_video_frame_buffer)),
+ Type::kI420);
return rtc::make_ref_counted<AndroidVideoI420Buffer>(jni, width, height,
j_video_frame_buffer);
}
@@ -164,6 +184,9 @@
rtc::scoped_refptr<AndroidVideoBuffer> AndroidVideoBuffer::Adopt(
JNIEnv* jni,
const JavaRef<jobject>& j_video_frame_buffer) {
+ RTC_DCHECK_EQ(
+ static_cast<Type>(Java_Buffer_getBufferType(jni, j_video_frame_buffer)),
+ Type::kNative);
return rtc::make_ref_counted<AndroidVideoBuffer>(jni, j_video_frame_buffer);
}
@@ -233,7 +256,20 @@
rtc::scoped_refptr<VideoFrameBuffer> JavaToNativeFrameBuffer(
JNIEnv* jni,
const JavaRef<jobject>& j_video_frame_buffer) {
- return AndroidVideoBuffer::Create(jni, j_video_frame_buffer);
+ VideoFrameBuffer::Type type = static_cast<VideoFrameBuffer::Type>(
+ Java_Buffer_getBufferType(jni, j_video_frame_buffer));
+ switch (type) {
+ case VideoFrameBuffer::Type::kI420: {
+ const int width = Java_Buffer_getWidth(jni, j_video_frame_buffer);
+ const int height = Java_Buffer_getHeight(jni, j_video_frame_buffer);
+ return AndroidVideoI420Buffer::Create(jni, width, height,
+ j_video_frame_buffer);
+ }
+ case VideoFrameBuffer::Type::kNative:
+ return AndroidVideoBuffer::Create(jni, j_video_frame_buffer);
+ default:
+ RTC_CHECK_NOTREACHED();
+ }
}
VideoFrame JavaToNativeFrame(JNIEnv* jni,
@@ -243,8 +279,8 @@
Java_VideoFrame_getBuffer(jni, j_video_frame);
int rotation = Java_VideoFrame_getRotation(jni, j_video_frame);
int64_t timestamp_ns = Java_VideoFrame_getTimestampNs(jni, j_video_frame);
- rtc::scoped_refptr<AndroidVideoBuffer> buffer =
- AndroidVideoBuffer::Create(jni, j_video_frame_buffer);
+ rtc::scoped_refptr<VideoFrameBuffer> buffer =
+ JavaToNativeFrameBuffer(jni, j_video_frame_buffer);
return VideoFrame::Builder()
.set_video_frame_buffer(buffer)
.set_timestamp_rtp(timestamp_rtp)