Add Android Camera To Unity Plugin
The existing unity plugin (an example in webrtc codebase) does not support camera access on Android platform. This CL implements such functionality.
TBR=gyzhou@chromium.org
BUG=webrtc:8067
Review-Url: https://codereview.webrtc.org/2993273002
Cr-Commit-Position: refs/heads/master@{#19277}
diff --git a/webrtc/examples/unityplugin/ANDROID_INSTRUCTION b/webrtc/examples/unityplugin/ANDROID_INSTRUCTION
new file mode 100644
index 0000000..d782e94
--- /dev/null
+++ b/webrtc/examples/unityplugin/ANDROID_INSTRUCTION
@@ -0,0 +1,33 @@
+Instruction of Running webrtc_unity_plugin on Android Unity
+
+1. On Linux machine, compile target webrtc_unity_plugin.
+ Checkout WebRTC codebase: fetch --no-hooks webrtc_android
+ If you already have a checkout for linux, add target_os=”android” into .gclient file.
+ Run gclient sync
+ Run gn args out/Android, and again set target_os=”android” in the args.gn
+ Modify file src/build/android/android_only_jni_exports.lst, to expose all functions. Namely, change "global" section to "*", and remove "local" section. Otherwise, Unity C# code will not be able to access the functions defined in the plugin.
+ Run ninja -C out/Android webrtc_unity_plugin
+
+2. On Linux machine, compile target libwebrtc_unity under webrtc checkout. This is the java code for webrtc to work on Android.
+
+3. Copy libwebrtc_unity.jar and libwebrtc_unity_plugin.so into Unity project folder, under Assets/Plugins/Android folder.
+
+4. Rename libwebrtc_unity_plugin.so to libjingle_peerconnection_so.so. Again, this is hacky, and the purpose is to let the java code in libwebrtc.jar to find their JNI implementation. And simultaneously, in your C# wrapper script for the native plugin libjingle_peerconnection_so.so, the dll_path should be set to “jingle_peerconnection_so”.
+
+5. In the Unity Main Scene’s Start method, write the following code to initialize the Java environment for webrtc (otherwise, webrtc will not be able to access audio device or camera from C++ code):
+
+#if UNITY_ANDROID
+ AndroidJavaClass playerClass = new AndroidJavaClass("com.unity3d.player.UnityPlayer");
+ AndroidJavaObject activity = playerClass.GetStatic<AndroidJavaObject>("currentActivity");
+ AndroidJavaClass webrtcClass = new AndroidJavaClass("org.webrtc.PeerConnectionFactory");
+ if (webrtcClass != null)
+ {
+ webrtcClass.CallStatic("initializeAndroidGlobals", new object[2] { activity, false });
+ }
+#endif
+
+6. Compile the unity project into an APK, and decompile the apk using apktool you can download from internet. And copy the AndroidManifest.xml to the Assets/Plugins/Android folder, and add two lines:
+ <uses-permission android:name=”android.permission.RECORD_AUDIO” />
+ <uses-permission android:name=”android.permission.CAMERA” />
+
+ The purpose of using apktool is to get a well-written android manifest xml file. If you know how to write manifest file from scratch, you can skip using apktool.
diff --git a/webrtc/examples/unityplugin/DEPS b/webrtc/examples/unityplugin/DEPS
new file mode 100644
index 0000000..ede414b
--- /dev/null
+++ b/webrtc/examples/unityplugin/DEPS
@@ -0,0 +1,3 @@
+include_rules = [
+ "+webrtc/sdk",
+]
diff --git a/webrtc/examples/unityplugin/README b/webrtc/examples/unityplugin/README
index 82a466a..5f26b89 100644
--- a/webrtc/examples/unityplugin/README
+++ b/webrtc/examples/unityplugin/README
@@ -1,7 +1,10 @@
-This directory contains an example Unity native plugin for Windows OS.
+This directory contains an example Unity native plugin for Windows OS and Android.
+
The APIs use Platform Invoke (P/Invoke) technology as required by Unity native plugin.
This plugin dll can also be used by Windows C# applications other than Unity.
+For detailed build instruction on Android, see ANDROID_INSTRUCTION
+
An example of wrapping native plugin into a C# managed class in Unity is given as following:
using System;
diff --git a/webrtc/examples/unityplugin/classreferenceholder.cc b/webrtc/examples/unityplugin/classreferenceholder.cc
new file mode 100644
index 0000000..8814699
--- /dev/null
+++ b/webrtc/examples/unityplugin/classreferenceholder.cc
@@ -0,0 +1,88 @@
+/*
+ * Copyright 2015 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+#include "webrtc/examples/unityplugin/classreferenceholder.h"
+
+#include <utility>
+
+#include "webrtc/sdk/android/src/jni/jni_helpers.h"
+
+namespace unity_plugin {
+
+// ClassReferenceHolder holds global reference to Java classes in app/webrtc.
+class ClassReferenceHolder {
+ public:
+ explicit ClassReferenceHolder(JNIEnv* jni);
+ ~ClassReferenceHolder();
+
+ void FreeReferences(JNIEnv* jni);
+ jclass GetClass(const std::string& name);
+
+ void LoadClass(JNIEnv* jni, const std::string& name);
+
+ private:
+ std::map<std::string, jclass> classes_;
+};
+
+// Allocated in LoadGlobalClassReferenceHolder(),
+// freed in FreeGlobalClassReferenceHolder().
+static ClassReferenceHolder* g_class_reference_holder = nullptr;
+
+void LoadGlobalClassReferenceHolder() {
+ RTC_CHECK(g_class_reference_holder == nullptr);
+ g_class_reference_holder = new ClassReferenceHolder(webrtc_jni::GetEnv());
+}
+
+void FreeGlobalClassReferenceHolder() {
+ g_class_reference_holder->FreeReferences(
+ webrtc_jni::AttachCurrentThreadIfNeeded());
+ delete g_class_reference_holder;
+ g_class_reference_holder = nullptr;
+}
+
+ClassReferenceHolder::ClassReferenceHolder(JNIEnv* jni) {
+ LoadClass(jni, "org/webrtc/UnityUtility");
+}
+
+ClassReferenceHolder::~ClassReferenceHolder() {
+ RTC_CHECK(classes_.empty()) << "Must call FreeReferences() before dtor!";
+}
+
+void ClassReferenceHolder::FreeReferences(JNIEnv* jni) {
+ for (std::map<std::string, jclass>::const_iterator it = classes_.begin();
+ it != classes_.end(); ++it) {
+ jni->DeleteGlobalRef(it->second);
+ }
+ classes_.clear();
+}
+
+jclass ClassReferenceHolder::GetClass(const std::string& name) {
+ std::map<std::string, jclass>::iterator it = classes_.find(name);
+ RTC_CHECK(it != classes_.end()) << "Unexpected GetClass() call for: " << name;
+ return it->second;
+}
+
+void ClassReferenceHolder::LoadClass(JNIEnv* jni, const std::string& name) {
+ jclass localRef = jni->FindClass(name.c_str());
+ CHECK_EXCEPTION(jni) << "error during FindClass: " << name;
+ RTC_CHECK(localRef) << name;
+ jclass globalRef = reinterpret_cast<jclass>(jni->NewGlobalRef(localRef));
+ CHECK_EXCEPTION(jni) << "error during NewGlobalRef: " << name;
+ RTC_CHECK(globalRef) << name;
+ bool inserted = classes_.insert(std::make_pair(name, globalRef)).second;
+ RTC_CHECK(inserted) << "Duplicate class name: " << name;
+}
+
+// Returns a global reference guaranteed to be valid for the lifetime of the
+// process.
+jclass FindClass(JNIEnv* jni, const char* name) {
+ return g_class_reference_holder->GetClass(name);
+}
+
+} // namespace unity_plugin
diff --git a/webrtc/examples/unityplugin/classreferenceholder.h b/webrtc/examples/unityplugin/classreferenceholder.h
new file mode 100644
index 0000000..a3f83e8
--- /dev/null
+++ b/webrtc/examples/unityplugin/classreferenceholder.h
@@ -0,0 +1,37 @@
+/*
+ * Copyright 2015 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+// This is a supplement of webrtc_jni::ClassReferenceHolder.
+// The purpose of this ClassReferenceHolder is to load the example
+// specific java class into JNI c++ side, so that our c++ code can
+// call those java functions.
+
+#ifndef WEBRTC_EXAMPLES_UNITYPLUGIN_CLASSREFERENCEHOLDER_H_
+#define WEBRTC_EXAMPLES_UNITYPLUGIN_CLASSREFERENCEHOLDER_H_
+
+#include <jni.h>
+#include <map>
+#include <string>
+#include <vector>
+
+namespace unity_plugin {
+
+// LoadGlobalClassReferenceHolder must be called in JNI_OnLoad.
+void LoadGlobalClassReferenceHolder();
+// FreeGlobalClassReferenceHolder must be called in JNI_UnLoad.
+void FreeGlobalClassReferenceHolder();
+
+// Returns a global reference guaranteed to be valid for the lifetime of the
+// process.
+jclass FindClass(JNIEnv* jni, const char* name);
+
+} // namespace unity_plugin
+
+#endif // WEBRTC_EXAMPLES_UNITYPLUGIN_CLASSREFERENCEHOLDER_H_
diff --git a/webrtc/examples/unityplugin/java/src/org/webrtc/UnityUtility.java b/webrtc/examples/unityplugin/java/src/org/webrtc/UnityUtility.java
new file mode 100644
index 0000000..b16d20a
--- /dev/null
+++ b/webrtc/examples/unityplugin/java/src/org/webrtc/UnityUtility.java
@@ -0,0 +1,64 @@
+/*
+ * Copyright 2017 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+package org.webrtc;
+
+import android.content.Context;
+import java.util.List;
+
+public class UnityUtility {
+ private static final String VIDEO_CAPTURER_THREAD_NAME = "VideoCapturerThread";
+
+ public static SurfaceTextureHelper LoadSurfaceTextureHelper() {
+ final SurfaceTextureHelper surfaceTextureHelper =
+ SurfaceTextureHelper.create(VIDEO_CAPTURER_THREAD_NAME, null);
+ return surfaceTextureHelper;
+ }
+
+ private static boolean useCamera2() {
+ return Camera2Enumerator.isSupported(ContextUtils.getApplicationContext());
+ }
+
+ private static VideoCapturer createCameraCapturer(CameraEnumerator enumerator) {
+ final String[] deviceNames = enumerator.getDeviceNames();
+
+ for (String deviceName : deviceNames) {
+ if (enumerator.isFrontFacing(deviceName)) {
+ VideoCapturer videoCapturer = enumerator.createCapturer(deviceName, null);
+
+ if (videoCapturer != null) {
+ return videoCapturer;
+ }
+ }
+ }
+
+ return null;
+ }
+
+ public static VideoCapturer LinkCamera(
+ long nativeTrackSource, SurfaceTextureHelper surfaceTextureHelper) {
+ VideoCapturer capturer =
+ createCameraCapturer(new Camera2Enumerator(ContextUtils.getApplicationContext()));
+
+ VideoCapturer.CapturerObserver capturerObserver =
+ new AndroidVideoTrackSourceObserver(nativeTrackSource);
+
+ capturer.initialize(
+ surfaceTextureHelper, ContextUtils.getApplicationContext(), capturerObserver);
+
+ capturer.startCapture(720, 480, 30);
+ return capturer;
+ }
+
+ public static void StopCamera(VideoCapturer camera) throws InterruptedException {
+ camera.stopCapture();
+ camera.dispose();
+ }
+}
diff --git a/webrtc/examples/unityplugin/jni_onload.cc b/webrtc/examples/unityplugin/jni_onload.cc
new file mode 100644
index 0000000..b8b4adf
--- /dev/null
+++ b/webrtc/examples/unityplugin/jni_onload.cc
@@ -0,0 +1,41 @@
+/*
+ * Copyright 2015 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include <jni.h>
+#undef JNIEXPORT
+#define JNIEXPORT __attribute__((visibility("default")))
+
+#include "webrtc/examples/unityplugin/classreferenceholder.h"
+#include "webrtc/rtc_base/ssladapter.h"
+#include "webrtc/sdk/android/src/jni/classreferenceholder.h"
+#include "webrtc/sdk/android/src/jni/jni_helpers.h"
+
+namespace webrtc_jni {
+
+extern "C" jint JNIEXPORT JNICALL JNI_OnLoad(JavaVM* jvm, void* reserved) {
+ jint ret = InitGlobalJniVariables(jvm);
+ RTC_DCHECK_GE(ret, 0);
+ if (ret < 0)
+ return -1;
+
+ RTC_CHECK(rtc::InitializeSSL()) << "Failed to InitializeSSL()";
+ LoadGlobalClassReferenceHolder();
+ unity_plugin::LoadGlobalClassReferenceHolder();
+
+ return ret;
+}
+
+extern "C" void JNIEXPORT JNICALL JNI_OnUnLoad(JavaVM* jvm, void* reserved) {
+ FreeGlobalClassReferenceHolder();
+ unity_plugin::FreeGlobalClassReferenceHolder();
+ RTC_CHECK(rtc::CleanupSSL()) << "Failed to CleanupSSL()";
+}
+
+} // namespace webrtc_jni
diff --git a/webrtc/examples/unityplugin/simple_peer_connection.cc b/webrtc/examples/unityplugin/simple_peer_connection.cc
index dd92f7b..57ab885 100644
--- a/webrtc/examples/unityplugin/simple_peer_connection.cc
+++ b/webrtc/examples/unityplugin/simple_peer_connection.cc
@@ -13,9 +13,16 @@
#include <utility>
#include "webrtc/api/test/fakeconstraints.h"
+#include "webrtc/api/videosourceproxy.h"
#include "webrtc/media/engine/webrtcvideocapturerfactory.h"
#include "webrtc/modules/video_capture/video_capture_factory.h"
+#if defined(WEBRTC_ANDROID)
+#include "webrtc/examples/unityplugin/classreferenceholder.h"
+#include "webrtc/sdk/android/src/jni/androidvideotracksource.h"
+#include "webrtc/sdk/android/src/jni/jni_helpers.h"
+#endif
+
// Names used for media stream labels.
const char kAudioLabel[] = "audio_label";
const char kVideoLabel[] = "video_label";
@@ -27,6 +34,12 @@
static std::unique_ptr<rtc::Thread> g_signaling_thread;
static rtc::scoped_refptr<webrtc::PeerConnectionFactoryInterface>
g_peer_connection_factory;
+#if defined(WEBRTC_ANDROID)
+// Android case: the video track does not own the capturer, and it
+// relies on the app to dispose the capturer when the peerconnection
+// shuts down.
+static jobject g_camera = nullptr;
+#endif
std::string GetEnvVarOrDefault(const char* env_var_name,
const char* default_value) {
@@ -149,6 +162,21 @@
void SimplePeerConnection::DeletePeerConnection() {
g_peer_count--;
+#if defined(WEBRTC_ANDROID)
+ if (g_camera) {
+ JNIEnv* env = webrtc_jni::GetEnv();
+ jclass pc_factory_class =
+ unity_plugin::FindClass(env, "org/webrtc/UnityUtility");
+ jmethodID stop_camera_method = webrtc_jni::GetStaticMethodID(
+ env, pc_factory_class, "StopCamera", "(Lorg/webrtc/VideoCapturer;)V");
+
+ env->CallStaticVoidMethod(pc_factory_class, stop_camera_method, g_camera);
+ CHECK_EXCEPTION(env);
+
+ g_camera = nullptr;
+ }
+#endif
+
CloseDataChannel();
peer_connection_ = nullptr;
active_streams_.clear();
@@ -380,6 +408,41 @@
stream->AddTrack(audio_track);
if (!audio_only) {
+#if defined(WEBRTC_ANDROID)
+ JNIEnv* env = webrtc_jni::GetEnv();
+ jclass pc_factory_class =
+ unity_plugin::FindClass(env, "org/webrtc/UnityUtility");
+ jmethodID load_texture_helper_method = webrtc_jni::GetStaticMethodID(
+ env, pc_factory_class, "LoadSurfaceTextureHelper",
+ "()Lorg/webrtc/SurfaceTextureHelper;");
+ jobject texture_helper = env->CallStaticObjectMethod(
+ pc_factory_class, load_texture_helper_method);
+ CHECK_EXCEPTION(env);
+ RTC_DCHECK(texture_helper != nullptr)
+ << "Cannot get the Surface Texture Helper.";
+
+ rtc::scoped_refptr<webrtc::AndroidVideoTrackSource> source(
+ new rtc::RefCountedObject<webrtc::AndroidVideoTrackSource>(
+ g_signaling_thread.get(), env, texture_helper, false));
+ rtc::scoped_refptr<webrtc::VideoTrackSourceProxy> proxy_source =
+ webrtc::VideoTrackSourceProxy::Create(g_signaling_thread.get(),
+ g_worker_thread.get(), source);
+
+ // link with VideoCapturer (Camera);
+ jmethodID link_camera_method = webrtc_jni::GetStaticMethodID(
+ env, pc_factory_class, "LinkCamera",
+ "(JLorg/webrtc/SurfaceTextureHelper;)Lorg/webrtc/VideoCapturer;");
+ jobject camera_tmp =
+ env->CallStaticObjectMethod(pc_factory_class, link_camera_method,
+ (jlong)proxy_source.get(), texture_helper);
+ CHECK_EXCEPTION(env);
+ g_camera = (jobject)env->NewGlobalRef(camera_tmp);
+
+ rtc::scoped_refptr<webrtc::VideoTrackInterface> video_track(
+ g_peer_connection_factory->CreateVideoTrack(kVideoLabel,
+ proxy_source.release()));
+ stream->AddTrack(video_track);
+#else
std::unique_ptr<cricket::VideoCapturer> capture = OpenVideoCaptureDevice();
if (capture) {
rtc::scoped_refptr<webrtc::VideoTrackInterface> video_track(
@@ -388,10 +451,11 @@
std::move(capture), nullptr)));
stream->AddTrack(video_track);
- if (local_video_observer_ && !stream->GetVideoTracks().empty()) {
- stream->GetVideoTracks()[0]->AddOrUpdateSink(
- local_video_observer_.get(), rtc::VideoSinkWants());
- }
+ }
+#endif
+ if (local_video_observer_ && !stream->GetVideoTracks().empty()) {
+ stream->GetVideoTracks()[0]->AddOrUpdateSink(local_video_observer_.get(),
+ rtc::VideoSinkWants());
}
}
diff --git a/webrtc/examples/unityplugin/unity_plugin_apis.h b/webrtc/examples/unityplugin/unity_plugin_apis.h
index ee7d709..529c7d2 100644
--- a/webrtc/examples/unityplugin/unity_plugin_apis.h
+++ b/webrtc/examples/unityplugin/unity_plugin_apis.h
@@ -37,7 +37,11 @@
int number_of_channels,
int number_of_frames);
+#if defined(WEBRTC_WIN)
#define WEBRTC_PLUGIN_API __declspec(dllexport)
+#elif defined(WEBRTC_ANDROID)
+#define WEBRTC_PLUGIN_API __attribute__((visibility("default")))
+#endif
extern "C" {
// Create a peerconnection and return a unique peer connection id.
WEBRTC_PLUGIN_API int CreatePeerConnection(const char** turn_urls,