Add Java support for AudioProcessing and PostProcessing injection
This allows injection of a user-defined post processing module from
the Android layer.
Bug: webrtc:8163
Change-Id: If3a6b4726c34c5f82d186b8cf95373c283cbd3f6
Reviewed-on: https://webrtc-review.googlesource.com/7610
Reviewed-by: Sami Kalliomäki <sakal@webrtc.org>
Reviewed-by: Taylor Brandstetter <deadbeef@webrtc.org>
Commit-Queue: Sam Zackrisson <saza@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#20367}diff --git a/sdk/android/BUILD.gn b/sdk/android/BUILD.gn
index 6753a55..d8781a6 100644
--- a/sdk/android/BUILD.gn
+++ b/sdk/android/BUILD.gn
@@ -67,12 +67,15 @@
rtc_static_library("audio_jni") {
sources = [
"src/jni/pc/audio_jni.cc",
+ "src/jni/pc/defaultaudioprocessingfactory_jni.cc",
]
deps = [
":base_jni",
"../../api/audio_codecs:builtin_audio_decoder_factory",
"../../api/audio_codecs:builtin_audio_encoder_factory",
+ "../../modules/audio_processing:audio_processing",
+ "../../rtc_base:rtc_base_approved",
"../../voice_engine:voice_engine",
]
}
@@ -374,6 +377,7 @@
java_files = [
"api/org/webrtc/AudioSource.java",
"api/org/webrtc/AudioTrack.java",
+ "api/org/webrtc/AudioProcessingFactory.java",
"api/org/webrtc/CallSessionFileRotatingLogSink.java",
"api/org/webrtc/Camera1Capturer.java",
"api/org/webrtc/Camera1Enumerator.java",
@@ -383,6 +387,7 @@
"api/org/webrtc/CameraEnumerator.java",
"api/org/webrtc/CameraVideoCapturer.java",
"api/org/webrtc/DataChannel.java",
+ "api/org/webrtc/DefaultAudioProcessingFactory.java",
"api/org/webrtc/DtmfSender.java",
"api/org/webrtc/EglBase.java",
"api/org/webrtc/EglRenderer.java",
@@ -407,6 +412,7 @@
"api/org/webrtc/NetworkMonitorAutoDetect.java",
"api/org/webrtc/PeerConnection.java",
"api/org/webrtc/PeerConnectionFactory.java",
+ "api/org/webrtc/PostProcessingFactory.java",
"api/org/webrtc/RendererCommon.java",
"api/org/webrtc/RTCStats.java",
"api/org/webrtc/RTCStatsCollectorCallback.java",
@@ -490,6 +496,7 @@
android_manifest = "instrumentationtests/AndroidManifest.xml"
java_files = [
+ "instrumentationtests/src/org/webrtc/DefaultAudioProcessingFactoryTest.java",
"instrumentationtests/src/org/webrtc/Camera1CapturerUsingByteBufferTest.java",
"instrumentationtests/src/org/webrtc/Camera1CapturerUsingTextureTest.java",
"instrumentationtests/src/org/webrtc/Camera2CapturerTest.java",
diff --git a/sdk/android/api/org/webrtc/AudioProcessingFactory.java b/sdk/android/api/org/webrtc/AudioProcessingFactory.java
new file mode 100644
index 0000000..bd8fdb8
--- /dev/null
+++ b/sdk/android/api/org/webrtc/AudioProcessingFactory.java
@@ -0,0 +1,20 @@
+/*
+ * Copyright 2017 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+package org.webrtc;
+
+/** Factory for creating webrtc::AudioProcessing instances. */
+public interface AudioProcessingFactory {
+ /**
+ * Dynamically allocates a webrtc::AudioProcessing instance and returns a pointer to it.
+ * The caller takes ownership of the object.
+ */
+ public long createNative();
+}
diff --git a/sdk/android/api/org/webrtc/DefaultAudioProcessingFactory.java b/sdk/android/api/org/webrtc/DefaultAudioProcessingFactory.java
new file mode 100644
index 0000000..6fcfe23
--- /dev/null
+++ b/sdk/android/api/org/webrtc/DefaultAudioProcessingFactory.java
@@ -0,0 +1,47 @@
+/*
+ * Copyright 2017 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+package org.webrtc;
+
+/** Factory for instantiating the default webrtc::AudioProcessing implementation. */
+public class DefaultAudioProcessingFactory implements AudioProcessingFactory {
+ public DefaultAudioProcessingFactory() {
+ this(null /* postProcessingFactory */);
+ }
+
+ /**
+ * Allows injecting a PostProcessingFactory. A null PostProcessingFactory creates a
+ * webrtc::AudioProcessing with nullptr webrtc::postProcessing.
+ */
+ public DefaultAudioProcessingFactory(PostProcessingFactory postProcessingFactory) {
+ this.postProcessingFactory = postProcessingFactory;
+ }
+
+ /**
+ * Creates a default webrtc::AudioProcessing module, which takes ownership of objects created by
+ * its factories.
+ */
+ @Override
+ public long createNative() {
+ long nativePostProcessor = 0;
+ if (postProcessingFactory != null) {
+ nativePostProcessor = postProcessingFactory.createNative();
+ if (nativePostProcessor == 0) {
+ throw new NullPointerException(
+ "PostProcessingFactory.createNative() may not return 0 (nullptr).");
+ }
+ }
+ return nativeCreateAudioProcessing(nativePostProcessor);
+ }
+
+ private PostProcessingFactory postProcessingFactory;
+
+ private static native long nativeCreateAudioProcessing(long nativePostProcessor);
+}
diff --git a/sdk/android/api/org/webrtc/PeerConnectionFactory.java b/sdk/android/api/org/webrtc/PeerConnectionFactory.java
index 14c367f..d4b158d 100644
--- a/sdk/android/api/org/webrtc/PeerConnectionFactory.java
+++ b/sdk/android/api/org/webrtc/PeerConnectionFactory.java
@@ -125,6 +125,14 @@
}
}
+ private void checkInitializeHasBeenCalled() {
+ if (!NativeLibrary.isLoaded() || ContextUtils.getApplicationContext() == null) {
+ throw new IllegalStateException(
+ "PeerConnectionFactory.initialize was not called before creating a "
+ + "PeerConnectionFactory.");
+ }
+ }
+
// Must be called at least once before creating a PeerConnectionFactory
// (for example, at application startup time).
private static native void nativeInitializeAndroidGlobals(
@@ -185,17 +193,27 @@
public PeerConnectionFactory(
Options options, VideoEncoderFactory encoderFactory, VideoDecoderFactory decoderFactory) {
- if (!NativeLibrary.isLoaded() || ContextUtils.getApplicationContext() == null) {
- throw new IllegalStateException(
- "PeerConnectionFactory.initialize was not called before creating a "
- + "PeerConnectionFactory.");
- }
+ checkInitializeHasBeenCalled();
nativeFactory = nativeCreatePeerConnectionFactory(options, encoderFactory, decoderFactory);
if (nativeFactory == 0) {
throw new RuntimeException("Failed to initialize PeerConnectionFactory!");
}
}
+ public PeerConnectionFactory(Options options, VideoEncoderFactory encoderFactory,
+ VideoDecoderFactory decoderFactory, AudioProcessingFactory audioProcessingFactory) {
+ checkInitializeHasBeenCalled();
+ if (audioProcessingFactory == null) {
+ throw new NullPointerException(
+ "PeerConnectionFactory constructor does not accept a null AudioProcessingFactory.");
+ }
+ nativeFactory = nativeCreatePeerConnectionFactoryWithAudioProcessing(
+ options, encoderFactory, decoderFactory, audioProcessingFactory.createNative());
+ if (nativeFactory == 0) {
+ throw new RuntimeException("Failed to initialize PeerConnectionFactory!");
+ }
+ }
+
public PeerConnection createPeerConnection(PeerConnection.RTCConfiguration rtcConfig,
MediaConstraints constraints, PeerConnection.Observer observer) {
long nativeObserver = nativeCreateObserver(observer);
@@ -337,6 +355,10 @@
private static native long nativeCreatePeerConnectionFactory(
Options options, VideoEncoderFactory encoderFactory, VideoDecoderFactory decoderFactory);
+ private static native long nativeCreatePeerConnectionFactoryWithAudioProcessing(Options options,
+ VideoEncoderFactory encoderFactory, VideoDecoderFactory decoderFactory,
+ long nativeAudioProcessor);
+
private static native long nativeCreateObserver(PeerConnection.Observer observer);
private static native long nativeCreatePeerConnection(long nativeFactory,
diff --git a/sdk/android/api/org/webrtc/PostProcessingFactory.java b/sdk/android/api/org/webrtc/PostProcessingFactory.java
new file mode 100644
index 0000000..11b8fdb
--- /dev/null
+++ b/sdk/android/api/org/webrtc/PostProcessingFactory.java
@@ -0,0 +1,20 @@
+/*
+ * Copyright 2017 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+package org.webrtc;
+
+/** Factory for creating webrtc::PostProcessing instances. */
+public interface PostProcessingFactory {
+ /**
+ * Dynamically allocates a webrtc::PostProcessing instance and returns a pointer to it.
+ * The caller takes ownership of the object.
+ */
+ public long createNative();
+}
diff --git a/sdk/android/instrumentationtests/src/org/webrtc/DefaultAudioProcessingFactoryTest.java b/sdk/android/instrumentationtests/src/org/webrtc/DefaultAudioProcessingFactoryTest.java
new file mode 100644
index 0000000..0ed8c35
--- /dev/null
+++ b/sdk/android/instrumentationtests/src/org/webrtc/DefaultAudioProcessingFactoryTest.java
@@ -0,0 +1,47 @@
+/*
+ * Copyright 2017 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+package org.webrtc;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+import android.support.test.InstrumentationRegistry;
+import android.support.test.filters.MediumTest;
+import android.support.test.filters.SmallTest;
+import org.chromium.base.test.BaseJUnit4ClassRunner;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+
+/** Unit tests for {@link DefaultAudioProcessingFactory}. */
+@RunWith(BaseJUnit4ClassRunner.class)
+public final class DefaultAudioProcessingFactoryTest {
+ @Before
+ public void setUp() {
+ PeerConnectionFactory.initialize(
+ PeerConnectionFactory.InitializationOptions.builder(InstrumentationRegistry.getContext())
+ .createInitializationOptions());
+ }
+
+ /**
+ * Tests that a PeerConnectionFactory can be initialized with an AudioProcessingFactory without
+ * crashing.
+ */
+ @Test
+ @MediumTest
+ public void testInitializePeerConnectionFactory() {
+ AudioProcessingFactory audioProcessingFactory = new DefaultAudioProcessingFactory();
+ PeerConnectionFactory peerConnectionFactory = new PeerConnectionFactory(null /* options */,
+ null /* encoderFactory */, null /* decoderFactory */, audioProcessingFactory);
+ peerConnectionFactory.dispose();
+ }
+}
diff --git a/sdk/android/src/jni/pc/audio_jni.cc b/sdk/android/src/jni/pc/audio_jni.cc
index cc73ea2..7dde025 100644
--- a/sdk/android/src/jni/pc/audio_jni.cc
+++ b/sdk/android/src/jni/pc/audio_jni.cc
@@ -12,6 +12,7 @@
#include "api/audio_codecs/builtin_audio_decoder_factory.h"
#include "api/audio_codecs/builtin_audio_encoder_factory.h"
+#include "modules/audio_processing/include/audio_processing.h"
namespace webrtc {
namespace jni {
@@ -24,5 +25,9 @@
return CreateBuiltinAudioEncoderFactory();
}
+rtc::scoped_refptr<AudioProcessing> CreateAudioProcessing() {
+ return AudioProcessing::Create();
+}
+
} // namespace jni
} // namespace webrtc
diff --git a/sdk/android/src/jni/pc/audio_jni.h b/sdk/android/src/jni/pc/audio_jni.h
index 06d2c3d..d6998b8 100644
--- a/sdk/android/src/jni/pc/audio_jni.h
+++ b/sdk/android/src/jni/pc/audio_jni.h
@@ -15,6 +15,7 @@
// We don't want this target depend on audio related targets
#include "api/audio_codecs/audio_decoder_factory.h" // nogncheck
#include "api/audio_codecs/audio_encoder_factory.h" // nogncheck
+#include "modules/audio_processing/include/audio_processing.h" // nogncheck
#include "rtc_base/scoped_ref_ptr.h"
namespace webrtc {
@@ -24,6 +25,8 @@
rtc::scoped_refptr<AudioEncoderFactory> CreateAudioEncoderFactory();
+rtc::scoped_refptr<AudioProcessing> CreateAudioProcessing();
+
} // namespace jni
} // namespace webrtc
diff --git a/sdk/android/src/jni/pc/defaultaudioprocessingfactory_jni.cc b/sdk/android/src/jni/pc/defaultaudioprocessingfactory_jni.cc
new file mode 100644
index 0000000..a0d2c03
--- /dev/null
+++ b/sdk/android/src/jni/pc/defaultaudioprocessingfactory_jni.cc
@@ -0,0 +1,36 @@
+/*
+ * Copyright 2017 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include <memory>
+
+#include "modules/audio_processing/include/audio_processing.h"
+#include "rtc_base/scoped_ref_ptr.h"
+#include "sdk/android/src/jni/jni_helpers.h"
+
+namespace webrtc {
+namespace jni {
+
+JNI_FUNCTION_DECLARATION(
+ jlong,
+ DefaultAudioProcessingFactory_nativeCreateAudioProcessing,
+ JNIEnv*,
+ jclass,
+ jlong native_post_processor) {
+ std::unique_ptr<PostProcessing> post_processor(
+ reinterpret_cast<PostProcessing*>(native_post_processor));
+ rtc::scoped_refptr<AudioProcessing> audio_processing =
+ AudioProcessing::Create(webrtc::Config(), std::move(post_processor),
+ nullptr /* echo_control_factory */,
+ nullptr /* beamformer */);
+ return jlongFromPointer(audio_processing.release());
+}
+
+} // namespace jni
+} // namespace webrtc
diff --git a/sdk/android/src/jni/pc/media_jni.cc b/sdk/android/src/jni/pc/media_jni.cc
index ec8ab49..32b9020 100644
--- a/sdk/android/src/jni/pc/media_jni.cc
+++ b/sdk/android/src/jni/pc/media_jni.cc
@@ -31,10 +31,11 @@
const rtc::scoped_refptr<AudioDecoderFactory>& audio_decoder_factory,
cricket::WebRtcVideoEncoderFactory* video_encoder_factory,
cricket::WebRtcVideoDecoderFactory* video_decoder_factory,
- rtc::scoped_refptr<AudioMixer> audio_mixer) {
+ rtc::scoped_refptr<AudioMixer> audio_mixer,
+ rtc::scoped_refptr<AudioProcessing> audio_processor) {
return cricket::WebRtcMediaEngineFactory::Create(
adm, audio_encoder_factory, audio_decoder_factory, video_encoder_factory,
- video_decoder_factory, audio_mixer, AudioProcessing::Create());
+ video_decoder_factory, audio_mixer, audio_processor);
}
} // namespace jni
diff --git a/sdk/android/src/jni/pc/media_jni.h b/sdk/android/src/jni/pc/media_jni.h
index 64afac4..13a8bb5 100644
--- a/sdk/android/src/jni/pc/media_jni.h
+++ b/sdk/android/src/jni/pc/media_jni.h
@@ -20,6 +20,7 @@
class AudioDecoderFactory;
class RtcEventLogFactoryInterface;
class AudioMixer;
+class AudioProcessing;
} // namespace webrtc
namespace cricket {
@@ -40,7 +41,8 @@
const rtc::scoped_refptr<AudioDecoderFactory>& audio_decoder_factory,
cricket::WebRtcVideoEncoderFactory* video_encoder_factory,
cricket::WebRtcVideoDecoderFactory* video_decoder_factory,
- rtc::scoped_refptr<AudioMixer> audio_mixer);
+ rtc::scoped_refptr<AudioMixer> audio_mixer,
+ rtc::scoped_refptr<AudioProcessing> audio_processor);
} // namespace jni
} // namespace webrtc
diff --git a/sdk/android/src/jni/pc/null_audio_jni.cc b/sdk/android/src/jni/pc/null_audio_jni.cc
index 7c508f5..0482a68 100644
--- a/sdk/android/src/jni/pc/null_audio_jni.cc
+++ b/sdk/android/src/jni/pc/null_audio_jni.cc
@@ -21,5 +21,9 @@
return nullptr;
}
+rtc::scoped_refptr<AudioProcessing> CreateAudioProcessing() {
+ return nullptr;
+}
+
} // namespace jni
} // namespace webrtc
diff --git a/sdk/android/src/jni/pc/null_media_jni.cc b/sdk/android/src/jni/pc/null_media_jni.cc
index 1c44db0..8a39028 100644
--- a/sdk/android/src/jni/pc/null_media_jni.cc
+++ b/sdk/android/src/jni/pc/null_media_jni.cc
@@ -27,7 +27,8 @@
const rtc::scoped_refptr<AudioDecoderFactory>& audio_decoder_factory,
cricket::WebRtcVideoEncoderFactory* video_encoder_factory,
cricket::WebRtcVideoDecoderFactory* video_decoder_factory,
- rtc::scoped_refptr<AudioMixer> audio_mixer) {
+ rtc::scoped_refptr<AudioMixer> audio_mixer,
+ rtc::scoped_refptr<AudioProcessing> audio_processor) {
return nullptr;
}
diff --git a/sdk/android/src/jni/pc/peerconnectionfactory_jni.cc b/sdk/android/src/jni/pc/peerconnectionfactory_jni.cc
index 466bf84..78a1630 100644
--- a/sdk/android/src/jni/pc/peerconnectionfactory_jni.cc
+++ b/sdk/android/src/jni/pc/peerconnectionfactory_jni.cc
@@ -14,6 +14,9 @@
#include "api/peerconnectioninterface.h"
#include "media/base/mediaengine.h"
#include "modules/utility/include/jvm_android.h"
+// We don't depend on the audio processing module implementation.
+// The user may pass in a nullptr.
+#include "modules/audio_processing/include/audio_processing.h" // nogncheck
#include "rtc_base/event_tracer.h"
#include "rtc_base/stringutils.h"
#include "rtc_base/thread.h"
@@ -132,14 +135,12 @@
rtc::tracing::ShutdownInternalTracer();
}
-JNI_FUNCTION_DECLARATION(
- jlong,
- PeerConnectionFactory_nativeCreatePeerConnectionFactory,
+jlong CreatePeerConnectionFactoryForJava(
JNIEnv* jni,
- jclass,
jobject joptions,
jobject jencoder_factory,
- jobject jdecoder_factory) {
+ jobject jdecoder_factory,
+ rtc::scoped_refptr<AudioProcessing> audio_processor) {
// talk/ assumes pretty widely that the current Thread is ThreadManager'd, but
// ThreadManager only WrapCurrentThread()s the thread where it is first
// created. Since the semantics around when auto-wrapping happens in
@@ -190,7 +191,7 @@
CreateRtcEventLogFactory());
std::unique_ptr<cricket::MediaEngineInterface> media_engine(CreateMediaEngine(
adm, audio_encoder_factory, audio_decoder_factory, video_encoder_factory,
- video_decoder_factory, audio_mixer));
+ video_decoder_factory, audio_mixer, audio_processor));
rtc::scoped_refptr<PeerConnectionFactoryInterface> factory(
CreateModularPeerConnectionFactory(
@@ -212,6 +213,35 @@
return jlongFromPointer(owned_factory);
}
+JNI_FUNCTION_DECLARATION(
+ jlong,
+ PeerConnectionFactory_nativeCreatePeerConnectionFactory,
+ JNIEnv* jni,
+ jclass,
+ jobject joptions,
+ jobject jencoder_factory,
+ jobject jdecoder_factory) {
+ return CreatePeerConnectionFactoryForJava(jni, joptions, jencoder_factory,
+ jdecoder_factory,
+ CreateAudioProcessing());
+}
+
+JNI_FUNCTION_DECLARATION(
+ jlong,
+ PeerConnectionFactory_nativeCreatePeerConnectionFactoryWithAudioProcessing,
+ JNIEnv* jni,
+ jclass,
+ jobject joptions,
+ jobject jencoder_factory,
+ jobject jdecoder_factory,
+ jlong native_audio_processor) {
+ rtc::scoped_refptr<AudioProcessing> audio_processor =
+ reinterpret_cast<AudioProcessing*>(native_audio_processor);
+ RTC_DCHECK(audio_processor);
+ return CreatePeerConnectionFactoryForJava(jni, joptions, jencoder_factory,
+ jdecoder_factory, audio_processor);
+}
+
JNI_FUNCTION_DECLARATION(void,
PeerConnectionFactory_nativeFreeFactory,
JNIEnv*,