Async audio processing API
API to injecting a heavy audio processing operation into WebRTC audio capture pipeline
Bug: webrtc:12003
Change-Id: I9f6f58f468bd84efd0a9d53d703db6229a03959e
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/165788
Reviewed-by: Karl Wiberg <kwiberg@webrtc.org>
Commit-Queue: Olga Sharonova <olka@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#32291}
diff --git a/api/audio/BUILD.gn b/api/audio/BUILD.gn
index 117e5cc..d0465bb 100644
--- a/api/audio/BUILD.gn
+++ b/api/audio/BUILD.gn
@@ -24,6 +24,11 @@
]
}
+rtc_source_set("audio_frame_processor") {
+ visibility = [ "*" ]
+ sources = [ "audio_frame_processor.h" ]
+}
+
rtc_source_set("audio_mixer_api") {
visibility = [ "*" ]
sources = [ "audio_mixer.h" ]
diff --git a/api/audio/audio_frame_processor.h b/api/audio/audio_frame_processor.h
new file mode 100644
index 0000000..bc21d14
--- /dev/null
+++ b/api/audio/audio_frame_processor.h
@@ -0,0 +1,43 @@
+/*
+ * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef API_AUDIO_AUDIO_FRAME_PROCESSOR_H_
+#define API_AUDIO_AUDIO_FRAME_PROCESSOR_H_
+
+#include <functional>
+#include <memory>
+
+namespace webrtc {
+
+class AudioFrame;
+
+// If passed into PeerConnectionFactory, will be used for additional
+// processing of captured audio frames, performed before encoding.
+// Implementations must be thread-safe.
+class AudioFrameProcessor {
+ public:
+ using OnAudioFrameCallback = std::function<void(std::unique_ptr<AudioFrame>)>;
+ virtual ~AudioFrameProcessor() = default;
+
+ // Processes the frame received from WebRTC, is called by WebRTC off the
+ // realtime audio capturing path. AudioFrameProcessor must reply with
+ // processed frames by calling |sink_callback| if it was provided in SetSink()
+ // call. |sink_callback| can be called in the context of Process().
+ virtual void Process(std::unique_ptr<AudioFrame> frame) = 0;
+
+ // Atomically replaces the current sink with the new one. Before the
+ // first call to this function, or if the provided |sink_callback| is nullptr,
+ // processed frames are simply discarded.
+ virtual void SetSink(OnAudioFrameCallback sink_callback) = 0;
+};
+
+} // namespace webrtc
+
+#endif // API_AUDIO_AUDIO_FRAME_PROCESSOR_H_
diff --git a/api/create_peerconnection_factory.cc b/api/create_peerconnection_factory.cc
index 6eba7d4..008fce3 100644
--- a/api/create_peerconnection_factory.cc
+++ b/api/create_peerconnection_factory.cc
@@ -37,7 +37,8 @@
std::unique_ptr<VideoEncoderFactory> video_encoder_factory,
std::unique_ptr<VideoDecoderFactory> video_decoder_factory,
rtc::scoped_refptr<AudioMixer> audio_mixer,
- rtc::scoped_refptr<AudioProcessing> audio_processing) {
+ rtc::scoped_refptr<AudioProcessing> audio_processing,
+ AudioFrameProcessor* audio_frame_processor) {
PeerConnectionFactoryDependencies dependencies;
dependencies.network_thread = network_thread;
dependencies.worker_thread = worker_thread;
@@ -53,6 +54,7 @@
media_dependencies.adm = std::move(default_adm);
media_dependencies.audio_encoder_factory = std::move(audio_encoder_factory);
media_dependencies.audio_decoder_factory = std::move(audio_decoder_factory);
+ media_dependencies.audio_frame_processor = audio_frame_processor;
if (audio_processing) {
media_dependencies.audio_processing = std::move(audio_processing);
} else {
diff --git a/api/create_peerconnection_factory.h b/api/create_peerconnection_factory.h
index ac50736..4eb0a00 100644
--- a/api/create_peerconnection_factory.h
+++ b/api/create_peerconnection_factory.h
@@ -31,6 +31,7 @@
namespace webrtc {
class AudioDeviceModule;
+class AudioFrameProcessor;
class AudioProcessing;
// Create a new instance of PeerConnectionFactoryInterface with optional video
@@ -47,7 +48,8 @@
std::unique_ptr<VideoEncoderFactory> video_encoder_factory,
std::unique_ptr<VideoDecoderFactory> video_decoder_factory,
rtc::scoped_refptr<AudioMixer> audio_mixer,
- rtc::scoped_refptr<AudioProcessing> audio_processing);
+ rtc::scoped_refptr<AudioProcessing> audio_processing,
+ AudioFrameProcessor* audio_frame_processor = nullptr);
} // namespace webrtc