Async audio processing API
API to injecting a heavy audio processing operation into WebRTC audio capture pipeline
Bug: webrtc:12003
Change-Id: I9f6f58f468bd84efd0a9d53d703db6229a03959e
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/165788
Reviewed-by: Karl Wiberg <kwiberg@webrtc.org>
Commit-Queue: Olga Sharonova <olka@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#32291}
diff --git a/modules/async_audio_processing/BUILD.gn b/modules/async_audio_processing/BUILD.gn
new file mode 100644
index 0000000..6a2a95e
--- /dev/null
+++ b/modules/async_audio_processing/BUILD.gn
@@ -0,0 +1,44 @@
+# Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
+#
+# Use of this source code is governed by a BSD-style license
+# that can be found in the LICENSE file in the root of the source
+# tree. An additional intellectual property rights grant can be found
+# in the file PATENTS. All contributing project authors may
+# be found in the AUTHORS file in the root of the source tree.
+
+import("../../webrtc.gni")
+
+rtc_library("async_audio_processing") {
+ sources = [
+ "async_audio_processing.cc",
+ "async_audio_processing.h",
+ ]
+
+ public = [ "async_audio_processing.h" ]
+
+ deps = [
+ "../../api:scoped_refptr",
+ "../../api/audio:audio_frame_api",
+ "../../api/audio:audio_frame_processor",
+ "../../api/task_queue:task_queue",
+ "../../rtc_base:checks",
+ "../../rtc_base:rtc_base_approved",
+ "../../rtc_base:rtc_task_queue",
+ "../../rtc_base/synchronization:sequence_checker",
+ ]
+}
+
+if (rtc_include_tests) {
+ rtc_library("async_audio_processing_test") {
+ testonly = true
+
+ sources = []
+
+ deps = [
+ ":async_audio_processing",
+ "../../api/audio:audio_frame_api",
+ "../../rtc_base:checks",
+ "../../rtc_base:rtc_base_approved",
+ ]
+ }
+}
diff --git a/modules/async_audio_processing/async_audio_processing.cc b/modules/async_audio_processing/async_audio_processing.cc
new file mode 100644
index 0000000..9452f3b
--- /dev/null
+++ b/modules/async_audio_processing/async_audio_processing.cc
@@ -0,0 +1,61 @@
+
+/*
+ * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+#include "modules/async_audio_processing/async_audio_processing.h"
+
+#include <utility>
+
+#include "api/audio/audio_frame.h"
+#include "api/task_queue/task_queue_factory.h"
+#include "rtc_base/checks.h"
+
+namespace webrtc {
+
+AsyncAudioProcessing::Factory::~Factory() = default;
+AsyncAudioProcessing::Factory::Factory(AudioFrameProcessor& frame_processor,
+ TaskQueueFactory& task_queue_factory)
+ : frame_processor_(frame_processor),
+ task_queue_factory_(task_queue_factory) {}
+
+std::unique_ptr<AsyncAudioProcessing>
+AsyncAudioProcessing::Factory::CreateAsyncAudioProcessing(
+ AudioFrameProcessor::OnAudioFrameCallback on_frame_processed_callback) {
+ return std::make_unique<AsyncAudioProcessing>(
+ frame_processor_, task_queue_factory_,
+ std::move(on_frame_processed_callback));
+}
+
+AsyncAudioProcessing::~AsyncAudioProcessing() {
+ frame_processor_.SetSink(nullptr);
+}
+
+AsyncAudioProcessing::AsyncAudioProcessing(
+ AudioFrameProcessor& frame_processor,
+ TaskQueueFactory& task_queue_factory,
+ AudioFrameProcessor::OnAudioFrameCallback on_frame_processed_callback)
+ : on_frame_processed_callback_(std::move(on_frame_processed_callback)),
+ frame_processor_(frame_processor),
+ task_queue_(task_queue_factory.CreateTaskQueue(
+ "AsyncAudioProcessing",
+ TaskQueueFactory::Priority::NORMAL)) {
+ frame_processor_.SetSink([this](std::unique_ptr<AudioFrame> frame) {
+ task_queue_.PostTask([this, frame = std::move(frame)]() mutable {
+ on_frame_processed_callback_(std::move(frame));
+ });
+ });
+}
+
+void AsyncAudioProcessing::Process(std::unique_ptr<AudioFrame> frame) {
+ task_queue_.PostTask([this, frame = std::move(frame)]() mutable {
+ frame_processor_.Process(std::move(frame));
+ });
+}
+
+} // namespace webrtc
diff --git a/modules/async_audio_processing/async_audio_processing.h b/modules/async_audio_processing/async_audio_processing.h
new file mode 100644
index 0000000..7e09d69
--- /dev/null
+++ b/modules/async_audio_processing/async_audio_processing.h
@@ -0,0 +1,76 @@
+/*
+ * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef MODULES_ASYNC_AUDIO_PROCESSING_ASYNC_AUDIO_PROCESSING_H_
+#define MODULES_ASYNC_AUDIO_PROCESSING_ASYNC_AUDIO_PROCESSING_H_
+
+#include <memory>
+
+#include "api/audio/audio_frame_processor.h"
+#include "rtc_base/ref_count.h"
+#include "rtc_base/task_queue.h"
+
+namespace webrtc {
+
+class AudioFrame;
+class TaskQueueFactory;
+
+// Helper class taking care of interactions with AudioFrameProcessor
+// in asynchronous manner. Offloads AudioFrameProcessor::Process calls
+// to a dedicated task queue. Makes sure that it's always safe for
+// AudioFrameProcessor to pass processed frames back to its sink.
+class AsyncAudioProcessing final {
+ public:
+ // Helper class passing AudioFrameProcessor and TaskQueueFactory into
+ // AsyncAudioProcessing constructor.
+ class Factory : public rtc::RefCountInterface {
+ public:
+ Factory(const Factory&) = delete;
+ Factory& operator=(const Factory&) = delete;
+
+ ~Factory();
+ Factory(AudioFrameProcessor& frame_processor,
+ TaskQueueFactory& task_queue_factory);
+
+ std::unique_ptr<AsyncAudioProcessing> CreateAsyncAudioProcessing(
+ AudioFrameProcessor::OnAudioFrameCallback on_frame_processed_callback);
+
+ private:
+ AudioFrameProcessor& frame_processor_;
+ TaskQueueFactory& task_queue_factory_;
+ };
+
+ AsyncAudioProcessing(const AsyncAudioProcessing&) = delete;
+ AsyncAudioProcessing& operator=(const AsyncAudioProcessing&) = delete;
+
+ ~AsyncAudioProcessing();
+
+ // Creates AsyncAudioProcessing which will pass audio frames to
+ // |frame_processor| on |task_queue_| and reply with processed frames passed
+ // into |on_frame_processed_callback|, which is posted back onto
+ // |task_queue_|. |task_queue_| is created using the provided
+ // |task_queue_factory|.
+ AsyncAudioProcessing(
+ AudioFrameProcessor& frame_processor,
+ TaskQueueFactory& task_queue_factory,
+ AudioFrameProcessor::OnAudioFrameCallback on_frame_processed_callback);
+
+ // Accepts |frame| for asynchronous processing. Thread-safe.
+ void Process(std::unique_ptr<AudioFrame> frame);
+
+ private:
+ AudioFrameProcessor::OnAudioFrameCallback on_frame_processed_callback_;
+ AudioFrameProcessor& frame_processor_;
+ rtc::TaskQueue task_queue_;
+};
+
+} // namespace webrtc
+
+#endif // MODULES_ASYNC_AUDIO_PROCESSING_ASYNC_AUDIO_PROCESSING_H_