blob: a5353104063dd887290b39fa36af29bb018dd7e6 [file] [log] [blame]
niklase@google.com470e71d2011-07-07 08:21:25 +00001/*
andrew@webrtc.org6f9f8172012-03-06 19:03:39 +00002 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
niklase@google.com470e71d2011-07-07 08:21:25 +00003 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020011#ifndef MODULES_AUDIO_PROCESSING_AUDIO_PROCESSING_IMPL_H_
12#define MODULES_AUDIO_PROCESSING_AUDIO_PROCESSING_IMPL_H_
niklase@google.com470e71d2011-07-07 08:21:25 +000013
Per Åhgren09e9a832020-05-11 11:03:47 +020014#include <stdio.h>
15
Niels Möller7a669002022-06-27 09:47:02 +020016#include <atomic>
niklase@google.com470e71d2011-07-07 08:21:25 +000017#include <list>
kwiberg88788ad2016-02-19 07:04:49 -080018#include <memory>
Per Åhgren09e9a832020-05-11 11:03:47 +020019#include <string>
Michael Graczyk86c6d332015-07-23 11:41:39 -070020#include <vector>
niklase@google.com470e71d2011-07-07 08:21:25 +000021
Ali Tofigh1fa87c42022-07-25 22:07:08 +020022#include "absl/strings/string_view.h"
Alessio Bazzicabab12852022-02-03 16:30:25 +010023#include "api/array_view.h"
Artem Titov741daaf2019-03-21 14:37:36 +010024#include "api/function_view.h"
saza1d600522019-10-18 13:29:43 +020025#include "modules/audio_processing/aec3/echo_canceller3.h"
26#include "modules/audio_processing/agc/agc_manager_direct.h"
Hanna Silen529131d2021-10-29 14:55:45 +020027#include "modules/audio_processing/agc/analog_gain_stats_reporter.h"
Sam Zackrisson41478c72019-10-15 10:10:26 +020028#include "modules/audio_processing/agc/gain_control.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020029#include "modules/audio_processing/audio_buffer.h"
Per Åhgrendb5d7282021-03-15 16:31:04 +000030#include "modules/audio_processing/capture_levels_adjuster/capture_levels_adjuster.h"
saza1d600522019-10-18 13:29:43 +020031#include "modules/audio_processing/echo_control_mobile_impl.h"
saza1d600522019-10-18 13:29:43 +020032#include "modules/audio_processing/gain_control_impl.h"
33#include "modules/audio_processing/gain_controller2.h"
34#include "modules/audio_processing/high_pass_filter.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020035#include "modules/audio_processing/include/aec_dump.h"
Per Åhgren71652f42020-03-17 13:23:58 +010036#include "modules/audio_processing/include/audio_frame_proxies.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020037#include "modules/audio_processing/include/audio_processing.h"
Sam Zackrissonb24c00f2018-11-26 16:18:25 +010038#include "modules/audio_processing/include/audio_processing_statistics.h"
Per Åhgren0cbb58e2019-10-29 22:59:44 +010039#include "modules/audio_processing/ns/noise_suppressor.h"
Sam Zackrissonb37e59d2020-04-27 08:39:33 +020040#include "modules/audio_processing/optionally_built_submodule_creators.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020041#include "modules/audio_processing/render_queue_item_verifier.h"
42#include "modules/audio_processing/rms_level.h"
saza1d600522019-10-18 13:29:43 +020043#include "modules/audio_processing/transient/transient_suppressor.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020044#include "rtc_base/gtest_prod_util.h"
45#include "rtc_base/ignore_wundef.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020046#include "rtc_base/swap_queue.h"
Markus Handell0df0fae2020-07-07 15:53:34 +020047#include "rtc_base/synchronization/mutex.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020048#include "rtc_base/thread_annotations.h"
peahdf3efa82015-11-28 12:35:15 -080049
niklase@google.com470e71d2011-07-07 08:21:25 +000050namespace webrtc {
andrew@webrtc.orgddbb8a22014-04-22 21:00:04 +000051
Per Åhgren13735822018-02-12 21:42:56 +010052class ApmDataDumper;
ekmeyerson60d9b332015-08-14 10:35:55 -070053class AudioConverter;
Michael Graczykdfa36052015-03-25 16:37:27 -070054
Henrik Lundin8a9aa552022-02-08 09:15:12 +000055constexpr int RuntimeSettingQueueSize() {
56 return 100;
57}
58
niklase@google.com470e71d2011-07-07 08:21:25 +000059class AudioProcessingImpl : public AudioProcessing {
60 public:
peahdf3efa82015-11-28 12:35:15 -080061 // Methods forcing APM to run in a single-threaded manner.
62 // Acquires both the render and capture locks.
Alessio Bazzicabe1b8982021-09-17 08:26:10 +020063 AudioProcessingImpl();
Alessio Bazzica20a9ac62021-10-14 10:55:08 +020064 AudioProcessingImpl(const AudioProcessing::Config& config,
65 std::unique_ptr<CustomProcessing> capture_post_processor,
Alex Loiko5825aa62017-12-18 16:02:40 +010066 std::unique_ptr<CustomProcessing> render_pre_processor,
Gustaf Ullberg002ef282017-10-12 15:13:17 +020067 std::unique_ptr<EchoControlFactory> echo_control_factory,
Valeriia Nemychnikovaf06eb572018-08-29 10:37:09 +020068 rtc::scoped_refptr<EchoDetector> echo_detector,
69 std::unique_ptr<CustomAudioAnalyzer> capture_analyzer);
kwiberg83ffe452016-08-29 14:46:07 -070070 ~AudioProcessingImpl() override;
kjellander@webrtc.org14665ff2015-03-04 12:58:35 +000071 int Initialize() override;
Michael Graczyk86c6d332015-07-23 11:41:39 -070072 int Initialize(const ProcessingConfig& processing_config) override;
peah88ac8532016-09-12 16:47:25 -070073 void ApplyConfig(const AudioProcessing::Config& config) override;
Ali Tofigh1fa87c42022-07-25 22:07:08 +020074 bool CreateAndAttachAecDump(absl::string_view file_name,
75 int64_t max_log_size_bytes,
76 rtc::TaskQueue* worker_queue) override;
Per Åhgren09e9a832020-05-11 11:03:47 +020077 bool CreateAndAttachAecDump(FILE* handle,
78 int64_t max_log_size_bytes,
79 rtc::TaskQueue* worker_queue) override;
80 // TODO(webrtc:5298) Deprecated variant.
aleloi868f32f2017-05-23 07:20:05 -070081 void AttachAecDump(std::unique_ptr<AecDump> aec_dump) override;
82 void DetachAecDump() override;
Alessio Bazzicac054e782018-04-16 12:10:09 +020083 void SetRuntimeSetting(RuntimeSetting setting) override;
Per Åhgren0a144a72021-02-09 08:47:51 +010084 bool PostRuntimeSetting(RuntimeSetting setting) override;
Alessio Bazzicac054e782018-04-16 12:10:09 +020085
peahdf3efa82015-11-28 12:35:15 -080086 // Capture-side exclusive methods possibly running APM in a
87 // multi-threaded manner. Acquire the capture lock.
Per Åhgren645f24c2020-03-16 12:06:02 +010088 int ProcessStream(const int16_t* const src,
89 const StreamConfig& input_config,
90 const StreamConfig& output_config,
Per Åhgrendc5522b2020-03-19 14:55:58 +010091 int16_t* const dest) override;
kjellander@webrtc.org14665ff2015-03-04 12:58:35 +000092 int ProcessStream(const float* const* src,
Michael Graczyk86c6d332015-07-23 11:41:39 -070093 const StreamConfig& input_config,
94 const StreamConfig& output_config,
95 float* const* dest) override;
Per Åhgrenc20a19c2019-11-13 11:12:29 +010096 bool GetLinearAecOutput(
97 rtc::ArrayView<std::array<float, 160>> linear_output) const override;
peahdf3efa82015-11-28 12:35:15 -080098 void set_output_will_be_muted(bool muted) override;
Per Åhgren0a144a72021-02-09 08:47:51 +010099 void HandleCaptureOutputUsedSetting(bool capture_output_used)
100 RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_capture_);
peahdf3efa82015-11-28 12:35:15 -0800101 int set_stream_delay_ms(int delay) override;
peahdf3efa82015-11-28 12:35:15 -0800102 void set_stream_key_pressed(bool key_pressed) override;
Sam Zackrissonf0d1c032019-03-27 13:28:08 +0100103 void set_stream_analog_level(int level) override;
Markus Handell02ba1d22020-05-14 14:31:18 +0200104 int recommended_stream_analog_level() const
Markus Handell0df0fae2020-07-07 15:53:34 +0200105 RTC_LOCKS_EXCLUDED(mutex_capture_) override;
peahdf3efa82015-11-28 12:35:15 -0800106
107 // Render-side exclusive methods possibly running APM in a
108 // multi-threaded manner. Acquire the render lock.
Per Åhgren645f24c2020-03-16 12:06:02 +0100109 int ProcessReverseStream(const int16_t* const src,
110 const StreamConfig& input_config,
111 const StreamConfig& output_config,
112 int16_t* const dest) override;
kjellander@webrtc.org14665ff2015-03-04 12:58:35 +0000113 int AnalyzeReverseStream(const float* const* data,
Gustaf Ullberg8c51f2e2019-10-22 15:21:31 +0200114 const StreamConfig& reverse_config) override;
ekmeyerson60d9b332015-08-14 10:35:55 -0700115 int ProcessReverseStream(const float* const* src,
peahde65ddc2016-09-16 15:02:15 -0700116 const StreamConfig& input_config,
117 const StreamConfig& output_config,
ekmeyerson60d9b332015-08-14 10:35:55 -0700118 float* const* dest) override;
peahdf3efa82015-11-28 12:35:15 -0800119
120 // Methods only accessed from APM submodules or
121 // from AudioProcessing tests in a single-threaded manner.
122 // Hence there is no need for locks in these.
123 int proc_sample_rate_hz() const override;
124 int proc_split_sample_rate_hz() const override;
Peter Kasting69558702016-01-12 16:26:35 -0800125 size_t num_input_channels() const override;
126 size_t num_proc_channels() const override;
127 size_t num_output_channels() const override;
128 size_t num_reverse_channels() const override;
kjellander@webrtc.org14665ff2015-03-04 12:58:35 +0000129 int stream_delay_ms() const override;
peahdf3efa82015-11-28 12:35:15 -0800130
Per Åhgrencf4c8722019-12-30 14:32:14 +0100131 AudioProcessingStats GetStatistics(bool has_remote_tracks) override {
132 return GetStatistics();
133 }
134 AudioProcessingStats GetStatistics() override {
135 return stats_reporter_.GetStatistics();
136 }
ivoc3e9a5372016-10-28 07:55:33 -0700137
henrik.lundinadf06352017-04-05 05:48:24 -0700138 AudioProcessing::Config GetConfig() const override;
peah8271d042016-11-22 07:24:52 -0800139
andrew@webrtc.org60730cf2014-01-07 17:45:09 +0000140 protected:
andrew@webrtc.orga8b97372014-03-10 22:26:12 +0000141 // Overridden in a mock.
Per Åhgren0ade9832020-09-01 23:57:20 +0200142 virtual void InitializeLocked()
Markus Handell0df0fae2020-07-07 15:53:34 +0200143 RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_render_, mutex_capture_);
Niels Möller5b747232021-07-26 17:16:25 +0200144 void AssertLockedForTest()
145 RTC_ASSERT_EXCLUSIVE_LOCK(mutex_render_, mutex_capture_) {
146 mutex_render_.AssertHeld();
147 mutex_capture_.AssertHeld();
148 }
andrew@webrtc.org60730cf2014-01-07 17:45:09 +0000149
niklase@google.com470e71d2011-07-07 08:21:25 +0000150 private:
peahc19f3122016-10-07 14:54:10 -0700151 // TODO(peah): These friend classes should be removed as soon as the new
152 // parameter setting scheme allows.
153 FRIEND_TEST_ALL_PREFIXES(ApmConfiguration, DefaultBehavior);
154 FRIEND_TEST_ALL_PREFIXES(ApmConfiguration, ValidConfigBehavior);
155 FRIEND_TEST_ALL_PREFIXES(ApmConfiguration, InValidConfigBehavior);
Sam Zackrissonb37e59d2020-04-27 08:39:33 +0200156 FRIEND_TEST_ALL_PREFIXES(ApmWithSubmodulesExcludedTest,
157 ToggleTransientSuppressor);
158 FRIEND_TEST_ALL_PREFIXES(ApmWithSubmodulesExcludedTest,
159 ReinitializeTransientSuppressor);
160 FRIEND_TEST_ALL_PREFIXES(ApmWithSubmodulesExcludedTest,
161 BitexactWithDisabledModules);
162
Alessio Bazzica0c0c6022022-09-07 15:15:52 +0200163 void set_stream_analog_level_locked(int level)
164 RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_capture_);
Markus Handell02ba1d22020-05-14 14:31:18 +0200165 int recommended_stream_analog_level_locked() const
Markus Handell0df0fae2020-07-07 15:53:34 +0200166 RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_capture_);
Markus Handell02ba1d22020-05-14 14:31:18 +0200167
Sam Zackrissonb37e59d2020-04-27 08:39:33 +0200168 void OverrideSubmoduleCreationForTesting(
169 const ApmSubmoduleCreationOverrides& overrides);
Alessio Bazzicac054e782018-04-16 12:10:09 +0200170
171 // Class providing thread-safe message pipe functionality for
Artem Titov0b489302021-07-28 20:50:03 +0200172 // `runtime_settings_`.
Alessio Bazzicac054e782018-04-16 12:10:09 +0200173 class RuntimeSettingEnqueuer {
174 public:
175 explicit RuntimeSettingEnqueuer(
176 SwapQueue<RuntimeSetting>* runtime_settings);
177 ~RuntimeSettingEnqueuer();
Per Åhgren0a144a72021-02-09 08:47:51 +0100178
179 // Enqueue setting and return whether the setting was successfully enqueued.
180 bool Enqueue(RuntimeSetting setting);
Alessio Bazzicac054e782018-04-16 12:10:09 +0200181
182 private:
Alessio Bazzica33444dc2018-04-20 13:16:55 +0200183 SwapQueue<RuntimeSetting>& runtime_settings_;
Alex Loiko73ec0192018-05-15 10:52:28 +0200184 };
Alex Loiko73ec0192018-05-15 10:52:28 +0200185
Sam Zackrisson03cb7e52021-12-06 15:40:04 +0100186 const std::unique_ptr<ApmDataDumper> data_dumper_;
Niels Möller7a669002022-06-27 09:47:02 +0200187 static std::atomic<int> instance_count_;
Per Åhgrenb2b58d82019-12-02 14:59:40 +0100188 const bool use_setup_specific_default_aec3_config_;
Alex Loiko73ec0192018-05-15 10:52:28 +0200189
Alessio Bazzica0441bb62021-08-10 15:23:23 +0200190 const bool use_denormal_disabler_;
191
Alessio Bazzicaefbe3af2022-03-18 12:39:00 +0100192 const TransientSuppressor::VadMode transient_suppressor_vad_mode_;
193
Alex Loiko73ec0192018-05-15 10:52:28 +0200194 SwapQueue<RuntimeSetting> capture_runtime_settings_;
195 SwapQueue<RuntimeSetting> render_runtime_settings_;
196
197 RuntimeSettingEnqueuer capture_runtime_settings_enqueuer_;
198 RuntimeSettingEnqueuer render_runtime_settings_enqueuer_;
Alessio Bazzicac054e782018-04-16 12:10:09 +0200199
Gustaf Ullberg002ef282017-10-12 15:13:17 +0200200 // EchoControl factory.
Sam Zackrisson03cb7e52021-12-06 15:40:04 +0100201 const std::unique_ptr<EchoControlFactory> echo_control_factory_;
Gustaf Ullbergd8579e02017-10-11 16:29:02 +0200202
saza1d600522019-10-18 13:29:43 +0200203 class SubmoduleStates {
peah2ace3f92016-09-10 04:42:27 -0700204 public:
saza1d600522019-10-18 13:29:43 +0200205 SubmoduleStates(bool capture_post_processor_enabled,
206 bool render_pre_processor_enabled,
207 bool capture_analyzer_enabled);
peah2ace3f92016-09-10 04:42:27 -0700208 // Updates the submodule state and returns true if it has changed.
Sam Zackrissoncb1b5562018-09-28 14:15:09 +0200209 bool Update(bool high_pass_filter_enabled,
peah2ace3f92016-09-10 04:42:27 -0700210 bool mobile_echo_controller_enabled,
211 bool noise_suppressor_enabled,
peah2ace3f92016-09-10 04:42:27 -0700212 bool adaptive_gain_controller_enabled,
alessiob3ec96df2017-05-22 06:57:06 -0700213 bool gain_controller2_enabled,
Hanna Silen0c1ad292022-06-16 16:35:45 +0200214 bool voice_activity_detector_enabled,
Per Åhgrendb5d7282021-03-15 16:31:04 +0000215 bool gain_adjustment_enabled,
Gustaf Ullbergce045ac2017-10-16 13:49:04 +0200216 bool echo_controller_enabled,
peah2ace3f92016-09-10 04:42:27 -0700217 bool transient_suppressor_enabled);
218 bool CaptureMultiBandSubModulesActive() const;
Gustaf Ullberg8675eee2019-10-09 13:34:36 +0200219 bool CaptureMultiBandProcessingPresent() const;
220 bool CaptureMultiBandProcessingActive(bool ec_processing_active) const;
peah23ac8b42017-05-23 05:33:56 -0700221 bool CaptureFullBandProcessingActive() const;
Valeriia Nemychnikovaf06eb572018-08-29 10:37:09 +0200222 bool CaptureAnalyzerActive() const;
peah2ace3f92016-09-10 04:42:27 -0700223 bool RenderMultiBandSubModulesActive() const;
Alex Loiko5825aa62017-12-18 16:02:40 +0100224 bool RenderFullBandProcessingActive() const;
peah2ace3f92016-09-10 04:42:27 -0700225 bool RenderMultiBandProcessingActive() const;
Per Åhgren0aefbf02019-08-23 21:29:17 +0200226 bool HighPassFilteringRequired() const;
peah2ace3f92016-09-10 04:42:27 -0700227
228 private:
Sam Zackrisson0beac582017-09-25 12:04:02 +0200229 const bool capture_post_processor_enabled_ = false;
Alex Loiko5825aa62017-12-18 16:02:40 +0100230 const bool render_pre_processor_enabled_ = false;
Valeriia Nemychnikovaf06eb572018-08-29 10:37:09 +0200231 const bool capture_analyzer_enabled_ = false;
Sam Zackrissoncb1b5562018-09-28 14:15:09 +0200232 bool high_pass_filter_enabled_ = false;
peah2ace3f92016-09-10 04:42:27 -0700233 bool mobile_echo_controller_enabled_ = false;
234 bool noise_suppressor_enabled_ = false;
peah2ace3f92016-09-10 04:42:27 -0700235 bool adaptive_gain_controller_enabled_ = false;
Hanna Silen0c1ad292022-06-16 16:35:45 +0200236 bool voice_activity_detector_enabled_ = false;
alessiob3ec96df2017-05-22 06:57:06 -0700237 bool gain_controller2_enabled_ = false;
Per Åhgrendb5d7282021-03-15 16:31:04 +0000238 bool gain_adjustment_enabled_ = false;
Gustaf Ullbergce045ac2017-10-16 13:49:04 +0200239 bool echo_controller_enabled_ = false;
peah2ace3f92016-09-10 04:42:27 -0700240 bool transient_suppressor_enabled_ = false;
241 bool first_update_ = true;
242 };
243
Sam Zackrisson12e319a2020-01-03 14:54:20 +0100244 // Methods for modifying the formats struct that is used by both
245 // the render and capture threads. The check for whether modifications are
246 // needed is done while holding a single lock only, thereby avoiding that the
247 // capture thread blocks the render thread.
248 // Called by render: Holds the render lock when reading the format struct and
249 // acquires both locks if reinitialization is required.
peahdf3efa82015-11-28 12:35:15 -0800250 int MaybeInitializeRender(const ProcessingConfig& processing_config)
Markus Handell0df0fae2020-07-07 15:53:34 +0200251 RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_render_);
Sam Zackrisson12e319a2020-01-03 14:54:20 +0100252 // Called by capture: Holds the capture lock when reading the format struct
253 // and acquires both locks if reinitialization is needed.
254 int MaybeInitializeCapture(const StreamConfig& input_config,
255 const StreamConfig& output_config);
peahdf3efa82015-11-28 12:35:15 -0800256
peah2ace3f92016-09-10 04:42:27 -0700257 // Method for updating the state keeping track of the active submodules.
258 // Returns a bool indicating whether the state has changed.
danilchap56359be2017-09-07 07:53:45 -0700259 bool UpdateActiveSubmoduleStates()
Markus Handell0df0fae2020-07-07 15:53:34 +0200260 RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_capture_);
peahdf3efa82015-11-28 12:35:15 -0800261
Per Åhgrenc0734712020-01-02 15:15:36 +0100262 // Methods requiring APM running in a single-threaded manner, requiring both
263 // the render and capture lock to be acquired.
Michael Graczyk86c6d332015-07-23 11:41:39 -0700264 int InitializeLocked(const ProcessingConfig& config)
Markus Handell0df0fae2020-07-07 15:53:34 +0200265 RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_render_, mutex_capture_);
ivoc9f4a4a02016-10-28 05:39:16 -0700266 void InitializeResidualEchoDetector()
Markus Handell0df0fae2020-07-07 15:53:34 +0200267 RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_render_, mutex_capture_);
Per Åhgrenf204faf2019-04-25 15:18:06 +0200268 void InitializeEchoController()
Markus Handell0df0fae2020-07-07 15:53:34 +0200269 RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_render_, mutex_capture_);
Per Åhgrenc0734712020-01-02 15:15:36 +0100270
Alessio Bazzica38901042021-10-14 12:14:21 +0200271 // Initializations of capture-only sub-modules, requiring the capture lock
Per Åhgrenc0734712020-01-02 15:15:36 +0100272 // already acquired.
Per Åhgren0f14db22020-01-03 14:27:14 +0100273 void InitializeHighPassFilter(bool forced_reset)
Markus Handell0df0fae2020-07-07 15:53:34 +0200274 RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_capture_);
Markus Handell0df0fae2020-07-07 15:53:34 +0200275 void InitializeGainController1() RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_capture_);
Per Åhgrenc0734712020-01-02 15:15:36 +0100276 void InitializeTransientSuppressor()
Markus Handell0df0fae2020-07-07 15:53:34 +0200277 RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_capture_);
Alessio Bazzica38901042021-10-14 12:14:21 +0200278 // Initializes the `GainController2` sub-module. If the sub-module is enabled
279 // and `config_has_changed` is true, recreates the sub-module.
280 void InitializeGainController2(bool config_has_changed)
281 RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_capture_);
Hanna Silen0c1ad292022-06-16 16:35:45 +0200282 // Initializes the `VoiceActivityDetectorWrapper` sub-module. If the
283 // sub-module is enabled and `config_has_changed` is true, recreates the
284 // sub-module.
285 void InitializeVoiceActivityDetector(bool config_has_changed)
286 RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_capture_);
Markus Handell0df0fae2020-07-07 15:53:34 +0200287 void InitializeNoiseSuppressor() RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_capture_);
Per Åhgrendb5d7282021-03-15 16:31:04 +0000288 void InitializeCaptureLevelsAdjuster()
289 RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_capture_);
Markus Handell0df0fae2020-07-07 15:53:34 +0200290 void InitializePostProcessor() RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_capture_);
291 void InitializeAnalyzer() RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_capture_);
Per Åhgrenc0734712020-01-02 15:15:36 +0100292
293 // Initializations of render-only submodules, requiring the render lock
294 // already acquired.
Markus Handell0df0fae2020-07-07 15:53:34 +0200295 void InitializePreProcessor() RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_render_);
peahdf3efa82015-11-28 12:35:15 -0800296
Gustaf Ullberg422b9e02019-10-09 13:02:14 +0200297 // Sample rate used for the fullband processing.
298 int proc_fullband_sample_rate_hz() const
Markus Handell0df0fae2020-07-07 15:53:34 +0200299 RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_capture_);
Gustaf Ullberg422b9e02019-10-09 13:02:14 +0200300
Alex Loiko73ec0192018-05-15 10:52:28 +0200301 // Empties and handles the respective RuntimeSetting queues.
302 void HandleCaptureRuntimeSettings()
Markus Handell0df0fae2020-07-07 15:53:34 +0200303 RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_capture_);
304 void HandleRenderRuntimeSettings()
305 RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_render_);
Sam Zackrissonf0d1c032019-03-27 13:28:08 +0100306
Markus Handell0df0fae2020-07-07 15:53:34 +0200307 void EmptyQueuedRenderAudio() RTC_LOCKS_EXCLUDED(mutex_capture_);
Markus Handell02ba1d22020-05-14 14:31:18 +0200308 void EmptyQueuedRenderAudioLocked()
Markus Handell0df0fae2020-07-07 15:53:34 +0200309 RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_capture_);
peah764e3642016-10-22 05:04:30 -0700310 void AllocateRenderQueue()
Markus Handell0df0fae2020-07-07 15:53:34 +0200311 RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_render_, mutex_capture_);
peah9e6a2902017-05-15 07:19:21 -0700312 void QueueBandedRenderAudio(AudioBuffer* audio)
Markus Handell0df0fae2020-07-07 15:53:34 +0200313 RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_render_);
peah9e6a2902017-05-15 07:19:21 -0700314 void QueueNonbandedRenderAudio(AudioBuffer* audio)
Markus Handell0df0fae2020-07-07 15:53:34 +0200315 RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_render_);
peah764e3642016-10-22 05:04:30 -0700316
peahdf3efa82015-11-28 12:35:15 -0800317 // Capture-side exclusive methods possibly running APM in a multi-threaded
318 // manner that are called with the render lock already acquired.
Markus Handell0df0fae2020-07-07 15:53:34 +0200319 int ProcessCaptureStreamLocked() RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_capture_);
peahdf3efa82015-11-28 12:35:15 -0800320
321 // Render-side exclusive methods possibly running APM in a multi-threaded
322 // manner that are called with the render lock already acquired.
ekmeyerson60d9b332015-08-14 10:35:55 -0700323 // TODO(ekm): Remove once all clients updated to new interface.
peahdf3efa82015-11-28 12:35:15 -0800324 int AnalyzeReverseStreamLocked(const float* const* src,
325 const StreamConfig& input_config,
326 const StreamConfig& output_config)
Markus Handell0df0fae2020-07-07 15:53:34 +0200327 RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_render_);
328 int ProcessRenderStreamLocked() RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_render_);
andrew@webrtc.org17e40642014-03-04 20:58:13 +0000329
aleloi868f32f2017-05-23 07:20:05 -0700330 // Collects configuration settings from public and private
331 // submodules to be saved as an audioproc::Config message on the
Artem Titov0b489302021-07-28 20:50:03 +0200332 // AecDump if it is attached. If not `forced`, only writes the current
333 // config if it is different from the last saved one; if `forced`,
aleloi868f32f2017-05-23 07:20:05 -0700334 // writes the config regardless of the last saved.
335 void WriteAecDumpConfigMessage(bool forced)
Markus Handell0df0fae2020-07-07 15:53:34 +0200336 RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_capture_);
aleloi868f32f2017-05-23 07:20:05 -0700337
338 // Notifies attached AecDump of current configuration and capture data.
339 void RecordUnprocessedCaptureStream(const float* const* capture_stream)
Markus Handell0df0fae2020-07-07 15:53:34 +0200340 RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_capture_);
aleloi868f32f2017-05-23 07:20:05 -0700341
Per Åhgren645f24c2020-03-16 12:06:02 +0100342 void RecordUnprocessedCaptureStream(const int16_t* const data,
343 const StreamConfig& config)
Markus Handell0df0fae2020-07-07 15:53:34 +0200344 RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_capture_);
aleloi868f32f2017-05-23 07:20:05 -0700345
346 // Notifies attached AecDump of current configuration and
347 // processed capture data and issues a capture stream recording
348 // request.
349 void RecordProcessedCaptureStream(
350 const float* const* processed_capture_stream)
Markus Handell0df0fae2020-07-07 15:53:34 +0200351 RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_capture_);
aleloi868f32f2017-05-23 07:20:05 -0700352
Per Åhgren645f24c2020-03-16 12:06:02 +0100353 void RecordProcessedCaptureStream(const int16_t* const data,
354 const StreamConfig& config)
Markus Handell0df0fae2020-07-07 15:53:34 +0200355 RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_capture_);
aleloi868f32f2017-05-23 07:20:05 -0700356
357 // Notifies attached AecDump about current state (delay, drift, etc).
Markus Handell0df0fae2020-07-07 15:53:34 +0200358 void RecordAudioProcessingState()
359 RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_capture_);
aleloi868f32f2017-05-23 07:20:05 -0700360
Per Åhgren652ada52021-03-03 10:52:44 +0000361 // Ensures that overruns in the capture runtime settings queue is properly
362 // handled by the code, providing safe-fallbacks to mitigate the implications
363 // of any settings being missed.
364 void HandleOverrunInCaptureRuntimeSettingsQueue()
365 RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_capture_);
366
aleloi868f32f2017-05-23 07:20:05 -0700367 // AecDump instance used for optionally logging APM config, input
368 // and output to file in the AEC-dump format defined in debug.proto.
369 std::unique_ptr<AecDump> aec_dump_;
370
371 // Hold the last config written with AecDump for avoiding writing
372 // the same config twice.
Markus Handell0df0fae2020-07-07 15:53:34 +0200373 InternalAPMConfig apm_config_for_aec_dump_ RTC_GUARDED_BY(mutex_capture_);
aleloi868f32f2017-05-23 07:20:05 -0700374
peahdf3efa82015-11-28 12:35:15 -0800375 // Critical sections.
Markus Handell0df0fae2020-07-07 15:53:34 +0200376 mutable Mutex mutex_render_ RTC_ACQUIRED_BEFORE(mutex_capture_);
377 mutable Mutex mutex_capture_;
peahdf3efa82015-11-28 12:35:15 -0800378
peahc19f3122016-10-07 14:54:10 -0700379 // Struct containing the Config specifying the behavior of APM.
380 AudioProcessing::Config config_;
381
Sam Zackrissonb37e59d2020-04-27 08:39:33 +0200382 // Overrides for testing the exclusion of some submodules from the build.
383 ApmSubmoduleCreationOverrides submodule_creation_overrides_
Markus Handell0df0fae2020-07-07 15:53:34 +0200384 RTC_GUARDED_BY(mutex_capture_);
Sam Zackrissonb37e59d2020-04-27 08:39:33 +0200385
peah2ace3f92016-09-10 04:42:27 -0700386 // Class containing information about what submodules are active.
saza1d600522019-10-18 13:29:43 +0200387 SubmoduleStates submodule_states_;
peah2ace3f92016-09-10 04:42:27 -0700388
saza1d600522019-10-18 13:29:43 +0200389 // Struct containing the pointers to the submodules.
390 struct Submodules {
391 Submodules(std::unique_ptr<CustomProcessing> capture_post_processor,
392 std::unique_ptr<CustomProcessing> render_pre_processor,
393 rtc::scoped_refptr<EchoDetector> echo_detector,
Per Åhgren3daedb62019-11-22 12:11:40 +0100394 std::unique_ptr<CustomAudioAnalyzer> capture_analyzer)
saza1d600522019-10-18 13:29:43 +0200395 : echo_detector(std::move(echo_detector)),
396 capture_post_processor(std::move(capture_post_processor)),
397 render_pre_processor(std::move(render_pre_processor)),
Per Åhgren3daedb62019-11-22 12:11:40 +0100398 capture_analyzer(std::move(capture_analyzer)) {}
saza1d600522019-10-18 13:29:43 +0200399 // Accessed internally from capture or during initialization.
Sam Zackrisson03cb7e52021-12-06 15:40:04 +0100400 const rtc::scoped_refptr<EchoDetector> echo_detector;
401 const std::unique_ptr<CustomProcessing> capture_post_processor;
402 const std::unique_ptr<CustomProcessing> render_pre_processor;
403 const std::unique_ptr<CustomAudioAnalyzer> capture_analyzer;
saza1d600522019-10-18 13:29:43 +0200404 std::unique_ptr<AgcManagerDirect> agc_manager;
405 std::unique_ptr<GainControlImpl> gain_control;
saza1d600522019-10-18 13:29:43 +0200406 std::unique_ptr<GainController2> gain_controller2;
Hanna Silen0c1ad292022-06-16 16:35:45 +0200407 std::unique_ptr<VoiceActivityDetectorWrapper> voice_activity_detector;
saza1d600522019-10-18 13:29:43 +0200408 std::unique_ptr<HighPassFilter> high_pass_filter;
saza1d600522019-10-18 13:29:43 +0200409 std::unique_ptr<EchoControl> echo_controller;
410 std::unique_ptr<EchoControlMobileImpl> echo_control_mobile;
Per Åhgren0cbb58e2019-10-29 22:59:44 +0100411 std::unique_ptr<NoiseSuppressor> noise_suppressor;
saza1d600522019-10-18 13:29:43 +0200412 std::unique_ptr<TransientSuppressor> transient_suppressor;
Per Åhgrendb5d7282021-03-15 16:31:04 +0000413 std::unique_ptr<CaptureLevelsAdjuster> capture_levels_adjuster;
saza1d600522019-10-18 13:29:43 +0200414 } submodules_;
peahdf3efa82015-11-28 12:35:15 -0800415
peah192164e2015-11-17 02:16:45 -0800416 // State that is written to while holding both the render and capture locks
peahdf3efa82015-11-28 12:35:15 -0800417 // but can be read without any lock being held.
418 // As this is only accessed internally of APM, and all internal methods in APM
419 // either are holding the render or capture locks, this construct is safe as
420 // it is not possible to read the variables while writing them.
421 struct ApmFormatState {
422 ApmFormatState()
peah192164e2015-11-17 02:16:45 -0800423 : // Format of processing streams at input/output call sites.
Henrik Lundin64253a92022-02-04 09:02:48 +0000424 api_format({{{kSampleRate16kHz, 1},
425 {kSampleRate16kHz, 1},
426 {kSampleRate16kHz, 1},
427 {kSampleRate16kHz, 1}}}),
peahde65ddc2016-09-16 15:02:15 -0700428 render_processing_format(kSampleRate16kHz, 1) {}
peahdf3efa82015-11-28 12:35:15 -0800429 ProcessingConfig api_format;
peahde65ddc2016-09-16 15:02:15 -0700430 StreamConfig render_processing_format;
peahdf3efa82015-11-28 12:35:15 -0800431 } formats_;
Michael Graczyk86c6d332015-07-23 11:41:39 -0700432
peahdf3efa82015-11-28 12:35:15 -0800433 // APM constants.
434 const struct ApmConstants {
Per Åhgren0695df12020-01-13 14:43:13 +0100435 ApmConstants(bool multi_channel_render_support,
Per Åhgrenc0424252019-12-10 13:04:15 +0100436 bool multi_channel_capture_support,
Per Åhgren19775cb2021-03-12 23:08:09 +0000437 bool enforce_split_band_hpf,
Gustaf Ullberga399c822021-05-18 12:17:56 +0200438 bool minimize_processing_for_unused_output,
439 bool transient_suppressor_forced_off)
Per Åhgren0695df12020-01-13 14:43:13 +0100440 : multi_channel_render_support(multi_channel_render_support),
Per Åhgrenc0424252019-12-10 13:04:15 +0100441 multi_channel_capture_support(multi_channel_capture_support),
Per Åhgren19775cb2021-03-12 23:08:09 +0000442 enforce_split_band_hpf(enforce_split_band_hpf),
443 minimize_processing_for_unused_output(
Gustaf Ullberga399c822021-05-18 12:17:56 +0200444 minimize_processing_for_unused_output),
445 transient_suppressor_forced_off(transient_suppressor_forced_off) {}
Per Åhgrene14cb992019-11-27 09:34:22 +0100446 bool multi_channel_render_support;
447 bool multi_channel_capture_support;
Per Åhgrenc0424252019-12-10 13:04:15 +0100448 bool enforce_split_band_hpf;
Per Åhgren19775cb2021-03-12 23:08:09 +0000449 bool minimize_processing_for_unused_output;
Gustaf Ullberga399c822021-05-18 12:17:56 +0200450 bool transient_suppressor_forced_off;
peahdf3efa82015-11-28 12:35:15 -0800451 } constants_;
andrew@webrtc.orgddbb8a22014-04-22 21:00:04 +0000452
peahdf3efa82015-11-28 12:35:15 -0800453 struct ApmCaptureState {
Per Åhgrenc0734712020-01-02 15:15:36 +0100454 ApmCaptureState();
kwiberg83ffe452016-08-29 14:46:07 -0700455 ~ApmCaptureState();
peahdf3efa82015-11-28 12:35:15 -0800456 bool was_stream_delay_set;
Per Åhgren0a144a72021-02-09 08:47:51 +0100457 bool capture_output_used;
Per Åhgren55bc0772021-03-12 14:18:36 +0000458 bool capture_output_used_last_frame;
peahdf3efa82015-11-28 12:35:15 -0800459 bool key_pressed;
kwiberg88788ad2016-02-19 07:04:49 -0800460 std::unique_ptr<AudioBuffer> capture_audio;
Gustaf Ullberg422b9e02019-10-09 13:02:14 +0200461 std::unique_ptr<AudioBuffer> capture_fullband_audio;
Per Åhgrenc20a19c2019-11-13 11:12:29 +0100462 std::unique_ptr<AudioBuffer> linear_aec_output;
peahde65ddc2016-09-16 15:02:15 -0700463 // Only the rate and samples fields of capture_processing_format_ are used
464 // because the capture processing number of channels is mutable and is
465 // tracked by the capture_audio_.
466 StreamConfig capture_processing_format;
peahdf3efa82015-11-28 12:35:15 -0800467 int split_rate;
peah67995532017-04-10 14:12:41 -0700468 bool echo_path_gain_change;
Per Åhgren88cf0502018-07-16 17:08:41 +0200469 int prev_analog_mic_level;
Per Åhgrendb5d7282021-03-15 16:31:04 +0000470 float prev_pre_adjustment_gain;
Fredrik Hernqvistca362852019-05-10 15:50:02 +0200471 int playout_volume;
472 int prev_playout_volume;
Sam Zackrissonb24c00f2018-11-26 16:18:25 +0100473 AudioProcessingStats stats;
Per Åhgren0695df12020-01-13 14:43:13 +0100474 int cached_stream_analog_level_ = 0;
Markus Handell0df0fae2020-07-07 15:53:34 +0200475 } capture_ RTC_GUARDED_BY(mutex_capture_);
niklase@google.com470e71d2011-07-07 08:21:25 +0000476
peahdf3efa82015-11-28 12:35:15 -0800477 struct ApmCaptureNonLockedState {
Alessio Bazzicacc22f512018-08-30 13:01:34 +0200478 ApmCaptureNonLockedState()
peahde65ddc2016-09-16 15:02:15 -0700479 : capture_processing_format(kSampleRate16kHz),
peahdf3efa82015-11-28 12:35:15 -0800480 split_rate(kSampleRate16kHz),
Alessio Bazzicacc22f512018-08-30 13:01:34 +0200481 stream_delay_ms(0) {}
peahde65ddc2016-09-16 15:02:15 -0700482 // Only the rate and samples fields of capture_processing_format_ are used
483 // because the forward processing number of channels is mutable and is
484 // tracked by the capture_audio_.
485 StreamConfig capture_processing_format;
peahdf3efa82015-11-28 12:35:15 -0800486 int split_rate;
487 int stream_delay_ms;
Gustaf Ullbergce045ac2017-10-16 13:49:04 +0200488 bool echo_controller_enabled = false;
peahdf3efa82015-11-28 12:35:15 -0800489 } capture_nonlocked_;
andrew@webrtc.org75dd2882014-02-11 20:52:30 +0000490
peahdf3efa82015-11-28 12:35:15 -0800491 struct ApmRenderState {
kwiberg83ffe452016-08-29 14:46:07 -0700492 ApmRenderState();
493 ~ApmRenderState();
kwiberg88788ad2016-02-19 07:04:49 -0800494 std::unique_ptr<AudioConverter> render_converter;
495 std::unique_ptr<AudioBuffer> render_audio;
Markus Handell0df0fae2020-07-07 15:53:34 +0200496 } render_ RTC_GUARDED_BY(mutex_render_);
peah764e3642016-10-22 05:04:30 -0700497
Per Åhgrencf4c8722019-12-30 14:32:14 +0100498 // Class for statistics reporting. The class is thread-safe and no lock is
499 // needed when accessing it.
500 class ApmStatsReporter {
501 public:
502 ApmStatsReporter();
503 ~ApmStatsReporter();
504
505 // Returns the most recently reported statistics.
506 AudioProcessingStats GetStatistics();
507
508 // Update the cached statistics.
509 void UpdateStatistics(const AudioProcessingStats& new_stats);
510
511 private:
Markus Handell0df0fae2020-07-07 15:53:34 +0200512 Mutex mutex_stats_;
513 AudioProcessingStats cached_stats_ RTC_GUARDED_BY(mutex_stats_);
Per Åhgrencf4c8722019-12-30 14:32:14 +0100514 SwapQueue<AudioProcessingStats> stats_message_queue_;
515 } stats_reporter_;
516
Markus Handell0df0fae2020-07-07 15:53:34 +0200517 std::vector<int16_t> aecm_render_queue_buffer_ RTC_GUARDED_BY(mutex_render_);
518 std::vector<int16_t> aecm_capture_queue_buffer_
519 RTC_GUARDED_BY(mutex_capture_);
peah701d6282016-10-25 05:42:20 -0700520
Markus Handell0df0fae2020-07-07 15:53:34 +0200521 size_t agc_render_queue_element_max_size_ RTC_GUARDED_BY(mutex_render_)
522 RTC_GUARDED_BY(mutex_capture_) = 0;
523 std::vector<int16_t> agc_render_queue_buffer_ RTC_GUARDED_BY(mutex_render_);
524 std::vector<int16_t> agc_capture_queue_buffer_ RTC_GUARDED_BY(mutex_capture_);
peah764e3642016-10-22 05:04:30 -0700525
Markus Handell0df0fae2020-07-07 15:53:34 +0200526 size_t red_render_queue_element_max_size_ RTC_GUARDED_BY(mutex_render_)
527 RTC_GUARDED_BY(mutex_capture_) = 0;
528 std::vector<float> red_render_queue_buffer_ RTC_GUARDED_BY(mutex_render_);
529 std::vector<float> red_capture_queue_buffer_ RTC_GUARDED_BY(mutex_capture_);
ivoc9f4a4a02016-10-28 05:39:16 -0700530
Markus Handell0df0fae2020-07-07 15:53:34 +0200531 RmsLevel capture_input_rms_ RTC_GUARDED_BY(mutex_capture_);
532 RmsLevel capture_output_rms_ RTC_GUARDED_BY(mutex_capture_);
533 int capture_rms_interval_counter_ RTC_GUARDED_BY(mutex_capture_) = 0;
henrik.lundin290d43a2016-11-29 08:09:09 -0800534
Hanna Silen529131d2021-10-29 14:55:45 +0200535 AnalogGainStatsReporter analog_gain_stats_reporter_
536 RTC_GUARDED_BY(mutex_capture_);
537
peah764e3642016-10-22 05:04:30 -0700538 // Lock protection not needed.
peaha0624602016-10-25 04:45:24 -0700539 std::unique_ptr<
540 SwapQueue<std::vector<int16_t>, RenderQueueItemVerifier<int16_t>>>
peah701d6282016-10-25 05:42:20 -0700541 aecm_render_signal_queue_;
542 std::unique_ptr<
543 SwapQueue<std::vector<int16_t>, RenderQueueItemVerifier<int16_t>>>
544 agc_render_signal_queue_;
ivoc9f4a4a02016-10-28 05:39:16 -0700545 std::unique_ptr<SwapQueue<std::vector<float>, RenderQueueItemVerifier<float>>>
546 red_render_signal_queue_;
niklase@google.com470e71d2011-07-07 08:21:25 +0000547};
andrew@webrtc.orgddbb8a22014-04-22 21:00:04 +0000548
niklase@google.com470e71d2011-07-07 08:21:25 +0000549} // namespace webrtc
550
Mirko Bonadei92ea95e2017-09-15 06:47:31 +0200551#endif // MODULES_AUDIO_PROCESSING_AUDIO_PROCESSING_IMPL_H_