blob: 676e3cde9db33b6b0b40e8e2bef77fe38e05c8f2 [file] [log] [blame]
niklase@google.com470e71d2011-07-07 08:21:25 +00001/*
andrew@webrtc.org6f9f8172012-03-06 19:03:39 +00002 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
niklase@google.com470e71d2011-07-07 08:21:25 +00003 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020011#ifndef MODULES_AUDIO_PROCESSING_AUDIO_PROCESSING_IMPL_H_
12#define MODULES_AUDIO_PROCESSING_AUDIO_PROCESSING_IMPL_H_
niklase@google.com470e71d2011-07-07 08:21:25 +000013
Per Åhgren09e9a832020-05-11 11:03:47 +020014#include <stdio.h>
15
niklase@google.com470e71d2011-07-07 08:21:25 +000016#include <list>
kwiberg88788ad2016-02-19 07:04:49 -080017#include <memory>
Per Åhgren09e9a832020-05-11 11:03:47 +020018#include <string>
Michael Graczyk86c6d332015-07-23 11:41:39 -070019#include <vector>
niklase@google.com470e71d2011-07-07 08:21:25 +000020
Artem Titov741daaf2019-03-21 14:37:36 +010021#include "api/function_view.h"
saza1d600522019-10-18 13:29:43 +020022#include "modules/audio_processing/aec3/echo_canceller3.h"
23#include "modules/audio_processing/agc/agc_manager_direct.h"
Sam Zackrisson41478c72019-10-15 10:10:26 +020024#include "modules/audio_processing/agc/gain_control.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020025#include "modules/audio_processing/audio_buffer.h"
saza1d600522019-10-18 13:29:43 +020026#include "modules/audio_processing/echo_control_mobile_impl.h"
saza1d600522019-10-18 13:29:43 +020027#include "modules/audio_processing/gain_control_impl.h"
28#include "modules/audio_processing/gain_controller2.h"
29#include "modules/audio_processing/high_pass_filter.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020030#include "modules/audio_processing/include/aec_dump.h"
Per Åhgren71652f42020-03-17 13:23:58 +010031#include "modules/audio_processing/include/audio_frame_proxies.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020032#include "modules/audio_processing/include/audio_processing.h"
Sam Zackrissonb24c00f2018-11-26 16:18:25 +010033#include "modules/audio_processing/include/audio_processing_statistics.h"
saza1d600522019-10-18 13:29:43 +020034#include "modules/audio_processing/level_estimator.h"
Per Åhgren0cbb58e2019-10-29 22:59:44 +010035#include "modules/audio_processing/ns/noise_suppressor.h"
Sam Zackrissonb37e59d2020-04-27 08:39:33 +020036#include "modules/audio_processing/optionally_built_submodule_creators.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020037#include "modules/audio_processing/render_queue_item_verifier.h"
saza1d600522019-10-18 13:29:43 +020038#include "modules/audio_processing/residual_echo_detector.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020039#include "modules/audio_processing/rms_level.h"
saza1d600522019-10-18 13:29:43 +020040#include "modules/audio_processing/transient/transient_suppressor.h"
41#include "modules/audio_processing/voice_detection.h"
Steve Anton10542f22019-01-11 09:11:00 -080042#include "rtc_base/critical_section.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020043#include "rtc_base/gtest_prod_util.h"
44#include "rtc_base/ignore_wundef.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020045#include "rtc_base/swap_queue.h"
46#include "rtc_base/thread_annotations.h"
peahdf3efa82015-11-28 12:35:15 -080047
niklase@google.com470e71d2011-07-07 08:21:25 +000048namespace webrtc {
andrew@webrtc.orgddbb8a22014-04-22 21:00:04 +000049
Per Åhgren13735822018-02-12 21:42:56 +010050class ApmDataDumper;
ekmeyerson60d9b332015-08-14 10:35:55 -070051class AudioConverter;
Michael Graczykdfa36052015-03-25 16:37:27 -070052
niklase@google.com470e71d2011-07-07 08:21:25 +000053class AudioProcessingImpl : public AudioProcessing {
54 public:
peahdf3efa82015-11-28 12:35:15 -080055 // Methods forcing APM to run in a single-threaded manner.
56 // Acquires both the render and capture locks.
peah88ac8532016-09-12 16:47:25 -070057 explicit AudioProcessingImpl(const webrtc::Config& config);
Sam Zackrissondb389722018-06-21 10:12:24 +020058 // AudioProcessingImpl takes ownership of capture post processor.
peah88ac8532016-09-12 16:47:25 -070059 AudioProcessingImpl(const webrtc::Config& config,
Alex Loiko5825aa62017-12-18 16:02:40 +010060 std::unique_ptr<CustomProcessing> capture_post_processor,
61 std::unique_ptr<CustomProcessing> render_pre_processor,
Gustaf Ullberg002ef282017-10-12 15:13:17 +020062 std::unique_ptr<EchoControlFactory> echo_control_factory,
Valeriia Nemychnikovaf06eb572018-08-29 10:37:09 +020063 rtc::scoped_refptr<EchoDetector> echo_detector,
64 std::unique_ptr<CustomAudioAnalyzer> capture_analyzer);
kwiberg83ffe452016-08-29 14:46:07 -070065 ~AudioProcessingImpl() override;
kjellander@webrtc.org14665ff2015-03-04 12:58:35 +000066 int Initialize() override;
peahde65ddc2016-09-16 15:02:15 -070067 int Initialize(int capture_input_sample_rate_hz,
68 int capture_output_sample_rate_hz,
69 int render_sample_rate_hz,
70 ChannelLayout capture_input_layout,
71 ChannelLayout capture_output_layout,
72 ChannelLayout render_input_layout) override;
Michael Graczyk86c6d332015-07-23 11:41:39 -070073 int Initialize(const ProcessingConfig& processing_config) override;
peah88ac8532016-09-12 16:47:25 -070074 void ApplyConfig(const AudioProcessing::Config& config) override;
75 void SetExtraOptions(const webrtc::Config& config) override;
Per Åhgren09e9a832020-05-11 11:03:47 +020076 bool CreateAndAttachAecDump(const std::string& file_name,
77 int64_t max_log_size_bytes,
78 rtc::TaskQueue* worker_queue) override;
79 bool CreateAndAttachAecDump(FILE* handle,
80 int64_t max_log_size_bytes,
81 rtc::TaskQueue* worker_queue) override;
82 // TODO(webrtc:5298) Deprecated variant.
aleloi868f32f2017-05-23 07:20:05 -070083 void AttachAecDump(std::unique_ptr<AecDump> aec_dump) override;
84 void DetachAecDump() override;
Alessio Bazzicac054e782018-04-16 12:10:09 +020085 void SetRuntimeSetting(RuntimeSetting setting) override;
86
peahdf3efa82015-11-28 12:35:15 -080087 // Capture-side exclusive methods possibly running APM in a
88 // multi-threaded manner. Acquire the capture lock.
Per Åhgren645f24c2020-03-16 12:06:02 +010089 int ProcessStream(const int16_t* const src,
90 const StreamConfig& input_config,
91 const StreamConfig& output_config,
Per Åhgrendc5522b2020-03-19 14:55:58 +010092 int16_t* const dest) override;
kjellander@webrtc.org14665ff2015-03-04 12:58:35 +000093 int ProcessStream(const float* const* src,
Michael Graczyk86c6d332015-07-23 11:41:39 -070094 const StreamConfig& input_config,
95 const StreamConfig& output_config,
96 float* const* dest) override;
Per Åhgrenc20a19c2019-11-13 11:12:29 +010097 bool GetLinearAecOutput(
98 rtc::ArrayView<std::array<float, 160>> linear_output) const override;
peahdf3efa82015-11-28 12:35:15 -080099 void set_output_will_be_muted(bool muted) override;
100 int set_stream_delay_ms(int delay) override;
peahdf3efa82015-11-28 12:35:15 -0800101 void set_stream_key_pressed(bool key_pressed) override;
Sam Zackrissonf0d1c032019-03-27 13:28:08 +0100102 void set_stream_analog_level(int level) override;
Markus Handell02ba1d22020-05-14 14:31:18 +0200103 int recommended_stream_analog_level() const
104 RTC_LOCKS_EXCLUDED(crit_capture_) override;
peahdf3efa82015-11-28 12:35:15 -0800105
106 // Render-side exclusive methods possibly running APM in a
107 // multi-threaded manner. Acquire the render lock.
Per Åhgren645f24c2020-03-16 12:06:02 +0100108 int ProcessReverseStream(const int16_t* const src,
109 const StreamConfig& input_config,
110 const StreamConfig& output_config,
111 int16_t* const dest) override;
kjellander@webrtc.org14665ff2015-03-04 12:58:35 +0000112 int AnalyzeReverseStream(const float* const* data,
Gustaf Ullberg8c51f2e2019-10-22 15:21:31 +0200113 const StreamConfig& reverse_config) override;
ekmeyerson60d9b332015-08-14 10:35:55 -0700114 int ProcessReverseStream(const float* const* src,
peahde65ddc2016-09-16 15:02:15 -0700115 const StreamConfig& input_config,
116 const StreamConfig& output_config,
ekmeyerson60d9b332015-08-14 10:35:55 -0700117 float* const* dest) override;
peahdf3efa82015-11-28 12:35:15 -0800118
119 // Methods only accessed from APM submodules or
120 // from AudioProcessing tests in a single-threaded manner.
121 // Hence there is no need for locks in these.
122 int proc_sample_rate_hz() const override;
123 int proc_split_sample_rate_hz() const override;
Peter Kasting69558702016-01-12 16:26:35 -0800124 size_t num_input_channels() const override;
125 size_t num_proc_channels() const override;
126 size_t num_output_channels() const override;
127 size_t num_reverse_channels() const override;
kjellander@webrtc.org14665ff2015-03-04 12:58:35 +0000128 int stream_delay_ms() const override;
peahdf3efa82015-11-28 12:35:15 -0800129
Per Åhgrencf4c8722019-12-30 14:32:14 +0100130 AudioProcessingStats GetStatistics(bool has_remote_tracks) override {
131 return GetStatistics();
132 }
133 AudioProcessingStats GetStatistics() override {
134 return stats_reporter_.GetStatistics();
135 }
ivoc3e9a5372016-10-28 07:55:33 -0700136
henrik.lundinadf06352017-04-05 05:48:24 -0700137 // TODO(peah): Remove MutateConfig once the new API allows that.
peah8271d042016-11-22 07:24:52 -0800138 void MutateConfig(rtc::FunctionView<void(AudioProcessing::Config*)> mutator);
henrik.lundinadf06352017-04-05 05:48:24 -0700139 AudioProcessing::Config GetConfig() const override;
peah8271d042016-11-22 07:24:52 -0800140
andrew@webrtc.org60730cf2014-01-07 17:45:09 +0000141 protected:
andrew@webrtc.orga8b97372014-03-10 22:26:12 +0000142 // Overridden in a mock.
peahdf3efa82015-11-28 12:35:15 -0800143 virtual int InitializeLocked()
danilchap56359be2017-09-07 07:53:45 -0700144 RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_render_, crit_capture_);
andrew@webrtc.org60730cf2014-01-07 17:45:09 +0000145
niklase@google.com470e71d2011-07-07 08:21:25 +0000146 private:
peahc19f3122016-10-07 14:54:10 -0700147 // TODO(peah): These friend classes should be removed as soon as the new
148 // parameter setting scheme allows.
149 FRIEND_TEST_ALL_PREFIXES(ApmConfiguration, DefaultBehavior);
150 FRIEND_TEST_ALL_PREFIXES(ApmConfiguration, ValidConfigBehavior);
151 FRIEND_TEST_ALL_PREFIXES(ApmConfiguration, InValidConfigBehavior);
Sam Zackrissonb37e59d2020-04-27 08:39:33 +0200152 FRIEND_TEST_ALL_PREFIXES(ApmWithSubmodulesExcludedTest,
153 ToggleTransientSuppressor);
154 FRIEND_TEST_ALL_PREFIXES(ApmWithSubmodulesExcludedTest,
155 ReinitializeTransientSuppressor);
156 FRIEND_TEST_ALL_PREFIXES(ApmWithSubmodulesExcludedTest,
157 BitexactWithDisabledModules);
158
Markus Handell02ba1d22020-05-14 14:31:18 +0200159 int recommended_stream_analog_level_locked() const
160 RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_capture_);
161
Sam Zackrissonb37e59d2020-04-27 08:39:33 +0200162 void OverrideSubmoduleCreationForTesting(
163 const ApmSubmoduleCreationOverrides& overrides);
Alessio Bazzicac054e782018-04-16 12:10:09 +0200164
165 // Class providing thread-safe message pipe functionality for
166 // |runtime_settings_|.
167 class RuntimeSettingEnqueuer {
168 public:
169 explicit RuntimeSettingEnqueuer(
170 SwapQueue<RuntimeSetting>* runtime_settings);
171 ~RuntimeSettingEnqueuer();
172 void Enqueue(RuntimeSetting setting);
173
174 private:
Alessio Bazzica33444dc2018-04-20 13:16:55 +0200175 SwapQueue<RuntimeSetting>& runtime_settings_;
Alex Loiko73ec0192018-05-15 10:52:28 +0200176 };
Alex Loiko73ec0192018-05-15 10:52:28 +0200177
178 std::unique_ptr<ApmDataDumper> data_dumper_;
179 static int instance_count_;
Per Åhgrenb2b58d82019-12-02 14:59:40 +0100180 const bool use_setup_specific_default_aec3_config_;
Alex Loiko73ec0192018-05-15 10:52:28 +0200181
182 SwapQueue<RuntimeSetting> capture_runtime_settings_;
183 SwapQueue<RuntimeSetting> render_runtime_settings_;
184
185 RuntimeSettingEnqueuer capture_runtime_settings_enqueuer_;
186 RuntimeSettingEnqueuer render_runtime_settings_enqueuer_;
Alessio Bazzicac054e782018-04-16 12:10:09 +0200187
Gustaf Ullberg002ef282017-10-12 15:13:17 +0200188 // EchoControl factory.
189 std::unique_ptr<EchoControlFactory> echo_control_factory_;
Gustaf Ullbergd8579e02017-10-11 16:29:02 +0200190
saza1d600522019-10-18 13:29:43 +0200191 class SubmoduleStates {
peah2ace3f92016-09-10 04:42:27 -0700192 public:
saza1d600522019-10-18 13:29:43 +0200193 SubmoduleStates(bool capture_post_processor_enabled,
194 bool render_pre_processor_enabled,
195 bool capture_analyzer_enabled);
peah2ace3f92016-09-10 04:42:27 -0700196 // Updates the submodule state and returns true if it has changed.
Sam Zackrissoncb1b5562018-09-28 14:15:09 +0200197 bool Update(bool high_pass_filter_enabled,
peah2ace3f92016-09-10 04:42:27 -0700198 bool mobile_echo_controller_enabled,
ivoc9f4a4a02016-10-28 05:39:16 -0700199 bool residual_echo_detector_enabled,
peah2ace3f92016-09-10 04:42:27 -0700200 bool noise_suppressor_enabled,
peah2ace3f92016-09-10 04:42:27 -0700201 bool adaptive_gain_controller_enabled,
alessiob3ec96df2017-05-22 06:57:06 -0700202 bool gain_controller2_enabled,
Alex Loikob5c9a792018-04-16 16:31:22 +0200203 bool pre_amplifier_enabled,
Gustaf Ullbergce045ac2017-10-16 13:49:04 +0200204 bool echo_controller_enabled,
Sam Zackrisson0824c6f2019-10-07 14:03:56 +0200205 bool voice_detector_enabled,
peah2ace3f92016-09-10 04:42:27 -0700206 bool transient_suppressor_enabled);
207 bool CaptureMultiBandSubModulesActive() const;
Gustaf Ullberg8675eee2019-10-09 13:34:36 +0200208 bool CaptureMultiBandProcessingPresent() const;
209 bool CaptureMultiBandProcessingActive(bool ec_processing_active) const;
peah23ac8b42017-05-23 05:33:56 -0700210 bool CaptureFullBandProcessingActive() const;
Valeriia Nemychnikovaf06eb572018-08-29 10:37:09 +0200211 bool CaptureAnalyzerActive() const;
peah2ace3f92016-09-10 04:42:27 -0700212 bool RenderMultiBandSubModulesActive() const;
Alex Loiko5825aa62017-12-18 16:02:40 +0100213 bool RenderFullBandProcessingActive() const;
peah2ace3f92016-09-10 04:42:27 -0700214 bool RenderMultiBandProcessingActive() const;
Per Åhgren0aefbf02019-08-23 21:29:17 +0200215 bool HighPassFilteringRequired() const;
peah2ace3f92016-09-10 04:42:27 -0700216
217 private:
Sam Zackrisson0beac582017-09-25 12:04:02 +0200218 const bool capture_post_processor_enabled_ = false;
Alex Loiko5825aa62017-12-18 16:02:40 +0100219 const bool render_pre_processor_enabled_ = false;
Valeriia Nemychnikovaf06eb572018-08-29 10:37:09 +0200220 const bool capture_analyzer_enabled_ = false;
Sam Zackrissoncb1b5562018-09-28 14:15:09 +0200221 bool high_pass_filter_enabled_ = false;
peah2ace3f92016-09-10 04:42:27 -0700222 bool mobile_echo_controller_enabled_ = false;
ivoc9f4a4a02016-10-28 05:39:16 -0700223 bool residual_echo_detector_enabled_ = false;
peah2ace3f92016-09-10 04:42:27 -0700224 bool noise_suppressor_enabled_ = false;
peah2ace3f92016-09-10 04:42:27 -0700225 bool adaptive_gain_controller_enabled_ = false;
alessiob3ec96df2017-05-22 06:57:06 -0700226 bool gain_controller2_enabled_ = false;
Alex Loikob5c9a792018-04-16 16:31:22 +0200227 bool pre_amplifier_enabled_ = false;
Gustaf Ullbergce045ac2017-10-16 13:49:04 +0200228 bool echo_controller_enabled_ = false;
Sam Zackrisson0824c6f2019-10-07 14:03:56 +0200229 bool voice_detector_enabled_ = false;
peah2ace3f92016-09-10 04:42:27 -0700230 bool transient_suppressor_enabled_ = false;
231 bool first_update_ = true;
232 };
233
Sam Zackrisson12e319a2020-01-03 14:54:20 +0100234 // Methods for modifying the formats struct that is used by both
235 // the render and capture threads. The check for whether modifications are
236 // needed is done while holding a single lock only, thereby avoiding that the
237 // capture thread blocks the render thread.
238 // Called by render: Holds the render lock when reading the format struct and
239 // acquires both locks if reinitialization is required.
peahdf3efa82015-11-28 12:35:15 -0800240 int MaybeInitializeRender(const ProcessingConfig& processing_config)
danilchap56359be2017-09-07 07:53:45 -0700241 RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_render_);
Sam Zackrisson12e319a2020-01-03 14:54:20 +0100242 // Called by capture: Holds the capture lock when reading the format struct
243 // and acquires both locks if reinitialization is needed.
244 int MaybeInitializeCapture(const StreamConfig& input_config,
245 const StreamConfig& output_config);
peahdf3efa82015-11-28 12:35:15 -0800246
peah2ace3f92016-09-10 04:42:27 -0700247 // Method for updating the state keeping track of the active submodules.
248 // Returns a bool indicating whether the state has changed.
danilchap56359be2017-09-07 07:53:45 -0700249 bool UpdateActiveSubmoduleStates()
250 RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_capture_);
peahdf3efa82015-11-28 12:35:15 -0800251
Per Åhgrenc0734712020-01-02 15:15:36 +0100252 // Methods requiring APM running in a single-threaded manner, requiring both
253 // the render and capture lock to be acquired.
Michael Graczyk86c6d332015-07-23 11:41:39 -0700254 int InitializeLocked(const ProcessingConfig& config)
danilchap56359be2017-09-07 07:53:45 -0700255 RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_render_, crit_capture_);
ivoc9f4a4a02016-10-28 05:39:16 -0700256 void InitializeResidualEchoDetector()
danilchap56359be2017-09-07 07:53:45 -0700257 RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_render_, crit_capture_);
Per Åhgrenf204faf2019-04-25 15:18:06 +0200258 void InitializeEchoController()
259 RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_render_, crit_capture_);
Per Åhgrenc0734712020-01-02 15:15:36 +0100260
261 // Initializations of capture-only submodules, requiring the capture lock
262 // already acquired.
Per Åhgren0f14db22020-01-03 14:27:14 +0100263 void InitializeHighPassFilter(bool forced_reset)
264 RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_capture_);
Per Åhgrenc0734712020-01-02 15:15:36 +0100265 void InitializeVoiceDetector() RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_capture_);
Per Åhgren0695df12020-01-13 14:43:13 +0100266 void InitializeGainController1() RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_capture_);
Per Åhgrenc0734712020-01-02 15:15:36 +0100267 void InitializeTransientSuppressor()
268 RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_capture_);
Sam Zackrisson0beac582017-09-25 12:04:02 +0200269 void InitializeGainController2() RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_capture_);
saza0bad15f2019-10-16 11:46:11 +0200270 void InitializeNoiseSuppressor() RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_capture_);
Alex Loikob5c9a792018-04-16 16:31:22 +0200271 void InitializePreAmplifier() RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_capture_);
Sam Zackrisson0beac582017-09-25 12:04:02 +0200272 void InitializePostProcessor() RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_capture_);
Valeriia Nemychnikovaf06eb572018-08-29 10:37:09 +0200273 void InitializeAnalyzer() RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_capture_);
Per Åhgrenc0734712020-01-02 15:15:36 +0100274
275 // Initializations of render-only submodules, requiring the render lock
276 // already acquired.
Alex Loiko5825aa62017-12-18 16:02:40 +0100277 void InitializePreProcessor() RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_render_);
peahdf3efa82015-11-28 12:35:15 -0800278
Gustaf Ullberg422b9e02019-10-09 13:02:14 +0200279 // Sample rate used for the fullband processing.
280 int proc_fullband_sample_rate_hz() const
281 RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_capture_);
282
Alex Loiko73ec0192018-05-15 10:52:28 +0200283 // Empties and handles the respective RuntimeSetting queues.
284 void HandleCaptureRuntimeSettings()
285 RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_capture_);
286 void HandleRenderRuntimeSettings() RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_render_);
Sam Zackrissonf0d1c032019-03-27 13:28:08 +0100287
Markus Handell02ba1d22020-05-14 14:31:18 +0200288 void EmptyQueuedRenderAudio() RTC_LOCKS_EXCLUDED(crit_capture_);
289 void EmptyQueuedRenderAudioLocked()
290 RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_capture_);
peah764e3642016-10-22 05:04:30 -0700291 void AllocateRenderQueue()
danilchap56359be2017-09-07 07:53:45 -0700292 RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_render_, crit_capture_);
peah9e6a2902017-05-15 07:19:21 -0700293 void QueueBandedRenderAudio(AudioBuffer* audio)
danilchap56359be2017-09-07 07:53:45 -0700294 RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_render_);
peah9e6a2902017-05-15 07:19:21 -0700295 void QueueNonbandedRenderAudio(AudioBuffer* audio)
danilchap56359be2017-09-07 07:53:45 -0700296 RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_render_);
peah764e3642016-10-22 05:04:30 -0700297
peahdf3efa82015-11-28 12:35:15 -0800298 // Capture-side exclusive methods possibly running APM in a multi-threaded
299 // manner that are called with the render lock already acquired.
danilchap56359be2017-09-07 07:53:45 -0700300 int ProcessCaptureStreamLocked() RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_capture_);
peahdf3efa82015-11-28 12:35:15 -0800301
302 // Render-side exclusive methods possibly running APM in a multi-threaded
303 // manner that are called with the render lock already acquired.
ekmeyerson60d9b332015-08-14 10:35:55 -0700304 // TODO(ekm): Remove once all clients updated to new interface.
peahdf3efa82015-11-28 12:35:15 -0800305 int AnalyzeReverseStreamLocked(const float* const* src,
306 const StreamConfig& input_config,
307 const StreamConfig& output_config)
danilchap56359be2017-09-07 07:53:45 -0700308 RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_render_);
309 int ProcessRenderStreamLocked() RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_render_);
andrew@webrtc.org17e40642014-03-04 20:58:13 +0000310
aleloi868f32f2017-05-23 07:20:05 -0700311 // Collects configuration settings from public and private
312 // submodules to be saved as an audioproc::Config message on the
313 // AecDump if it is attached. If not |forced|, only writes the current
314 // config if it is different from the last saved one; if |forced|,
315 // writes the config regardless of the last saved.
316 void WriteAecDumpConfigMessage(bool forced)
danilchap56359be2017-09-07 07:53:45 -0700317 RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_capture_);
aleloi868f32f2017-05-23 07:20:05 -0700318
319 // Notifies attached AecDump of current configuration and capture data.
320 void RecordUnprocessedCaptureStream(const float* const* capture_stream)
danilchap56359be2017-09-07 07:53:45 -0700321 RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_capture_);
aleloi868f32f2017-05-23 07:20:05 -0700322
Per Åhgren645f24c2020-03-16 12:06:02 +0100323 void RecordUnprocessedCaptureStream(const int16_t* const data,
324 const StreamConfig& config)
danilchap56359be2017-09-07 07:53:45 -0700325 RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_capture_);
aleloi868f32f2017-05-23 07:20:05 -0700326
327 // Notifies attached AecDump of current configuration and
328 // processed capture data and issues a capture stream recording
329 // request.
330 void RecordProcessedCaptureStream(
331 const float* const* processed_capture_stream)
danilchap56359be2017-09-07 07:53:45 -0700332 RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_capture_);
aleloi868f32f2017-05-23 07:20:05 -0700333
Per Åhgren645f24c2020-03-16 12:06:02 +0100334 void RecordProcessedCaptureStream(const int16_t* const data,
335 const StreamConfig& config)
danilchap56359be2017-09-07 07:53:45 -0700336 RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_capture_);
aleloi868f32f2017-05-23 07:20:05 -0700337
338 // Notifies attached AecDump about current state (delay, drift, etc).
danilchap56359be2017-09-07 07:53:45 -0700339 void RecordAudioProcessingState() RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_capture_);
aleloi868f32f2017-05-23 07:20:05 -0700340
aleloi868f32f2017-05-23 07:20:05 -0700341 // AecDump instance used for optionally logging APM config, input
342 // and output to file in the AEC-dump format defined in debug.proto.
343 std::unique_ptr<AecDump> aec_dump_;
344
345 // Hold the last config written with AecDump for avoiding writing
346 // the same config twice.
danilchap56359be2017-09-07 07:53:45 -0700347 InternalAPMConfig apm_config_for_aec_dump_ RTC_GUARDED_BY(crit_capture_);
aleloi868f32f2017-05-23 07:20:05 -0700348
peahdf3efa82015-11-28 12:35:15 -0800349 // Critical sections.
danilchap56359be2017-09-07 07:53:45 -0700350 rtc::CriticalSection crit_render_ RTC_ACQUIRED_BEFORE(crit_capture_);
pbos5ad935c2016-01-25 03:52:44 -0800351 rtc::CriticalSection crit_capture_;
peahdf3efa82015-11-28 12:35:15 -0800352
peahc19f3122016-10-07 14:54:10 -0700353 // Struct containing the Config specifying the behavior of APM.
354 AudioProcessing::Config config_;
355
Sam Zackrissonb37e59d2020-04-27 08:39:33 +0200356 // Overrides for testing the exclusion of some submodules from the build.
357 ApmSubmoduleCreationOverrides submodule_creation_overrides_
358 RTC_GUARDED_BY(crit_capture_);
359
peah2ace3f92016-09-10 04:42:27 -0700360 // Class containing information about what submodules are active.
saza1d600522019-10-18 13:29:43 +0200361 SubmoduleStates submodule_states_;
peah2ace3f92016-09-10 04:42:27 -0700362
saza1d600522019-10-18 13:29:43 +0200363 // Struct containing the pointers to the submodules.
364 struct Submodules {
365 Submodules(std::unique_ptr<CustomProcessing> capture_post_processor,
366 std::unique_ptr<CustomProcessing> render_pre_processor,
367 rtc::scoped_refptr<EchoDetector> echo_detector,
Per Åhgren3daedb62019-11-22 12:11:40 +0100368 std::unique_ptr<CustomAudioAnalyzer> capture_analyzer)
saza1d600522019-10-18 13:29:43 +0200369 : echo_detector(std::move(echo_detector)),
370 capture_post_processor(std::move(capture_post_processor)),
371 render_pre_processor(std::move(render_pre_processor)),
Per Åhgren3daedb62019-11-22 12:11:40 +0100372 capture_analyzer(std::move(capture_analyzer)) {}
saza1d600522019-10-18 13:29:43 +0200373 // Accessed internally from capture or during initialization.
374 std::unique_ptr<AgcManagerDirect> agc_manager;
375 std::unique_ptr<GainControlImpl> gain_control;
saza1d600522019-10-18 13:29:43 +0200376 std::unique_ptr<GainController2> gain_controller2;
377 std::unique_ptr<HighPassFilter> high_pass_filter;
378 rtc::scoped_refptr<EchoDetector> echo_detector;
saza1d600522019-10-18 13:29:43 +0200379 std::unique_ptr<EchoControl> echo_controller;
380 std::unique_ptr<EchoControlMobileImpl> echo_control_mobile;
Per Åhgren0cbb58e2019-10-29 22:59:44 +0100381 std::unique_ptr<NoiseSuppressor> noise_suppressor;
saza1d600522019-10-18 13:29:43 +0200382 std::unique_ptr<TransientSuppressor> transient_suppressor;
383 std::unique_ptr<CustomProcessing> capture_post_processor;
384 std::unique_ptr<CustomProcessing> render_pre_processor;
385 std::unique_ptr<GainApplier> pre_amplifier;
386 std::unique_ptr<CustomAudioAnalyzer> capture_analyzer;
387 std::unique_ptr<LevelEstimator> output_level_estimator;
388 std::unique_ptr<VoiceDetection> voice_detector;
389 } submodules_;
peahdf3efa82015-11-28 12:35:15 -0800390
peah192164e2015-11-17 02:16:45 -0800391 // State that is written to while holding both the render and capture locks
peahdf3efa82015-11-28 12:35:15 -0800392 // but can be read without any lock being held.
393 // As this is only accessed internally of APM, and all internal methods in APM
394 // either are holding the render or capture locks, this construct is safe as
395 // it is not possible to read the variables while writing them.
396 struct ApmFormatState {
397 ApmFormatState()
peah192164e2015-11-17 02:16:45 -0800398 : // Format of processing streams at input/output call sites.
peahdf3efa82015-11-28 12:35:15 -0800399 api_format({{{kSampleRate16kHz, 1, false},
400 {kSampleRate16kHz, 1, false},
401 {kSampleRate16kHz, 1, false},
402 {kSampleRate16kHz, 1, false}}}),
peahde65ddc2016-09-16 15:02:15 -0700403 render_processing_format(kSampleRate16kHz, 1) {}
peahdf3efa82015-11-28 12:35:15 -0800404 ProcessingConfig api_format;
peahde65ddc2016-09-16 15:02:15 -0700405 StreamConfig render_processing_format;
peahdf3efa82015-11-28 12:35:15 -0800406 } formats_;
Michael Graczyk86c6d332015-07-23 11:41:39 -0700407
peahdf3efa82015-11-28 12:35:15 -0800408 // APM constants.
409 const struct ApmConstants {
Per Åhgren0695df12020-01-13 14:43:13 +0100410 ApmConstants(bool multi_channel_render_support,
Per Åhgrenc0424252019-12-10 13:04:15 +0100411 bool multi_channel_capture_support,
412 bool enforce_split_band_hpf)
Per Åhgren0695df12020-01-13 14:43:13 +0100413 : multi_channel_render_support(multi_channel_render_support),
Per Åhgrenc0424252019-12-10 13:04:15 +0100414 multi_channel_capture_support(multi_channel_capture_support),
415 enforce_split_band_hpf(enforce_split_band_hpf) {}
Per Åhgrene14cb992019-11-27 09:34:22 +0100416 bool multi_channel_render_support;
417 bool multi_channel_capture_support;
Per Åhgrenc0424252019-12-10 13:04:15 +0100418 bool enforce_split_band_hpf;
peahdf3efa82015-11-28 12:35:15 -0800419 } constants_;
andrew@webrtc.orgddbb8a22014-04-22 21:00:04 +0000420
peahdf3efa82015-11-28 12:35:15 -0800421 struct ApmCaptureState {
Per Åhgrenc0734712020-01-02 15:15:36 +0100422 ApmCaptureState();
kwiberg83ffe452016-08-29 14:46:07 -0700423 ~ApmCaptureState();
peahdf3efa82015-11-28 12:35:15 -0800424 bool was_stream_delay_set;
peahdf3efa82015-11-28 12:35:15 -0800425 bool output_will_be_muted;
426 bool key_pressed;
kwiberg88788ad2016-02-19 07:04:49 -0800427 std::unique_ptr<AudioBuffer> capture_audio;
Gustaf Ullberg422b9e02019-10-09 13:02:14 +0200428 std::unique_ptr<AudioBuffer> capture_fullband_audio;
Per Åhgrenc20a19c2019-11-13 11:12:29 +0100429 std::unique_ptr<AudioBuffer> linear_aec_output;
peahde65ddc2016-09-16 15:02:15 -0700430 // Only the rate and samples fields of capture_processing_format_ are used
431 // because the capture processing number of channels is mutable and is
432 // tracked by the capture_audio_.
433 StreamConfig capture_processing_format;
peahdf3efa82015-11-28 12:35:15 -0800434 int split_rate;
peah67995532017-04-10 14:12:41 -0700435 bool echo_path_gain_change;
Per Åhgren88cf0502018-07-16 17:08:41 +0200436 int prev_analog_mic_level;
Per Åhgrend2650d12018-10-02 17:00:59 +0200437 float prev_pre_amp_gain;
Fredrik Hernqvistca362852019-05-10 15:50:02 +0200438 int playout_volume;
439 int prev_playout_volume;
Sam Zackrissonb24c00f2018-11-26 16:18:25 +0100440 AudioProcessingStats stats;
Per Åhgrena1351272019-08-15 12:15:46 +0200441 struct KeyboardInfo {
442 void Extract(const float* const* data, const StreamConfig& stream_config);
443 size_t num_keyboard_frames = 0;
444 const float* keyboard_data = nullptr;
445 } keyboard_info;
Per Åhgren0695df12020-01-13 14:43:13 +0100446 int cached_stream_analog_level_ = 0;
danilchap56359be2017-09-07 07:53:45 -0700447 } capture_ RTC_GUARDED_BY(crit_capture_);
niklase@google.com470e71d2011-07-07 08:21:25 +0000448
peahdf3efa82015-11-28 12:35:15 -0800449 struct ApmCaptureNonLockedState {
Alessio Bazzicacc22f512018-08-30 13:01:34 +0200450 ApmCaptureNonLockedState()
peahde65ddc2016-09-16 15:02:15 -0700451 : capture_processing_format(kSampleRate16kHz),
peahdf3efa82015-11-28 12:35:15 -0800452 split_rate(kSampleRate16kHz),
Alessio Bazzicacc22f512018-08-30 13:01:34 +0200453 stream_delay_ms(0) {}
peahde65ddc2016-09-16 15:02:15 -0700454 // Only the rate and samples fields of capture_processing_format_ are used
455 // because the forward processing number of channels is mutable and is
456 // tracked by the capture_audio_.
457 StreamConfig capture_processing_format;
peahdf3efa82015-11-28 12:35:15 -0800458 int split_rate;
459 int stream_delay_ms;
Gustaf Ullbergce045ac2017-10-16 13:49:04 +0200460 bool echo_controller_enabled = false;
peahdf3efa82015-11-28 12:35:15 -0800461 } capture_nonlocked_;
andrew@webrtc.org75dd2882014-02-11 20:52:30 +0000462
peahdf3efa82015-11-28 12:35:15 -0800463 struct ApmRenderState {
kwiberg83ffe452016-08-29 14:46:07 -0700464 ApmRenderState();
465 ~ApmRenderState();
kwiberg88788ad2016-02-19 07:04:49 -0800466 std::unique_ptr<AudioConverter> render_converter;
467 std::unique_ptr<AudioBuffer> render_audio;
danilchap56359be2017-09-07 07:53:45 -0700468 } render_ RTC_GUARDED_BY(crit_render_);
peah764e3642016-10-22 05:04:30 -0700469
Per Åhgrencf4c8722019-12-30 14:32:14 +0100470 // Class for statistics reporting. The class is thread-safe and no lock is
471 // needed when accessing it.
472 class ApmStatsReporter {
473 public:
474 ApmStatsReporter();
475 ~ApmStatsReporter();
476
477 // Returns the most recently reported statistics.
478 AudioProcessingStats GetStatistics();
479
480 // Update the cached statistics.
481 void UpdateStatistics(const AudioProcessingStats& new_stats);
482
483 private:
484 rtc::CriticalSection crit_stats_;
485 AudioProcessingStats cached_stats_ RTC_GUARDED_BY(crit_stats_);
486 SwapQueue<AudioProcessingStats> stats_message_queue_;
487 } stats_reporter_;
488
danilchap56359be2017-09-07 07:53:45 -0700489 std::vector<int16_t> aecm_render_queue_buffer_ RTC_GUARDED_BY(crit_render_);
490 std::vector<int16_t> aecm_capture_queue_buffer_ RTC_GUARDED_BY(crit_capture_);
peah701d6282016-10-25 05:42:20 -0700491
danilchap56359be2017-09-07 07:53:45 -0700492 size_t agc_render_queue_element_max_size_ RTC_GUARDED_BY(crit_render_)
493 RTC_GUARDED_BY(crit_capture_) = 0;
494 std::vector<int16_t> agc_render_queue_buffer_ RTC_GUARDED_BY(crit_render_);
495 std::vector<int16_t> agc_capture_queue_buffer_ RTC_GUARDED_BY(crit_capture_);
peah764e3642016-10-22 05:04:30 -0700496
danilchap56359be2017-09-07 07:53:45 -0700497 size_t red_render_queue_element_max_size_ RTC_GUARDED_BY(crit_render_)
498 RTC_GUARDED_BY(crit_capture_) = 0;
499 std::vector<float> red_render_queue_buffer_ RTC_GUARDED_BY(crit_render_);
500 std::vector<float> red_capture_queue_buffer_ RTC_GUARDED_BY(crit_capture_);
ivoc9f4a4a02016-10-28 05:39:16 -0700501
danilchap56359be2017-09-07 07:53:45 -0700502 RmsLevel capture_input_rms_ RTC_GUARDED_BY(crit_capture_);
503 RmsLevel capture_output_rms_ RTC_GUARDED_BY(crit_capture_);
504 int capture_rms_interval_counter_ RTC_GUARDED_BY(crit_capture_) = 0;
henrik.lundin290d43a2016-11-29 08:09:09 -0800505
peah764e3642016-10-22 05:04:30 -0700506 // Lock protection not needed.
peaha0624602016-10-25 04:45:24 -0700507 std::unique_ptr<
508 SwapQueue<std::vector<int16_t>, RenderQueueItemVerifier<int16_t>>>
peah701d6282016-10-25 05:42:20 -0700509 aecm_render_signal_queue_;
510 std::unique_ptr<
511 SwapQueue<std::vector<int16_t>, RenderQueueItemVerifier<int16_t>>>
512 agc_render_signal_queue_;
ivoc9f4a4a02016-10-28 05:39:16 -0700513 std::unique_ptr<SwapQueue<std::vector<float>, RenderQueueItemVerifier<float>>>
514 red_render_signal_queue_;
niklase@google.com470e71d2011-07-07 08:21:25 +0000515};
andrew@webrtc.orgddbb8a22014-04-22 21:00:04 +0000516
niklase@google.com470e71d2011-07-07 08:21:25 +0000517} // namespace webrtc
518
Mirko Bonadei92ea95e2017-09-15 06:47:31 +0200519#endif // MODULES_AUDIO_PROCESSING_AUDIO_PROCESSING_IMPL_H_