blob: a95e15057338993cbdc9b9e5903b26d69639310b [file] [log] [blame]
niklase@google.com470e71d2011-07-07 08:21:25 +00001/*
andrew@webrtc.org6f9f8172012-03-06 19:03:39 +00002 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
niklase@google.com470e71d2011-07-07 08:21:25 +00003 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020011#ifndef MODULES_AUDIO_PROCESSING_AUDIO_PROCESSING_IMPL_H_
12#define MODULES_AUDIO_PROCESSING_AUDIO_PROCESSING_IMPL_H_
niklase@google.com470e71d2011-07-07 08:21:25 +000013
14#include <list>
kwiberg88788ad2016-02-19 07:04:49 -080015#include <memory>
Michael Graczyk86c6d332015-07-23 11:41:39 -070016#include <vector>
niklase@google.com470e71d2011-07-07 08:21:25 +000017
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020018#include "modules/audio_processing/audio_buffer.h"
19#include "modules/audio_processing/include/aec_dump.h"
20#include "modules/audio_processing/include/audio_processing.h"
21#include "modules/audio_processing/render_queue_item_verifier.h"
22#include "modules/audio_processing/rms_level.h"
23#include "rtc_base/criticalsection.h"
24#include "rtc_base/function_view.h"
25#include "rtc_base/gtest_prod_util.h"
26#include "rtc_base/ignore_wundef.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020027#include "rtc_base/swap_queue.h"
28#include "rtc_base/thread_annotations.h"
peahdf3efa82015-11-28 12:35:15 -080029
niklase@google.com470e71d2011-07-07 08:21:25 +000030namespace webrtc {
andrew@webrtc.orgddbb8a22014-04-22 21:00:04 +000031
Per Åhgren13735822018-02-12 21:42:56 +010032class ApmDataDumper;
ekmeyerson60d9b332015-08-14 10:35:55 -070033class AudioConverter;
Michael Graczykdfa36052015-03-25 16:37:27 -070034
niklase@google.com470e71d2011-07-07 08:21:25 +000035class AudioProcessingImpl : public AudioProcessing {
36 public:
peahdf3efa82015-11-28 12:35:15 -080037 // Methods forcing APM to run in a single-threaded manner.
38 // Acquires both the render and capture locks.
peah88ac8532016-09-12 16:47:25 -070039 explicit AudioProcessingImpl(const webrtc::Config& config);
Sam Zackrissondb389722018-06-21 10:12:24 +020040 // AudioProcessingImpl takes ownership of capture post processor.
peah88ac8532016-09-12 16:47:25 -070041 AudioProcessingImpl(const webrtc::Config& config,
Alex Loiko5825aa62017-12-18 16:02:40 +010042 std::unique_ptr<CustomProcessing> capture_post_processor,
43 std::unique_ptr<CustomProcessing> render_pre_processor,
Gustaf Ullberg002ef282017-10-12 15:13:17 +020044 std::unique_ptr<EchoControlFactory> echo_control_factory,
Valeriia Nemychnikovaf06eb572018-08-29 10:37:09 +020045 rtc::scoped_refptr<EchoDetector> echo_detector,
46 std::unique_ptr<CustomAudioAnalyzer> capture_analyzer);
kwiberg83ffe452016-08-29 14:46:07 -070047 ~AudioProcessingImpl() override;
kjellander@webrtc.org14665ff2015-03-04 12:58:35 +000048 int Initialize() override;
peahde65ddc2016-09-16 15:02:15 -070049 int Initialize(int capture_input_sample_rate_hz,
50 int capture_output_sample_rate_hz,
51 int render_sample_rate_hz,
52 ChannelLayout capture_input_layout,
53 ChannelLayout capture_output_layout,
54 ChannelLayout render_input_layout) override;
Michael Graczyk86c6d332015-07-23 11:41:39 -070055 int Initialize(const ProcessingConfig& processing_config) override;
peah88ac8532016-09-12 16:47:25 -070056 void ApplyConfig(const AudioProcessing::Config& config) override;
57 void SetExtraOptions(const webrtc::Config& config) override;
peahdf3efa82015-11-28 12:35:15 -080058 void UpdateHistogramsOnCallEnd() override;
aleloi868f32f2017-05-23 07:20:05 -070059 void AttachAecDump(std::unique_ptr<AecDump> aec_dump) override;
60 void DetachAecDump() override;
Sam Zackrisson4d364492018-03-02 16:03:21 +010061 void AttachPlayoutAudioGenerator(
62 std::unique_ptr<AudioGenerator> audio_generator) override;
63 void DetachPlayoutAudioGenerator() override;
peahdf3efa82015-11-28 12:35:15 -080064
Alessio Bazzicac054e782018-04-16 12:10:09 +020065 void SetRuntimeSetting(RuntimeSetting setting) override;
66
peahdf3efa82015-11-28 12:35:15 -080067 // Capture-side exclusive methods possibly running APM in a
68 // multi-threaded manner. Acquire the capture lock.
kjellander@webrtc.org14665ff2015-03-04 12:58:35 +000069 int ProcessStream(AudioFrame* frame) override;
70 int ProcessStream(const float* const* src,
Peter Kastingdce40cf2015-08-24 14:52:23 -070071 size_t samples_per_channel,
kjellander@webrtc.org14665ff2015-03-04 12:58:35 +000072 int input_sample_rate_hz,
73 ChannelLayout input_layout,
74 int output_sample_rate_hz,
75 ChannelLayout output_layout,
76 float* const* dest) override;
Michael Graczyk86c6d332015-07-23 11:41:39 -070077 int ProcessStream(const float* const* src,
78 const StreamConfig& input_config,
79 const StreamConfig& output_config,
80 float* const* dest) override;
peahdf3efa82015-11-28 12:35:15 -080081 void set_output_will_be_muted(bool muted) override;
82 int set_stream_delay_ms(int delay) override;
83 void set_delay_offset_ms(int offset) override;
84 int delay_offset_ms() const override;
85 void set_stream_key_pressed(bool key_pressed) override;
86
87 // Render-side exclusive methods possibly running APM in a
88 // multi-threaded manner. Acquire the render lock.
ekmeyerson60d9b332015-08-14 10:35:55 -070089 int ProcessReverseStream(AudioFrame* frame) override;
kjellander@webrtc.org14665ff2015-03-04 12:58:35 +000090 int AnalyzeReverseStream(const float* const* data,
Peter Kastingdce40cf2015-08-24 14:52:23 -070091 size_t samples_per_channel,
kjellander@webrtc.org14665ff2015-03-04 12:58:35 +000092 int sample_rate_hz,
93 ChannelLayout layout) override;
ekmeyerson60d9b332015-08-14 10:35:55 -070094 int ProcessReverseStream(const float* const* src,
peahde65ddc2016-09-16 15:02:15 -070095 const StreamConfig& input_config,
96 const StreamConfig& output_config,
ekmeyerson60d9b332015-08-14 10:35:55 -070097 float* const* dest) override;
peahdf3efa82015-11-28 12:35:15 -080098
99 // Methods only accessed from APM submodules or
100 // from AudioProcessing tests in a single-threaded manner.
101 // Hence there is no need for locks in these.
102 int proc_sample_rate_hz() const override;
103 int proc_split_sample_rate_hz() const override;
Peter Kasting69558702016-01-12 16:26:35 -0800104 size_t num_input_channels() const override;
105 size_t num_proc_channels() const override;
106 size_t num_output_channels() const override;
107 size_t num_reverse_channels() const override;
kjellander@webrtc.org14665ff2015-03-04 12:58:35 +0000108 int stream_delay_ms() const override;
peahdf3efa82015-11-28 12:35:15 -0800109 bool was_stream_delay_set() const override
danilchap56359be2017-09-07 07:53:45 -0700110 RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_capture_);
peahdf3efa82015-11-28 12:35:15 -0800111
ivoc3e9a5372016-10-28 07:55:33 -0700112 AudioProcessingStatistics GetStatistics() const override;
Ivo Creusenae026092017-11-20 13:07:16 +0100113 AudioProcessingStats GetStatistics(bool has_remote_tracks) const override;
ivoc3e9a5372016-10-28 07:55:33 -0700114
peahdf3efa82015-11-28 12:35:15 -0800115 // Methods returning pointers to APM submodules.
116 // No locks are aquired in those, as those locks
117 // would offer no protection (the submodules are
118 // created only once in a single-treaded manner
119 // during APM creation).
kjellander@webrtc.org14665ff2015-03-04 12:58:35 +0000120 EchoCancellation* echo_cancellation() const override;
121 EchoControlMobile* echo_control_mobile() const override;
122 GainControl* gain_control() const override;
peah8271d042016-11-22 07:24:52 -0800123 // TODO(peah): Deprecate this API call.
kjellander@webrtc.org14665ff2015-03-04 12:58:35 +0000124 HighPassFilter* high_pass_filter() const override;
125 LevelEstimator* level_estimator() const override;
126 NoiseSuppression* noise_suppression() const override;
127 VoiceDetection* voice_detection() const override;
niklase@google.com470e71d2011-07-07 08:21:25 +0000128
henrik.lundinadf06352017-04-05 05:48:24 -0700129 // TODO(peah): Remove MutateConfig once the new API allows that.
peah8271d042016-11-22 07:24:52 -0800130 void MutateConfig(rtc::FunctionView<void(AudioProcessing::Config*)> mutator);
henrik.lundinadf06352017-04-05 05:48:24 -0700131 AudioProcessing::Config GetConfig() const override;
peah8271d042016-11-22 07:24:52 -0800132
andrew@webrtc.org60730cf2014-01-07 17:45:09 +0000133 protected:
andrew@webrtc.orga8b97372014-03-10 22:26:12 +0000134 // Overridden in a mock.
peahdf3efa82015-11-28 12:35:15 -0800135 virtual int InitializeLocked()
danilchap56359be2017-09-07 07:53:45 -0700136 RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_render_, crit_capture_);
andrew@webrtc.org60730cf2014-01-07 17:45:09 +0000137
niklase@google.com470e71d2011-07-07 08:21:25 +0000138 private:
peahc19f3122016-10-07 14:54:10 -0700139 // TODO(peah): These friend classes should be removed as soon as the new
140 // parameter setting scheme allows.
141 FRIEND_TEST_ALL_PREFIXES(ApmConfiguration, DefaultBehavior);
142 FRIEND_TEST_ALL_PREFIXES(ApmConfiguration, ValidConfigBehavior);
143 FRIEND_TEST_ALL_PREFIXES(ApmConfiguration, InValidConfigBehavior);
Alessio Bazzicac054e782018-04-16 12:10:09 +0200144
145 // Class providing thread-safe message pipe functionality for
146 // |runtime_settings_|.
147 class RuntimeSettingEnqueuer {
148 public:
149 explicit RuntimeSettingEnqueuer(
150 SwapQueue<RuntimeSetting>* runtime_settings);
151 ~RuntimeSettingEnqueuer();
152 void Enqueue(RuntimeSetting setting);
153
154 private:
Alessio Bazzica33444dc2018-04-20 13:16:55 +0200155 SwapQueue<RuntimeSetting>& runtime_settings_;
Alex Loiko73ec0192018-05-15 10:52:28 +0200156 };
157 struct ApmPublicSubmodules;
158 struct ApmPrivateSubmodules;
159
160 std::unique_ptr<ApmDataDumper> data_dumper_;
161 static int instance_count_;
162
163 SwapQueue<RuntimeSetting> capture_runtime_settings_;
164 SwapQueue<RuntimeSetting> render_runtime_settings_;
165
166 RuntimeSettingEnqueuer capture_runtime_settings_enqueuer_;
167 RuntimeSettingEnqueuer render_runtime_settings_enqueuer_;
Alessio Bazzicac054e782018-04-16 12:10:09 +0200168
peah8271d042016-11-22 07:24:52 -0800169 // Submodule interface implementations.
170 std::unique_ptr<HighPassFilter> high_pass_filter_impl_;
171
Gustaf Ullberg002ef282017-10-12 15:13:17 +0200172 // EchoControl factory.
173 std::unique_ptr<EchoControlFactory> echo_control_factory_;
Gustaf Ullbergd8579e02017-10-11 16:29:02 +0200174
peah2ace3f92016-09-10 04:42:27 -0700175 class ApmSubmoduleStates {
176 public:
Alex Loiko5825aa62017-12-18 16:02:40 +0100177 ApmSubmoduleStates(bool capture_post_processor_enabled,
Valeriia Nemychnikovaf06eb572018-08-29 10:37:09 +0200178 bool render_pre_processor_enabled,
179 bool capture_analyzer_enabled);
peah2ace3f92016-09-10 04:42:27 -0700180 // Updates the submodule state and returns true if it has changed.
peah8271d042016-11-22 07:24:52 -0800181 bool Update(bool low_cut_filter_enabled,
peah2ace3f92016-09-10 04:42:27 -0700182 bool echo_canceller_enabled,
183 bool mobile_echo_controller_enabled,
ivoc9f4a4a02016-10-28 05:39:16 -0700184 bool residual_echo_detector_enabled,
peah2ace3f92016-09-10 04:42:27 -0700185 bool noise_suppressor_enabled,
peah2ace3f92016-09-10 04:42:27 -0700186 bool adaptive_gain_controller_enabled,
alessiob3ec96df2017-05-22 06:57:06 -0700187 bool gain_controller2_enabled,
Alex Loikob5c9a792018-04-16 16:31:22 +0200188 bool pre_amplifier_enabled,
Gustaf Ullbergce045ac2017-10-16 13:49:04 +0200189 bool echo_controller_enabled,
peah2ace3f92016-09-10 04:42:27 -0700190 bool voice_activity_detector_enabled,
191 bool level_estimator_enabled,
192 bool transient_suppressor_enabled);
193 bool CaptureMultiBandSubModulesActive() const;
194 bool CaptureMultiBandProcessingActive() const;
peah23ac8b42017-05-23 05:33:56 -0700195 bool CaptureFullBandProcessingActive() const;
Valeriia Nemychnikovaf06eb572018-08-29 10:37:09 +0200196 bool CaptureAnalyzerActive() const;
peah2ace3f92016-09-10 04:42:27 -0700197 bool RenderMultiBandSubModulesActive() const;
Alex Loiko5825aa62017-12-18 16:02:40 +0100198 bool RenderFullBandProcessingActive() const;
peah2ace3f92016-09-10 04:42:27 -0700199 bool RenderMultiBandProcessingActive() const;
200
201 private:
Sam Zackrisson0beac582017-09-25 12:04:02 +0200202 const bool capture_post_processor_enabled_ = false;
Alex Loiko5825aa62017-12-18 16:02:40 +0100203 const bool render_pre_processor_enabled_ = false;
Valeriia Nemychnikovaf06eb572018-08-29 10:37:09 +0200204 const bool capture_analyzer_enabled_ = false;
peah8271d042016-11-22 07:24:52 -0800205 bool low_cut_filter_enabled_ = false;
peah2ace3f92016-09-10 04:42:27 -0700206 bool echo_canceller_enabled_ = false;
207 bool mobile_echo_controller_enabled_ = false;
ivoc9f4a4a02016-10-28 05:39:16 -0700208 bool residual_echo_detector_enabled_ = false;
peah2ace3f92016-09-10 04:42:27 -0700209 bool noise_suppressor_enabled_ = false;
peah2ace3f92016-09-10 04:42:27 -0700210 bool adaptive_gain_controller_enabled_ = false;
alessiob3ec96df2017-05-22 06:57:06 -0700211 bool gain_controller2_enabled_ = false;
Alex Loikob5c9a792018-04-16 16:31:22 +0200212 bool pre_amplifier_enabled_ = false;
Gustaf Ullbergce045ac2017-10-16 13:49:04 +0200213 bool echo_controller_enabled_ = false;
peah2ace3f92016-09-10 04:42:27 -0700214 bool level_estimator_enabled_ = false;
215 bool voice_activity_detector_enabled_ = false;
216 bool transient_suppressor_enabled_ = false;
217 bool first_update_ = true;
218 };
219
peahdf3efa82015-11-28 12:35:15 -0800220 // Method for modifying the formats struct that are called from both
221 // the render and capture threads. The check for whether modifications
222 // are needed is done while holding the render lock only, thereby avoiding
223 // that the capture thread blocks the render thread.
224 // The struct is modified in a single-threaded manner by holding both the
225 // render and capture locks.
peah2ace3f92016-09-10 04:42:27 -0700226 int MaybeInitialize(const ProcessingConfig& config, bool force_initialization)
danilchap56359be2017-09-07 07:53:45 -0700227 RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_render_);
peahdf3efa82015-11-28 12:35:15 -0800228
229 int MaybeInitializeRender(const ProcessingConfig& processing_config)
danilchap56359be2017-09-07 07:53:45 -0700230 RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_render_);
peahdf3efa82015-11-28 12:35:15 -0800231
peah2ace3f92016-09-10 04:42:27 -0700232 int MaybeInitializeCapture(const ProcessingConfig& processing_config,
233 bool force_initialization)
danilchap56359be2017-09-07 07:53:45 -0700234 RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_render_);
peahdf3efa82015-11-28 12:35:15 -0800235
peah2ace3f92016-09-10 04:42:27 -0700236 // Method for updating the state keeping track of the active submodules.
237 // Returns a bool indicating whether the state has changed.
danilchap56359be2017-09-07 07:53:45 -0700238 bool UpdateActiveSubmoduleStates()
239 RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_capture_);
peahdf3efa82015-11-28 12:35:15 -0800240
241 // Methods requiring APM running in a single-threaded manner.
242 // Are called with both the render and capture locks already
243 // acquired.
peahdf3efa82015-11-28 12:35:15 -0800244 void InitializeTransient()
danilchap56359be2017-09-07 07:53:45 -0700245 RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_render_, crit_capture_);
Michael Graczyk86c6d332015-07-23 11:41:39 -0700246 int InitializeLocked(const ProcessingConfig& config)
danilchap56359be2017-09-07 07:53:45 -0700247 RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_render_, crit_capture_);
ivoc9f4a4a02016-10-28 05:39:16 -0700248 void InitializeResidualEchoDetector()
danilchap56359be2017-09-07 07:53:45 -0700249 RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_render_, crit_capture_);
250 void InitializeLowCutFilter() RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_capture_);
Gustaf Ullberg8eb9c7d2017-10-14 08:28:46 +0200251 void InitializeEchoController() RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_capture_);
Sam Zackrisson0beac582017-09-25 12:04:02 +0200252 void InitializeGainController2() RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_capture_);
Alex Loikob5c9a792018-04-16 16:31:22 +0200253 void InitializePreAmplifier() RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_capture_);
Sam Zackrisson0beac582017-09-25 12:04:02 +0200254 void InitializePostProcessor() RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_capture_);
Valeriia Nemychnikovaf06eb572018-08-29 10:37:09 +0200255 void InitializeAnalyzer() RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_capture_);
Alex Loiko5825aa62017-12-18 16:02:40 +0100256 void InitializePreProcessor() RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_render_);
peahdf3efa82015-11-28 12:35:15 -0800257
Alex Loiko73ec0192018-05-15 10:52:28 +0200258 // Empties and handles the respective RuntimeSetting queues.
259 void HandleCaptureRuntimeSettings()
260 RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_capture_);
261 void HandleRenderRuntimeSettings() RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_render_);
Alessio Bazzicac054e782018-04-16 12:10:09 +0200262
peah764e3642016-10-22 05:04:30 -0700263 void EmptyQueuedRenderAudio();
264 void AllocateRenderQueue()
danilchap56359be2017-09-07 07:53:45 -0700265 RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_render_, crit_capture_);
peah9e6a2902017-05-15 07:19:21 -0700266 void QueueBandedRenderAudio(AudioBuffer* audio)
danilchap56359be2017-09-07 07:53:45 -0700267 RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_render_);
peah9e6a2902017-05-15 07:19:21 -0700268 void QueueNonbandedRenderAudio(AudioBuffer* audio)
danilchap56359be2017-09-07 07:53:45 -0700269 RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_render_);
peah764e3642016-10-22 05:04:30 -0700270
peahdf3efa82015-11-28 12:35:15 -0800271 // Capture-side exclusive methods possibly running APM in a multi-threaded
272 // manner that are called with the render lock already acquired.
danilchap56359be2017-09-07 07:53:45 -0700273 int ProcessCaptureStreamLocked() RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_capture_);
274 void MaybeUpdateHistograms() RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_capture_);
peahdf3efa82015-11-28 12:35:15 -0800275
276 // Render-side exclusive methods possibly running APM in a multi-threaded
277 // manner that are called with the render lock already acquired.
ekmeyerson60d9b332015-08-14 10:35:55 -0700278 // TODO(ekm): Remove once all clients updated to new interface.
peahdf3efa82015-11-28 12:35:15 -0800279 int AnalyzeReverseStreamLocked(const float* const* src,
280 const StreamConfig& input_config,
281 const StreamConfig& output_config)
danilchap56359be2017-09-07 07:53:45 -0700282 RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_render_);
283 int ProcessRenderStreamLocked() RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_render_);
andrew@webrtc.org17e40642014-03-04 20:58:13 +0000284
aleloi868f32f2017-05-23 07:20:05 -0700285 // Collects configuration settings from public and private
286 // submodules to be saved as an audioproc::Config message on the
287 // AecDump if it is attached. If not |forced|, only writes the current
288 // config if it is different from the last saved one; if |forced|,
289 // writes the config regardless of the last saved.
290 void WriteAecDumpConfigMessage(bool forced)
danilchap56359be2017-09-07 07:53:45 -0700291 RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_capture_);
aleloi868f32f2017-05-23 07:20:05 -0700292
293 // Notifies attached AecDump of current configuration and capture data.
294 void RecordUnprocessedCaptureStream(const float* const* capture_stream)
danilchap56359be2017-09-07 07:53:45 -0700295 RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_capture_);
aleloi868f32f2017-05-23 07:20:05 -0700296
297 void RecordUnprocessedCaptureStream(const AudioFrame& capture_frame)
danilchap56359be2017-09-07 07:53:45 -0700298 RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_capture_);
aleloi868f32f2017-05-23 07:20:05 -0700299
300 // Notifies attached AecDump of current configuration and
301 // processed capture data and issues a capture stream recording
302 // request.
303 void RecordProcessedCaptureStream(
304 const float* const* processed_capture_stream)
danilchap56359be2017-09-07 07:53:45 -0700305 RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_capture_);
aleloi868f32f2017-05-23 07:20:05 -0700306
307 void RecordProcessedCaptureStream(const AudioFrame& processed_capture_frame)
danilchap56359be2017-09-07 07:53:45 -0700308 RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_capture_);
aleloi868f32f2017-05-23 07:20:05 -0700309
310 // Notifies attached AecDump about current state (delay, drift, etc).
danilchap56359be2017-09-07 07:53:45 -0700311 void RecordAudioProcessingState() RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_capture_);
aleloi868f32f2017-05-23 07:20:05 -0700312
aleloi868f32f2017-05-23 07:20:05 -0700313 // AecDump instance used for optionally logging APM config, input
314 // and output to file in the AEC-dump format defined in debug.proto.
315 std::unique_ptr<AecDump> aec_dump_;
316
317 // Hold the last config written with AecDump for avoiding writing
318 // the same config twice.
danilchap56359be2017-09-07 07:53:45 -0700319 InternalAPMConfig apm_config_for_aec_dump_ RTC_GUARDED_BY(crit_capture_);
aleloi868f32f2017-05-23 07:20:05 -0700320
peahdf3efa82015-11-28 12:35:15 -0800321 // Critical sections.
danilchap56359be2017-09-07 07:53:45 -0700322 rtc::CriticalSection crit_render_ RTC_ACQUIRED_BEFORE(crit_capture_);
pbos5ad935c2016-01-25 03:52:44 -0800323 rtc::CriticalSection crit_capture_;
peahdf3efa82015-11-28 12:35:15 -0800324
peahc19f3122016-10-07 14:54:10 -0700325 // Struct containing the Config specifying the behavior of APM.
326 AudioProcessing::Config config_;
327
peah2ace3f92016-09-10 04:42:27 -0700328 // Class containing information about what submodules are active.
329 ApmSubmoduleStates submodule_states_;
330
peahdf3efa82015-11-28 12:35:15 -0800331 // Structs containing the pointers to the submodules.
kwiberg88788ad2016-02-19 07:04:49 -0800332 std::unique_ptr<ApmPublicSubmodules> public_submodules_;
ivoc9f4a4a02016-10-28 05:39:16 -0700333 std::unique_ptr<ApmPrivateSubmodules> private_submodules_;
peahdf3efa82015-11-28 12:35:15 -0800334
peah192164e2015-11-17 02:16:45 -0800335 // State that is written to while holding both the render and capture locks
peahdf3efa82015-11-28 12:35:15 -0800336 // but can be read without any lock being held.
337 // As this is only accessed internally of APM, and all internal methods in APM
338 // either are holding the render or capture locks, this construct is safe as
339 // it is not possible to read the variables while writing them.
340 struct ApmFormatState {
341 ApmFormatState()
peah192164e2015-11-17 02:16:45 -0800342 : // Format of processing streams at input/output call sites.
peahdf3efa82015-11-28 12:35:15 -0800343 api_format({{{kSampleRate16kHz, 1, false},
344 {kSampleRate16kHz, 1, false},
345 {kSampleRate16kHz, 1, false},
346 {kSampleRate16kHz, 1, false}}}),
peahde65ddc2016-09-16 15:02:15 -0700347 render_processing_format(kSampleRate16kHz, 1) {}
peahdf3efa82015-11-28 12:35:15 -0800348 ProcessingConfig api_format;
peahde65ddc2016-09-16 15:02:15 -0700349 StreamConfig render_processing_format;
peahdf3efa82015-11-28 12:35:15 -0800350 } formats_;
Michael Graczyk86c6d332015-07-23 11:41:39 -0700351
peahdf3efa82015-11-28 12:35:15 -0800352 // APM constants.
353 const struct ApmConstants {
henrik.lundinbd681b92016-12-05 09:08:42 -0800354 ApmConstants(int agc_startup_min_volume,
355 int agc_clipped_level_min,
Alex Loiko64cb83b2018-07-02 13:38:19 +0200356 bool use_experimental_agc,
357 bool use_experimental_agc_agc2_level_estimation,
358 bool use_experimental_agc_agc2_digital_adaptive)
peahdf3efa82015-11-28 12:35:15 -0800359 : // Format of processing streams at input/output call sites.
360 agc_startup_min_volume(agc_startup_min_volume),
henrik.lundinbd681b92016-12-05 09:08:42 -0800361 agc_clipped_level_min(agc_clipped_level_min),
Alex Loiko64cb83b2018-07-02 13:38:19 +0200362 use_experimental_agc(use_experimental_agc),
363 use_experimental_agc_agc2_level_estimation(
364 use_experimental_agc_agc2_level_estimation),
365 use_experimental_agc_agc2_digital_adaptive(
366 use_experimental_agc_agc2_digital_adaptive) {}
peahdf3efa82015-11-28 12:35:15 -0800367 int agc_startup_min_volume;
henrik.lundinbd681b92016-12-05 09:08:42 -0800368 int agc_clipped_level_min;
peahbe615622016-02-13 16:40:47 -0800369 bool use_experimental_agc;
Alex Loiko64cb83b2018-07-02 13:38:19 +0200370 bool use_experimental_agc_agc2_level_estimation;
371 bool use_experimental_agc_agc2_digital_adaptive;
372
peahdf3efa82015-11-28 12:35:15 -0800373 } constants_;
andrew@webrtc.orgddbb8a22014-04-22 21:00:04 +0000374
peahdf3efa82015-11-28 12:35:15 -0800375 struct ApmCaptureState {
Sam Zackrisson9394f6f2018-06-14 10:11:35 +0200376 ApmCaptureState(bool transient_suppressor_enabled);
kwiberg83ffe452016-08-29 14:46:07 -0700377 ~ApmCaptureState();
peahdf3efa82015-11-28 12:35:15 -0800378 int aec_system_delay_jumps;
379 int delay_offset_ms;
380 bool was_stream_delay_set;
381 int last_stream_delay_ms;
382 int last_aec_system_delay_ms;
383 int stream_delay_jumps;
384 bool output_will_be_muted;
385 bool key_pressed;
386 bool transient_suppressor_enabled;
kwiberg88788ad2016-02-19 07:04:49 -0800387 std::unique_ptr<AudioBuffer> capture_audio;
peahde65ddc2016-09-16 15:02:15 -0700388 // Only the rate and samples fields of capture_processing_format_ are used
389 // because the capture processing number of channels is mutable and is
390 // tracked by the capture_audio_.
391 StreamConfig capture_processing_format;
peahdf3efa82015-11-28 12:35:15 -0800392 int split_rate;
peah67995532017-04-10 14:12:41 -0700393 bool echo_path_gain_change;
Per Åhgren88cf0502018-07-16 17:08:41 +0200394 int prev_analog_mic_level;
danilchap56359be2017-09-07 07:53:45 -0700395 } capture_ RTC_GUARDED_BY(crit_capture_);
niklase@google.com470e71d2011-07-07 08:21:25 +0000396
peahdf3efa82015-11-28 12:35:15 -0800397 struct ApmCaptureNonLockedState {
Alessio Bazzicacc22f512018-08-30 13:01:34 +0200398 ApmCaptureNonLockedState()
peahde65ddc2016-09-16 15:02:15 -0700399 : capture_processing_format(kSampleRate16kHz),
peahdf3efa82015-11-28 12:35:15 -0800400 split_rate(kSampleRate16kHz),
Alessio Bazzicacc22f512018-08-30 13:01:34 +0200401 stream_delay_ms(0) {}
peahde65ddc2016-09-16 15:02:15 -0700402 // Only the rate and samples fields of capture_processing_format_ are used
403 // because the forward processing number of channels is mutable and is
404 // tracked by the capture_audio_.
405 StreamConfig capture_processing_format;
peahdf3efa82015-11-28 12:35:15 -0800406 int split_rate;
407 int stream_delay_ms;
Gustaf Ullbergce045ac2017-10-16 13:49:04 +0200408 bool echo_controller_enabled = false;
peahdf3efa82015-11-28 12:35:15 -0800409 } capture_nonlocked_;
andrew@webrtc.org75dd2882014-02-11 20:52:30 +0000410
peahdf3efa82015-11-28 12:35:15 -0800411 struct ApmRenderState {
kwiberg83ffe452016-08-29 14:46:07 -0700412 ApmRenderState();
413 ~ApmRenderState();
kwiberg88788ad2016-02-19 07:04:49 -0800414 std::unique_ptr<AudioConverter> render_converter;
415 std::unique_ptr<AudioBuffer> render_audio;
danilchap56359be2017-09-07 07:53:45 -0700416 } render_ RTC_GUARDED_BY(crit_render_);
peah764e3642016-10-22 05:04:30 -0700417
danilchap56359be2017-09-07 07:53:45 -0700418 size_t aec_render_queue_element_max_size_ RTC_GUARDED_BY(crit_render_)
419 RTC_GUARDED_BY(crit_capture_) = 0;
420 std::vector<float> aec_render_queue_buffer_ RTC_GUARDED_BY(crit_render_);
421 std::vector<float> aec_capture_queue_buffer_ RTC_GUARDED_BY(crit_capture_);
peaha0624602016-10-25 04:45:24 -0700422
danilchap56359be2017-09-07 07:53:45 -0700423 size_t aecm_render_queue_element_max_size_ RTC_GUARDED_BY(crit_render_)
424 RTC_GUARDED_BY(crit_capture_) = 0;
425 std::vector<int16_t> aecm_render_queue_buffer_ RTC_GUARDED_BY(crit_render_);
426 std::vector<int16_t> aecm_capture_queue_buffer_ RTC_GUARDED_BY(crit_capture_);
peah701d6282016-10-25 05:42:20 -0700427
danilchap56359be2017-09-07 07:53:45 -0700428 size_t agc_render_queue_element_max_size_ RTC_GUARDED_BY(crit_render_)
429 RTC_GUARDED_BY(crit_capture_) = 0;
430 std::vector<int16_t> agc_render_queue_buffer_ RTC_GUARDED_BY(crit_render_);
431 std::vector<int16_t> agc_capture_queue_buffer_ RTC_GUARDED_BY(crit_capture_);
peah764e3642016-10-22 05:04:30 -0700432
danilchap56359be2017-09-07 07:53:45 -0700433 size_t red_render_queue_element_max_size_ RTC_GUARDED_BY(crit_render_)
434 RTC_GUARDED_BY(crit_capture_) = 0;
435 std::vector<float> red_render_queue_buffer_ RTC_GUARDED_BY(crit_render_);
436 std::vector<float> red_capture_queue_buffer_ RTC_GUARDED_BY(crit_capture_);
ivoc9f4a4a02016-10-28 05:39:16 -0700437
danilchap56359be2017-09-07 07:53:45 -0700438 RmsLevel capture_input_rms_ RTC_GUARDED_BY(crit_capture_);
439 RmsLevel capture_output_rms_ RTC_GUARDED_BY(crit_capture_);
440 int capture_rms_interval_counter_ RTC_GUARDED_BY(crit_capture_) = 0;
henrik.lundin290d43a2016-11-29 08:09:09 -0800441
peah764e3642016-10-22 05:04:30 -0700442 // Lock protection not needed.
443 std::unique_ptr<SwapQueue<std::vector<float>, RenderQueueItemVerifier<float>>>
peah701d6282016-10-25 05:42:20 -0700444 aec_render_signal_queue_;
peaha0624602016-10-25 04:45:24 -0700445 std::unique_ptr<
446 SwapQueue<std::vector<int16_t>, RenderQueueItemVerifier<int16_t>>>
peah701d6282016-10-25 05:42:20 -0700447 aecm_render_signal_queue_;
448 std::unique_ptr<
449 SwapQueue<std::vector<int16_t>, RenderQueueItemVerifier<int16_t>>>
450 agc_render_signal_queue_;
ivoc9f4a4a02016-10-28 05:39:16 -0700451 std::unique_ptr<SwapQueue<std::vector<float>, RenderQueueItemVerifier<float>>>
452 red_render_signal_queue_;
niklase@google.com470e71d2011-07-07 08:21:25 +0000453};
andrew@webrtc.orgddbb8a22014-04-22 21:00:04 +0000454
niklase@google.com470e71d2011-07-07 08:21:25 +0000455} // namespace webrtc
456
Mirko Bonadei92ea95e2017-09-15 06:47:31 +0200457#endif // MODULES_AUDIO_PROCESSING_AUDIO_PROCESSING_IMPL_H_