blob: 9935167750468f6f6620c3ea22a12e047f2cda58 [file] [log] [blame]
niklase@google.com470e71d2011-07-07 08:21:25 +00001/*
andrew@webrtc.org648af742012-02-08 01:57:29 +00002 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
niklase@google.com470e71d2011-07-07 08:21:25 +00003 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020011#ifndef MODULES_AUDIO_PROCESSING_INCLUDE_AUDIO_PROCESSING_H_
12#define MODULES_AUDIO_PROCESSING_INCLUDE_AUDIO_PROCESSING_H_
niklase@google.com470e71d2011-07-07 08:21:25 +000013
Alejandro Luebscb3f9bd2015-10-29 18:21:34 -070014// MSVC++ requires this to be set before any other includes to get M_PI.
Patrik Höglund3ff90f12017-12-12 14:41:53 +010015#ifndef _USE_MATH_DEFINES
Alejandro Luebscb3f9bd2015-10-29 18:21:34 -070016#define _USE_MATH_DEFINES
Patrik Höglund3ff90f12017-12-12 14:41:53 +010017#endif
Alejandro Luebscb3f9bd2015-10-29 18:21:34 -070018
19#include <math.h>
andrew@webrtc.orgd72b3d62012-11-15 21:46:06 +000020#include <stddef.h> // size_t
Yves Gerey665174f2018-06-19 15:03:05 +020021#include <stdio.h> // FILE
peah8cee56f2017-08-24 22:36:53 -070022#include <string.h>
Jonas Olssona4d87372019-07-05 19:08:33 +020023
aluebs@webrtc.orgfb7a0392015-01-05 21:58:58 +000024#include <vector>
ajm@google.com22e65152011-07-18 18:03:01 +000025
Ali Tofigh1fa87c42022-07-25 22:07:08 +020026#include "absl/strings/string_view.h"
Danil Chapovalovdb9f7ab2018-06-19 10:50:11 +020027#include "absl/types/optional.h"
Sam Zackrissonab866a22020-05-07 13:07:49 +020028#include "api/array_view.h"
Gustaf Ullbergbffa3002018-02-14 15:12:00 +010029#include "api/audio/echo_canceller3_config.h"
Gustaf Ullbergfd4ce502018-02-15 10:09:09 +010030#include "api/audio/echo_control.h"
Mirko Bonadeid9708072019-01-25 20:26:48 +010031#include "api/scoped_refptr.h"
Ivo Creusen56d46092017-11-24 17:29:59 +010032#include "modules/audio_processing/include/audio_processing_statistics.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020033#include "rtc_base/arraysize.h"
Steve Anton10542f22019-01-11 09:11:00 -080034#include "rtc_base/ref_count.h"
Per Åhgren09e9a832020-05-11 11:03:47 +020035#include "rtc_base/system/file_wrapper.h"
Mirko Bonadei3d255302018-10-11 10:50:45 +020036#include "rtc_base/system/rtc_export.h"
niklase@google.com470e71d2011-07-07 08:21:25 +000037
Per Åhgren09e9a832020-05-11 11:03:47 +020038namespace rtc {
39class TaskQueue;
40} // namespace rtc
41
niklase@google.com470e71d2011-07-07 08:21:25 +000042namespace webrtc {
43
aleloi868f32f2017-05-23 07:20:05 -070044class AecDump;
Sam Zackrisson0beac582017-09-25 12:04:02 +020045class AudioBuffer;
Michael Graczykdfa36052015-03-25 16:37:27 -070046
Michael Graczyk86c6d332015-07-23 11:41:39 -070047class StreamConfig;
48class ProcessingConfig;
49
Ivo Creusen09fa4b02018-01-11 16:08:54 +010050class EchoDetector;
Valeriia Nemychnikovaf06eb572018-08-29 10:37:09 +020051class CustomAudioAnalyzer;
Alex Loiko5825aa62017-12-18 16:02:40 +010052class CustomProcessing;
niklase@google.com470e71d2011-07-07 08:21:25 +000053
54// The Audio Processing Module (APM) provides a collection of voice processing
55// components designed for real-time communications software.
56//
57// APM operates on two audio streams on a frame-by-frame basis. Frames of the
58// primary stream, on which all processing is applied, are passed to
Artem Titov0b489302021-07-28 20:50:03 +020059// `ProcessStream()`. Frames of the reverse direction stream are passed to
60// `ProcessReverseStream()`. On the client-side, this will typically be the
aluebsb0319552016-03-17 20:39:53 -070061// near-end (capture) and far-end (render) streams, respectively. APM should be
62// placed in the signal chain as close to the audio hardware abstraction layer
63// (HAL) as possible.
niklase@google.com470e71d2011-07-07 08:21:25 +000064//
65// On the server-side, the reverse stream will normally not be used, with
66// processing occurring on each incoming stream.
67//
68// Component interfaces follow a similar pattern and are accessed through
69// corresponding getters in APM. All components are disabled at create-time,
70// with default settings that are recommended for most situations. New settings
71// can be applied without enabling a component. Enabling a component triggers
72// memory allocation and initialization to allow it to start processing the
73// streams.
74//
75// Thread safety is provided with the following assumptions to reduce locking
76// overhead:
77// 1. The stream getters and setters are called from the same thread as
78// ProcessStream(). More precisely, stream functions are never called
79// concurrently with ProcessStream().
80// 2. Parameter getters are never called concurrently with the corresponding
81// setter.
82//
Sam Zackrisson3bd444f2022-08-03 14:37:00 +020083// APM accepts only linear PCM audio data in chunks of ~10 ms (see
Sam Zackrisson5dd54822022-11-17 11:26:58 +010084// AudioProcessing::GetFrameSize() for details) and sample rates ranging from
85// 8000 Hz to 384000 Hz. The int16 interfaces use interleaved data, while the
86// float interfaces use deinterleaved data.
niklase@google.com470e71d2011-07-07 08:21:25 +000087//
88// Usage example, omitting error checking:
Sam Zackrisson5dd54822022-11-17 11:26:58 +010089// rtc::scoped_refptr<AudioProcessing> apm = AudioProcessingBuilder().Create();
niklase@google.com470e71d2011-07-07 08:21:25 +000090//
peah88ac8532016-09-12 16:47:25 -070091// AudioProcessing::Config config;
Sam Zackrissoncdf0e6d2018-09-17 11:05:17 +020092// config.echo_canceller.enabled = true;
93// config.echo_canceller.mobile_mode = false;
Sam Zackrisson41478c72019-10-15 10:10:26 +020094//
95// config.gain_controller1.enabled = true;
96// config.gain_controller1.mode =
97// AudioProcessing::Config::GainController1::kAdaptiveAnalog;
98// config.gain_controller1.analog_level_minimum = 0;
99// config.gain_controller1.analog_level_maximum = 255;
100//
Sam Zackrissonab1aee02018-03-05 15:59:06 +0100101// config.gain_controller2.enabled = true;
Sam Zackrisson41478c72019-10-15 10:10:26 +0200102//
103// config.high_pass_filter.enabled = true;
104//
peah88ac8532016-09-12 16:47:25 -0700105// apm->ApplyConfig(config)
106//
niklase@google.com470e71d2011-07-07 08:21:25 +0000107// // Start a voice call...
108//
109// // ... Render frame arrives bound for the audio HAL ...
aluebsb0319552016-03-17 20:39:53 -0700110// apm->ProcessReverseStream(render_frame);
niklase@google.com470e71d2011-07-07 08:21:25 +0000111//
112// // ... Capture frame arrives from the audio HAL ...
113// // Call required set_stream_ functions.
114// apm->set_stream_delay_ms(delay_ms);
Sam Zackrisson41478c72019-10-15 10:10:26 +0200115// apm->set_stream_analog_level(analog_level);
niklase@google.com470e71d2011-07-07 08:21:25 +0000116//
117// apm->ProcessStream(capture_frame);
118//
119// // Call required stream_ functions.
Sam Zackrisson41478c72019-10-15 10:10:26 +0200120// analog_level = apm->recommended_stream_analog_level();
niklase@google.com470e71d2011-07-07 08:21:25 +0000121// has_voice = apm->stream_has_voice();
122//
Hua, Chunboe61a40e2021-01-08 16:34:49 +0800123// // Repeat render and capture processing for the duration of the call...
niklase@google.com470e71d2011-07-07 08:21:25 +0000124// // Start a new call...
125// apm->Initialize();
126//
127// // Close the application...
Sam Zackrisson5dd54822022-11-17 11:26:58 +0100128// apm.reset();
niklase@google.com470e71d2011-07-07 08:21:25 +0000129//
Mirko Bonadei35214fc2019-09-23 14:54:28 +0200130class RTC_EXPORT AudioProcessing : public rtc::RefCountInterface {
niklase@google.com470e71d2011-07-07 08:21:25 +0000131 public:
peah88ac8532016-09-12 16:47:25 -0700132 // The struct below constitutes the new parameter scheme for the audio
133 // processing. It is being introduced gradually and until it is fully
134 // introduced, it is prone to change.
135 // TODO(peah): Remove this comment once the new config scheme is fully rolled
136 // out.
137 //
138 // The parameters and behavior of the audio processing module are controlled
139 // by changing the default values in the AudioProcessing::Config struct.
140 // The config is applied by passing the struct to the ApplyConfig method.
Sam Zackrissonf0d1c032019-03-27 13:28:08 +0100141 //
142 // This config is intended to be used during setup, and to enable/disable
143 // top-level processing effects. Use during processing may cause undesired
144 // submodule resets, affecting the audio quality. Use the RuntimeSetting
145 // construct for runtime configuration.
Mirko Bonadeid4002a72019-11-12 20:11:48 +0100146 struct RTC_EXPORT Config {
Per Åhgrenfcbe4072019-09-15 00:27:58 +0200147 // Sets the properties of the audio processing pipeline.
Mirko Bonadeid4002a72019-11-12 20:11:48 +0100148 struct RTC_EXPORT Pipeline {
Alessio Bazzica504bd592022-12-01 13:26:26 +0100149 // Ways to downmix a multi-channel track to mono.
150 enum class DownmixMethod {
151 kAverageChannels, // Average across channels.
152 kUseFirstChannel // Use the first channel.
153 };
154
Per Åhgrenfcbe4072019-09-15 00:27:58 +0200155 // Maximum allowed processing rate used internally. May only be set to
Per Åhgren68c225d2021-01-21 23:03:32 +0100156 // 32000 or 48000 and any differing values will be treated as 48000.
157 int maximum_internal_processing_rate = 48000;
Per Åhgrene14cb992019-11-27 09:34:22 +0100158 // Allow multi-channel processing of render audio.
159 bool multi_channel_render = false;
160 // Allow multi-channel processing of capture audio when AEC3 is active
161 // or a custom AEC is injected..
162 bool multi_channel_capture = false;
Alessio Bazzica504bd592022-12-01 13:26:26 +0100163 // Indicates how to downmix multi-channel capture audio to mono (when
164 // needed).
165 DownmixMethod capture_downmix_method = DownmixMethod::kAverageChannels;
Per Åhgrenfcbe4072019-09-15 00:27:58 +0200166 } pipeline;
167
Sam Zackrisson23513132019-01-11 15:10:32 +0100168 // Enabled the pre-amplifier. It amplifies the capture signal
169 // before any other processing is done.
Per Åhgrendb5d7282021-03-15 16:31:04 +0000170 // TODO(webrtc:5298): Deprecate and use the pre-gain functionality in
171 // capture_level_adjustment instead.
Sam Zackrisson23513132019-01-11 15:10:32 +0100172 struct PreAmplifier {
173 bool enabled = false;
Alessio Bazzica841d74e2021-03-31 15:04:03 +0200174 float fixed_gain_factor = 1.0f;
Sam Zackrisson23513132019-01-11 15:10:32 +0100175 } pre_amplifier;
176
Per Åhgrendb5d7282021-03-15 16:31:04 +0000177 // Functionality for general level adjustment in the capture pipeline. This
178 // should not be used together with the legacy PreAmplifier functionality.
179 struct CaptureLevelAdjustment {
180 bool operator==(const CaptureLevelAdjustment& rhs) const;
181 bool operator!=(const CaptureLevelAdjustment& rhs) const {
182 return !(*this == rhs);
183 }
184 bool enabled = false;
185 // The `pre_gain_factor` scales the signal before any processing is done.
Alessio Bazzica841d74e2021-03-31 15:04:03 +0200186 float pre_gain_factor = 1.0f;
Per Åhgrendb5d7282021-03-15 16:31:04 +0000187 // The `post_gain_factor` scales the signal after all processing is done.
Alessio Bazzica841d74e2021-03-31 15:04:03 +0200188 float post_gain_factor = 1.0f;
Per Åhgrendb5d7282021-03-15 16:31:04 +0000189 struct AnalogMicGainEmulation {
190 bool operator==(const AnalogMicGainEmulation& rhs) const;
191 bool operator!=(const AnalogMicGainEmulation& rhs) const {
192 return !(*this == rhs);
193 }
194 bool enabled = false;
195 // Initial analog gain level to use for the emulated analog gain. Must
196 // be in the range [0...255].
197 int initial_level = 255;
198 } analog_mic_gain_emulation;
199 } capture_level_adjustment;
200
Sam Zackrisson23513132019-01-11 15:10:32 +0100201 struct HighPassFilter {
202 bool enabled = false;
Per Åhgrenc0424252019-12-10 13:04:15 +0100203 bool apply_in_full_band = true;
Sam Zackrisson23513132019-01-11 15:10:32 +0100204 } high_pass_filter;
205
Sam Zackrisson8b5d2cc2018-07-27 13:27:23 +0200206 struct EchoCanceller {
207 bool enabled = false;
208 bool mobile_mode = false;
Per Åhgrenc20a19c2019-11-13 11:12:29 +0100209 bool export_linear_aec_output = false;
Per Åhgrenb8106462019-12-04 08:34:12 +0100210 // Enforce the highpass filter to be on (has no effect for the mobile
211 // mode).
Per Åhgrenbcce4532019-12-03 13:52:40 +0100212 bool enforce_high_pass_filtering = true;
Sam Zackrisson8b5d2cc2018-07-27 13:27:23 +0200213 } echo_canceller;
214
Sam Zackrisson23513132019-01-11 15:10:32 +0100215 // Enables background noise suppression.
216 struct NoiseSuppression {
peah8271d042016-11-22 07:24:52 -0800217 bool enabled = false;
Sam Zackrisson23513132019-01-11 15:10:32 +0100218 enum Level { kLow, kModerate, kHigh, kVeryHigh };
219 Level level = kModerate;
Per Åhgren2e8e1c62019-12-20 00:42:22 +0100220 bool analyze_linear_aec_output_when_available = false;
Sam Zackrisson23513132019-01-11 15:10:32 +0100221 } noise_suppression;
peahe0eae3c2016-12-14 01:16:23 -0800222
Per Åhgrenc0734712020-01-02 15:15:36 +0100223 // Enables transient suppression.
224 struct TransientSuppression {
225 bool enabled = false;
226 } transient_suppression;
227
Sam Zackrissonf0d1c032019-03-27 13:28:08 +0100228 // Enables automatic gain control (AGC) functionality.
229 // The automatic gain control (AGC) component brings the signal to an
230 // appropriate range. This is done by applying a digital gain directly and,
231 // in the analog mode, prescribing an analog gain to be applied at the audio
232 // HAL.
233 // Recommended to be enabled on the client-side.
Alessio Bazzicadfc11d52021-05-07 11:58:11 +0200234 struct RTC_EXPORT GainController1 {
Alessio Bazzica3438a932020-10-14 12:47:50 +0200235 bool operator==(const GainController1& rhs) const;
236 bool operator!=(const GainController1& rhs) const {
237 return !(*this == rhs);
238 }
239
Sam Zackrissonf0d1c032019-03-27 13:28:08 +0100240 bool enabled = false;
241 enum Mode {
242 // Adaptive mode intended for use if an analog volume control is
243 // available on the capture device. It will require the user to provide
244 // coupling between the OS mixer controls and AGC through the
245 // stream_analog_level() functions.
246 // It consists of an analog gain prescription for the audio device and a
247 // digital compression stage.
248 kAdaptiveAnalog,
249 // Adaptive mode intended for situations in which an analog volume
250 // control is unavailable. It operates in a similar fashion to the
251 // adaptive analog mode, but with scaling instead applied in the digital
252 // domain. As with the analog mode, it additionally uses a digital
253 // compression stage.
254 kAdaptiveDigital,
255 // Fixed mode which enables only the digital compression stage also used
256 // by the two adaptive modes.
257 // It is distinguished from the adaptive modes by considering only a
258 // short time-window of the input signal. It applies a fixed gain
259 // through most of the input level range, and compresses (gradually
260 // reduces gain with increasing level) the input signal at higher
261 // levels. This mode is preferred on embedded devices where the capture
262 // signal level is predictable, so that a known gain can be applied.
263 kFixedDigital
264 };
265 Mode mode = kAdaptiveAnalog;
266 // Sets the target peak level (or envelope) of the AGC in dBFs (decibels
267 // from digital full-scale). The convention is to use positive values. For
268 // instance, passing in a value of 3 corresponds to -3 dBFs, or a target
269 // level 3 dB below full-scale. Limited to [0, 31].
270 int target_level_dbfs = 3;
271 // Sets the maximum gain the digital compression stage may apply, in dB. A
272 // higher number corresponds to greater compression, while a value of 0
273 // will leave the signal uncompressed. Limited to [0, 90].
274 // For updates after APM setup, use a RuntimeSetting instead.
275 int compression_gain_db = 9;
276 // When enabled, the compression stage will hard limit the signal to the
277 // target level. Otherwise, the signal will be compressed but not limited
278 // above the target level.
279 bool enable_limiter = true;
Per Åhgren0695df12020-01-13 14:43:13 +0100280
281 // Enables the analog gain controller functionality.
282 struct AnalogGainController {
283 bool enabled = true;
Alessio Bazzica7afd6982022-10-13 17:15:36 +0200284 // TODO(bugs.webrtc.org/7494): Deprecated. Stop using and remove.
285 int startup_min_volume = 0;
Per Åhgren0695df12020-01-13 14:43:13 +0100286 // Lowest analog microphone level that will be applied in response to
287 // clipping.
Alessio Bazzica488f6692022-10-13 13:06:05 +0200288 int clipped_level_min = 70;
Alessio Bazzica866caeb2022-07-19 12:18:38 +0200289 // If true, an adaptive digital gain is applied.
Per Åhgren0695df12020-01-13 14:43:13 +0100290 bool enable_digital_adaptive = true;
Hanna Silenb8dc7fa2021-05-20 17:37:56 +0200291 // Amount the microphone level is lowered with every clipping event.
292 // Limited to (0, 255].
293 int clipped_level_step = 15;
294 // Proportion of clipped samples required to declare a clipping event.
295 // Limited to (0.f, 1.f).
296 float clipped_ratio_threshold = 0.1f;
297 // Time in frames to wait after a clipping event before checking again.
298 // Limited to values higher than 0.
299 int clipped_wait_frames = 300;
Hanna Silena43953a2021-06-02 17:13:24 +0200300
301 // Enables clipping prediction functionality.
302 struct ClippingPredictor {
303 bool enabled = false;
304 enum Mode {
Alessio Bazzicab237a872021-06-11 12:37:54 +0200305 // Clipping event prediction mode with fixed step estimation.
Hanna Silena43953a2021-06-02 17:13:24 +0200306 kClippingEventPrediction,
Alessio Bazzicab237a872021-06-11 12:37:54 +0200307 // Clipped peak estimation mode with adaptive step estimation.
Hanna Silena43953a2021-06-02 17:13:24 +0200308 kAdaptiveStepClippingPeakPrediction,
Alessio Bazzicab237a872021-06-11 12:37:54 +0200309 // Clipped peak estimation mode with fixed step estimation.
Hanna Silena43953a2021-06-02 17:13:24 +0200310 kFixedStepClippingPeakPrediction,
311 };
312 Mode mode = kClippingEventPrediction;
Alessio Bazzicab237a872021-06-11 12:37:54 +0200313 // Number of frames in the sliding analysis window.
Hanna Silena43953a2021-06-02 17:13:24 +0200314 int window_length = 5;
Alessio Bazzicab237a872021-06-11 12:37:54 +0200315 // Number of frames in the sliding reference window.
Hanna Silena43953a2021-06-02 17:13:24 +0200316 int reference_window_length = 5;
Alessio Bazzicab237a872021-06-11 12:37:54 +0200317 // Reference window delay (unit: number of frames).
Hanna Silena43953a2021-06-02 17:13:24 +0200318 int reference_window_delay = 5;
Alessio Bazzicab237a872021-06-11 12:37:54 +0200319 // Clipping prediction threshold (dBFS).
Hanna Silena43953a2021-06-02 17:13:24 +0200320 float clipping_threshold = -1.0f;
321 // Crest factor drop threshold (dB).
322 float crest_factor_margin = 3.0f;
Alessio Bazzica42dacda2021-06-17 17:18:46 +0200323 // If true, the recommended clipped level step is used to modify the
324 // analog gain. Otherwise, the predictor runs without affecting the
325 // analog gain.
326 bool use_predicted_step = true;
Hanna Silena43953a2021-06-02 17:13:24 +0200327 } clipping_predictor;
Per Åhgren0695df12020-01-13 14:43:13 +0100328 } analog_gain_controller;
Sam Zackrissonf0d1c032019-03-27 13:28:08 +0100329 } gain_controller1;
330
Alex Loikoe5831742018-08-24 11:28:36 +0200331 // Enables the next generation AGC functionality. This feature replaces the
332 // standard methods of gain control in the previous AGC. Enabling this
333 // submodule enables an adaptive digital AGC followed by a limiter. By
Artem Titov0b489302021-07-28 20:50:03 +0200334 // setting `fixed_gain_db`, the limiter can be turned into a compressor that
Alex Loikoe5831742018-08-24 11:28:36 +0200335 // first applies a fixed gain. The adaptive digital AGC can be turned off by
336 // setting |adaptive_digital_mode=false|.
Alessio Bazzicadfc11d52021-05-07 11:58:11 +0200337 struct RTC_EXPORT GainController2 {
Alessio Bazzica3438a932020-10-14 12:47:50 +0200338 bool operator==(const GainController2& rhs) const;
339 bool operator!=(const GainController2& rhs) const {
340 return !(*this == rhs);
341 }
342
alessiob3ec96df2017-05-22 06:57:06 -0700343 bool enabled = false;
Alessio Bazzica253f8362020-11-27 16:02:38 +0100344 struct FixedDigital {
Alessio Bazzica841d74e2021-03-31 15:04:03 +0200345 float gain_db = 0.0f;
Alessio Bazzica1e2542f2018-11-13 14:44:15 +0100346 } fixed_digital;
Alessio Bazzicadfc11d52021-05-07 11:58:11 +0200347 struct RTC_EXPORT AdaptiveDigital {
Alessio Bazzicaa2efd152021-04-29 16:17:49 +0200348 bool operator==(const AdaptiveDigital& rhs) const;
349 bool operator!=(const AdaptiveDigital& rhs) const {
350 return !(*this == rhs);
351 }
352
Alessio Bazzica8da7b352018-11-21 10:50:58 +0100353 bool enabled = false;
Alessio Bazzicaa850e6c2021-10-04 13:35:55 +0200354 // When true, the adaptive digital controller runs but the signal is not
355 // modified.
Alessio Bazzicad66a6052021-04-29 16:13:25 +0200356 bool dry_run = false;
Alessio Bazzicaa850e6c2021-10-04 13:35:55 +0200357 float headroom_db = 6.0f;
358 // TODO(bugs.webrtc.org/7494): Consider removing and inferring from
359 // `max_output_noise_level_dbfs`.
360 float max_gain_db = 30.0f;
361 float initial_gain_db = 8.0f;
Alessio Bazzica841d74e2021-03-31 15:04:03 +0200362 int vad_reset_period_ms = 1500;
Alessio Bazzica980c4602021-04-14 19:09:17 +0200363 int adjacent_speech_frames_threshold = 12;
Alessio Bazzica841d74e2021-03-31 15:04:03 +0200364 float max_gain_change_db_per_second = 3.0f;
Alessio Bazzica980c4602021-04-14 19:09:17 +0200365 float max_output_noise_level_dbfs = -50.0f;
Alessio Bazzica1e2542f2018-11-13 14:44:15 +0100366 } adaptive_digital;
Hanna Silen9f06ef12022-11-01 17:17:54 +0100367
368 // Enables input volume control in AGC2.
369 struct InputVolumeController {
370 bool operator==(const InputVolumeController& rhs) const;
371 bool operator!=(const InputVolumeController& rhs) const {
372 return !(*this == rhs);
373 }
374 bool enabled = false;
375 } input_volume_controller;
alessiob3ec96df2017-05-22 06:57:06 -0700376 } gain_controller2;
peah8cee56f2017-08-24 22:36:53 -0700377
Artem Titov59bbd652019-08-02 11:31:37 +0200378 std::string ToString() const;
peah88ac8532016-09-12 16:47:25 -0700379 };
380
Alessio Bazzicac054e782018-04-16 12:10:09 +0200381 // Specifies the properties of a setting to be passed to AudioProcessing at
382 // runtime.
383 class RuntimeSetting {
384 public:
Alex Loiko73ec0192018-05-15 10:52:28 +0200385 enum class Type {
386 kNotSpecified,
387 kCapturePreGain,
Sam Zackrissonf0d1c032019-03-27 13:28:08 +0100388 kCaptureCompressionGain,
Per Åhgren6ee75fd2019-04-26 11:33:37 +0200389 kCaptureFixedPostGain,
Fredrik Hernqvistca362852019-05-10 15:50:02 +0200390 kPlayoutVolumeChange,
Alessio Bazzica7c19a702019-11-07 13:22:00 +0100391 kCustomRenderProcessingRuntimeSetting,
Per Åhgren552d3e32020-08-12 08:46:47 +0200392 kPlayoutAudioDeviceChange,
Per Åhgrendb5d7282021-03-15 16:31:04 +0000393 kCapturePostGain,
Per Åhgren552d3e32020-08-12 08:46:47 +0200394 kCaptureOutputUsed
Alessio Bazzica7c19a702019-11-07 13:22:00 +0100395 };
396
397 // Play-out audio device properties.
398 struct PlayoutAudioDeviceInfo {
399 int id; // Identifies the audio device.
400 int max_volume; // Maximum play-out volume.
Alex Loiko73ec0192018-05-15 10:52:28 +0200401 };
Alessio Bazzicac054e782018-04-16 12:10:09 +0200402
Alessio Bazzica841d74e2021-03-31 15:04:03 +0200403 RuntimeSetting() : type_(Type::kNotSpecified), value_(0.0f) {}
Alessio Bazzicac054e782018-04-16 12:10:09 +0200404 ~RuntimeSetting() = default;
405
406 static RuntimeSetting CreateCapturePreGain(float gain) {
Alessio Bazzicac054e782018-04-16 12:10:09 +0200407 return {Type::kCapturePreGain, gain};
408 }
409
Per Åhgrendb5d7282021-03-15 16:31:04 +0000410 static RuntimeSetting CreateCapturePostGain(float gain) {
411 return {Type::kCapturePostGain, gain};
412 }
413
Sam Zackrissonf0d1c032019-03-27 13:28:08 +0100414 // Corresponds to Config::GainController1::compression_gain_db, but for
415 // runtime configuration.
416 static RuntimeSetting CreateCompressionGainDb(int gain_db) {
417 RTC_DCHECK_GE(gain_db, 0);
418 RTC_DCHECK_LE(gain_db, 90);
419 return {Type::kCaptureCompressionGain, static_cast<float>(gain_db)};
420 }
421
Per Åhgren6ee75fd2019-04-26 11:33:37 +0200422 // Corresponds to Config::GainController2::fixed_digital::gain_db, but for
423 // runtime configuration.
424 static RuntimeSetting CreateCaptureFixedPostGain(float gain_db) {
Alessio Bazzica841d74e2021-03-31 15:04:03 +0200425 RTC_DCHECK_GE(gain_db, 0.0f);
426 RTC_DCHECK_LE(gain_db, 90.0f);
Per Åhgren6ee75fd2019-04-26 11:33:37 +0200427 return {Type::kCaptureFixedPostGain, gain_db};
428 }
429
Alessio Bazzica7c19a702019-11-07 13:22:00 +0100430 // Creates a runtime setting to notify play-out (aka render) audio device
431 // changes.
432 static RuntimeSetting CreatePlayoutAudioDeviceChange(
433 PlayoutAudioDeviceInfo audio_device) {
434 return {Type::kPlayoutAudioDeviceChange, audio_device};
435 }
436
437 // Creates a runtime setting to notify play-out (aka render) volume changes.
Artem Titov0b489302021-07-28 20:50:03 +0200438 // `volume` is the unnormalized volume, the maximum of which
Fredrik Hernqvistca362852019-05-10 15:50:02 +0200439 static RuntimeSetting CreatePlayoutVolumeChange(int volume) {
440 return {Type::kPlayoutVolumeChange, volume};
441 }
442
Alex Loiko73ec0192018-05-15 10:52:28 +0200443 static RuntimeSetting CreateCustomRenderSetting(float payload) {
444 return {Type::kCustomRenderProcessingRuntimeSetting, payload};
445 }
446
Per Åhgren652ada52021-03-03 10:52:44 +0000447 static RuntimeSetting CreateCaptureOutputUsedSetting(
448 bool capture_output_used) {
449 return {Type::kCaptureOutputUsed, capture_output_used};
Per Åhgren552d3e32020-08-12 08:46:47 +0200450 }
451
Alessio Bazzicac054e782018-04-16 12:10:09 +0200452 Type type() const { return type_; }
Alessio Bazzica7c19a702019-11-07 13:22:00 +0100453 // Getters do not return a value but instead modify the argument to protect
454 // from implicit casting.
Alessio Bazzicac054e782018-04-16 12:10:09 +0200455 void GetFloat(float* value) const {
456 RTC_DCHECK(value);
Fredrik Hernqvistca362852019-05-10 15:50:02 +0200457 *value = value_.float_value;
458 }
459 void GetInt(int* value) const {
460 RTC_DCHECK(value);
461 *value = value_.int_value;
Alessio Bazzicac054e782018-04-16 12:10:09 +0200462 }
Per Åhgren552d3e32020-08-12 08:46:47 +0200463 void GetBool(bool* value) const {
464 RTC_DCHECK(value);
465 *value = value_.bool_value;
466 }
Alessio Bazzica7c19a702019-11-07 13:22:00 +0100467 void GetPlayoutAudioDeviceInfo(PlayoutAudioDeviceInfo* value) const {
468 RTC_DCHECK(value);
469 *value = value_.playout_audio_device_info;
470 }
Alessio Bazzicac054e782018-04-16 12:10:09 +0200471
472 private:
473 RuntimeSetting(Type id, float value) : type_(id), value_(value) {}
Fredrik Hernqvistca362852019-05-10 15:50:02 +0200474 RuntimeSetting(Type id, int value) : type_(id), value_(value) {}
Alessio Bazzica7c19a702019-11-07 13:22:00 +0100475 RuntimeSetting(Type id, PlayoutAudioDeviceInfo value)
476 : type_(id), value_(value) {}
Alessio Bazzicac054e782018-04-16 12:10:09 +0200477 Type type_;
Fredrik Hernqvistca362852019-05-10 15:50:02 +0200478 union U {
479 U() {}
480 U(int value) : int_value(value) {}
481 U(float value) : float_value(value) {}
Alessio Bazzica7c19a702019-11-07 13:22:00 +0100482 U(PlayoutAudioDeviceInfo value) : playout_audio_device_info(value) {}
Fredrik Hernqvistca362852019-05-10 15:50:02 +0200483 float float_value;
484 int int_value;
Per Åhgren552d3e32020-08-12 08:46:47 +0200485 bool bool_value;
Alessio Bazzica7c19a702019-11-07 13:22:00 +0100486 PlayoutAudioDeviceInfo playout_audio_device_info;
Fredrik Hernqvistca362852019-05-10 15:50:02 +0200487 } value_;
Alessio Bazzicac054e782018-04-16 12:10:09 +0200488 };
489
peaha9cc40b2017-06-29 08:32:09 -0700490 ~AudioProcessing() override {}
niklase@google.com470e71d2011-07-07 08:21:25 +0000491
niklase@google.com470e71d2011-07-07 08:21:25 +0000492 // Initializes internal states, while retaining all user settings. This
493 // should be called before beginning to process a new audio stream. However,
494 // it is not necessary to call before processing the first stream after
andrew@webrtc.orgddbb8a22014-04-22 21:00:04 +0000495 // creation.
496 //
497 // It is also not necessary to call if the audio parameters (sample
andrew@webrtc.org60730cf2014-01-07 17:45:09 +0000498 // rate and number of channels) have changed. Passing updated parameters
Artem Titov0b489302021-07-28 20:50:03 +0200499 // directly to `ProcessStream()` and `ProcessReverseStream()` is permissible.
andrew@webrtc.orgddbb8a22014-04-22 21:00:04 +0000500 // If the parameters are known at init-time though, they may be provided.
Per Åhgren0ade9832020-09-01 23:57:20 +0200501 // TODO(webrtc:5298): Change to return void.
niklase@google.com470e71d2011-07-07 08:21:25 +0000502 virtual int Initialize() = 0;
andrew@webrtc.orgddbb8a22014-04-22 21:00:04 +0000503
504 // The int16 interfaces require:
Artem Titov0b489302021-07-28 20:50:03 +0200505 // - only `NativeRate`s be used
andrew@webrtc.orgddbb8a22014-04-22 21:00:04 +0000506 // - that the input, output and reverse rates must match
Artem Titovcfea2182021-08-10 01:22:31 +0200507 // - that `processing_config.output_stream()` matches
508 // `processing_config.input_stream()`.
andrew@webrtc.orgddbb8a22014-04-22 21:00:04 +0000509 //
Michael Graczyk86c6d332015-07-23 11:41:39 -0700510 // The float interfaces accept arbitrary rates and support differing input and
511 // output layouts, but the output must have either one channel or the same
512 // number of channels as the input.
513 virtual int Initialize(const ProcessingConfig& processing_config) = 0;
514
peah88ac8532016-09-12 16:47:25 -0700515 // TODO(peah): This method is a temporary solution used to take control
516 // over the parameters in the audio processing module and is likely to change.
517 virtual void ApplyConfig(const Config& config) = 0;
518
andrew@webrtc.orgddbb8a22014-04-22 21:00:04 +0000519 // TODO(ajm): Only intended for internal use. Make private and friend the
520 // necessary classes?
521 virtual int proc_sample_rate_hz() const = 0;
522 virtual int proc_split_sample_rate_hz() const = 0;
Peter Kasting69558702016-01-12 16:26:35 -0800523 virtual size_t num_input_channels() const = 0;
524 virtual size_t num_proc_channels() const = 0;
525 virtual size_t num_output_channels() const = 0;
526 virtual size_t num_reverse_channels() const = 0;
niklase@google.com470e71d2011-07-07 08:21:25 +0000527
andrew@webrtc.org17342e52014-02-12 22:28:31 +0000528 // Set to true when the output of AudioProcessing will be muted or in some
529 // other way not used. Ideally, the captured audio would still be processed,
530 // but some components may change behavior based on this information.
Per Åhgren0a144a72021-02-09 08:47:51 +0100531 // Default false. This method takes a lock. To achieve this in a lock-less
532 // manner the PostRuntimeSetting can instead be used.
andrew@webrtc.org17342e52014-02-12 22:28:31 +0000533 virtual void set_output_will_be_muted(bool muted) = 0;
andrew@webrtc.org17342e52014-02-12 22:28:31 +0000534
Per Åhgren0a144a72021-02-09 08:47:51 +0100535 // Enqueues a runtime setting.
Alessio Bazzicac054e782018-04-16 12:10:09 +0200536 virtual void SetRuntimeSetting(RuntimeSetting setting) = 0;
537
Per Åhgren0a144a72021-02-09 08:47:51 +0100538 // Enqueues a runtime setting. Returns a bool indicating whether the
539 // enqueueing was successfull.
Per Åhgren8eea1172021-02-09 23:15:07 +0100540 virtual bool PostRuntimeSetting(RuntimeSetting setting) = 0;
Per Åhgren0a144a72021-02-09 08:47:51 +0100541
Sam Zackrisson3bd444f2022-08-03 14:37:00 +0200542 // Accepts and produces a ~10 ms frame of interleaved 16 bit integer audio as
Artem Titov0b489302021-07-28 20:50:03 +0200543 // specified in `input_config` and `output_config`. `src` and `dest` may use
Per Åhgren645f24c2020-03-16 12:06:02 +0100544 // the same memory, if desired.
545 virtual int ProcessStream(const int16_t* const src,
546 const StreamConfig& input_config,
547 const StreamConfig& output_config,
Per Åhgrendc5522b2020-03-19 14:55:58 +0100548 int16_t* const dest) = 0;
Per Åhgren645f24c2020-03-16 12:06:02 +0100549
Michael Graczyk86c6d332015-07-23 11:41:39 -0700550 // Accepts deinterleaved float audio with the range [-1, 1]. Each element of
Artem Titov0b489302021-07-28 20:50:03 +0200551 // `src` points to a channel buffer, arranged according to `input_stream`. At
552 // output, the channels will be arranged according to `output_stream` in
553 // `dest`.
Michael Graczyk86c6d332015-07-23 11:41:39 -0700554 //
Artem Titov0b489302021-07-28 20:50:03 +0200555 // The output must have one channel or as many channels as the input. `src`
556 // and `dest` may use the same memory, if desired.
Michael Graczyk86c6d332015-07-23 11:41:39 -0700557 virtual int ProcessStream(const float* const* src,
558 const StreamConfig& input_config,
559 const StreamConfig& output_config,
560 float* const* dest) = 0;
561
Sam Zackrisson3bd444f2022-08-03 14:37:00 +0200562 // Accepts and produces a ~10 ms frame of interleaved 16 bit integer audio for
Artem Titov0b489302021-07-28 20:50:03 +0200563 // the reverse direction audio stream as specified in `input_config` and
564 // `output_config`. `src` and `dest` may use the same memory, if desired.
Per Åhgren645f24c2020-03-16 12:06:02 +0100565 virtual int ProcessReverseStream(const int16_t* const src,
566 const StreamConfig& input_config,
567 const StreamConfig& output_config,
568 int16_t* const dest) = 0;
569
Michael Graczyk86c6d332015-07-23 11:41:39 -0700570 // Accepts deinterleaved float audio with the range [-1, 1]. Each element of
Artem Titov0b489302021-07-28 20:50:03 +0200571 // `data` points to a channel buffer, arranged according to `reverse_config`.
ekmeyerson60d9b332015-08-14 10:35:55 -0700572 virtual int ProcessReverseStream(const float* const* src,
peahde65ddc2016-09-16 15:02:15 -0700573 const StreamConfig& input_config,
574 const StreamConfig& output_config,
ekmeyerson60d9b332015-08-14 10:35:55 -0700575 float* const* dest) = 0;
Michael Graczyk86c6d332015-07-23 11:41:39 -0700576
Gustaf Ullbergcb307262019-10-29 09:30:44 +0100577 // Accepts deinterleaved float audio with the range [-1, 1]. Each element
Artem Titov0b489302021-07-28 20:50:03 +0200578 // of `data` points to a channel buffer, arranged according to
579 // `reverse_config`.
Gustaf Ullbergcb307262019-10-29 09:30:44 +0100580 virtual int AnalyzeReverseStream(const float* const* data,
581 const StreamConfig& reverse_config) = 0;
582
Sam Zackrisson3bd444f2022-08-03 14:37:00 +0200583 // Returns the most recently produced ~10 ms of the linear AEC output at a
584 // rate of 16 kHz. If there is more than one capture channel, a mono
585 // representation of the input is returned. Returns true/false to indicate
586 // whether an output returned.
Per Åhgrenc20a19c2019-11-13 11:12:29 +0100587 virtual bool GetLinearAecOutput(
588 rtc::ArrayView<std::array<float, 160>> linear_output) const = 0;
589
Sam Zackrissonf0d1c032019-03-27 13:28:08 +0100590 // This must be called prior to ProcessStream() if and only if adaptive analog
591 // gain control is enabled, to pass the current analog level from the audio
Hanna Silencd597042021-11-02 11:02:48 +0100592 // HAL. Must be within the range [0, 255].
Sam Zackrissonf0d1c032019-03-27 13:28:08 +0100593 virtual void set_stream_analog_level(int level) = 0;
594
Alessio Bazzicafcf1af32022-09-07 17:14:26 +0200595 // When an analog mode is set, this should be called after
596 // `set_stream_analog_level()` and `ProcessStream()` to obtain the recommended
597 // new analog level for the audio HAL. It is the user's responsibility to
598 // apply this level.
Sam Zackrissonf0d1c032019-03-27 13:28:08 +0100599 virtual int recommended_stream_analog_level() const = 0;
600
niklase@google.com470e71d2011-07-07 08:21:25 +0000601 // This must be called if and only if echo processing is enabled.
602 //
Artem Titov0b489302021-07-28 20:50:03 +0200603 // Sets the `delay` in ms between ProcessReverseStream() receiving a far-end
niklase@google.com470e71d2011-07-07 08:21:25 +0000604 // frame and ProcessStream() receiving a near-end frame containing the
605 // corresponding echo. On the client-side this can be expressed as
606 // delay = (t_render - t_analyze) + (t_process - t_capture)
607 // where,
aluebsb0319552016-03-17 20:39:53 -0700608 // - t_analyze is the time a frame is passed to ProcessReverseStream() and
niklase@google.com470e71d2011-07-07 08:21:25 +0000609 // t_render is the time the first sample of the same frame is rendered by
610 // the audio hardware.
611 // - t_capture is the time the first sample of a frame is captured by the
alessiob13fc1802017-04-19 05:35:51 -0700612 // audio hardware and t_process is the time the same frame is passed to
niklase@google.com470e71d2011-07-07 08:21:25 +0000613 // ProcessStream().
614 virtual int set_stream_delay_ms(int delay) = 0;
615 virtual int stream_delay_ms() const = 0;
616
andrew@webrtc.org75dd2882014-02-11 20:52:30 +0000617 // Call to signal that a key press occurred (true) or did not occur (false)
618 // with this chunk of audio.
619 virtual void set_stream_key_pressed(bool key_pressed) = 0;
andrew@webrtc.org75dd2882014-02-11 20:52:30 +0000620
Per Åhgren09e9a832020-05-11 11:03:47 +0200621 // Creates and attaches an webrtc::AecDump for recording debugging
622 // information.
Artem Titov0b489302021-07-28 20:50:03 +0200623 // The `worker_queue` may not be null and must outlive the created
Per Åhgren09e9a832020-05-11 11:03:47 +0200624 // AecDump instance. |max_log_size_bytes == -1| means the log size
Artem Titov0b489302021-07-28 20:50:03 +0200625 // will be unlimited. `handle` may not be null. The AecDump takes
626 // responsibility for `handle` and closes it in the destructor. A
Per Åhgren09e9a832020-05-11 11:03:47 +0200627 // return value of true indicates that the file has been
628 // sucessfully opened, while a value of false indicates that
629 // opening the file failed.
Ali Tofigh1fa87c42022-07-25 22:07:08 +0200630 virtual bool CreateAndAttachAecDump(absl::string_view file_name,
631 int64_t max_log_size_bytes,
Ali Tofigh980ad0c2022-08-09 09:21:17 +0200632 rtc::TaskQueue* worker_queue) = 0;
Per Åhgren09e9a832020-05-11 11:03:47 +0200633 virtual bool CreateAndAttachAecDump(FILE* handle,
634 int64_t max_log_size_bytes,
635 rtc::TaskQueue* worker_queue) = 0;
636
637 // TODO(webrtc:5298) Deprecated variant.
aleloi868f32f2017-05-23 07:20:05 -0700638 // Attaches provided webrtc::AecDump for recording debugging
639 // information. Log file and maximum file size logic is supposed to
640 // be handled by implementing instance of AecDump. Calling this
641 // method when another AecDump is attached resets the active AecDump
642 // with a new one. This causes the d-tor of the earlier AecDump to
643 // be called. The d-tor call may block until all pending logging
644 // tasks are completed.
Alex Loikobe767e02017-06-08 09:45:03 +0200645 virtual void AttachAecDump(std::unique_ptr<AecDump> aec_dump) = 0;
aleloi868f32f2017-05-23 07:20:05 -0700646
647 // If no AecDump is attached, this has no effect. If an AecDump is
648 // attached, it's destructor is called. The d-tor may block until
649 // all pending logging tasks are completed.
Alex Loikobe767e02017-06-08 09:45:03 +0200650 virtual void DetachAecDump() = 0;
aleloi868f32f2017-05-23 07:20:05 -0700651
Per Åhgrencf4c8722019-12-30 14:32:14 +0100652 // Get audio processing statistics.
653 virtual AudioProcessingStats GetStatistics() = 0;
Artem Titov0b489302021-07-28 20:50:03 +0200654 // TODO(webrtc:5298) Deprecated variant. The `has_remote_tracks` argument
Per Åhgrencf4c8722019-12-30 14:32:14 +0100655 // should be set if there are active remote tracks (this would usually be true
656 // during a call). If there are no remote tracks some of the stats will not be
657 // set by AudioProcessing, because they only make sense if there is at least
658 // one remote track.
659 virtual AudioProcessingStats GetStatistics(bool has_remote_tracks) = 0;
Ivo Creusenae026092017-11-20 13:07:16 +0100660
henrik.lundinadf06352017-04-05 05:48:24 -0700661 // Returns the last applied configuration.
henrik.lundin77492862017-04-06 23:28:09 -0700662 virtual AudioProcessing::Config GetConfig() const = 0;
henrik.lundinadf06352017-04-05 05:48:24 -0700663
andrew@webrtc.org648af742012-02-08 01:57:29 +0000664 enum Error {
665 // Fatal errors.
niklase@google.com470e71d2011-07-07 08:21:25 +0000666 kNoError = 0,
667 kUnspecifiedError = -1,
668 kCreationFailedError = -2,
669 kUnsupportedComponentError = -3,
670 kUnsupportedFunctionError = -4,
671 kNullPointerError = -5,
672 kBadParameterError = -6,
673 kBadSampleRateError = -7,
674 kBadDataLengthError = -8,
675 kBadNumberChannelsError = -9,
676 kFileError = -10,
677 kStreamParameterNotSetError = -11,
andrew@webrtc.org648af742012-02-08 01:57:29 +0000678 kNotEnabledError = -12,
niklase@google.com470e71d2011-07-07 08:21:25 +0000679
andrew@webrtc.org648af742012-02-08 01:57:29 +0000680 // Warnings are non-fatal.
niklase@google.com470e71d2011-07-07 08:21:25 +0000681 // This results when a set_stream_ parameter is out of range. Processing
682 // will continue, but the parameter may have been truncated.
andrew@webrtc.org648af742012-02-08 01:57:29 +0000683 kBadStreamParameterWarning = -13
niklase@google.com470e71d2011-07-07 08:21:25 +0000684 };
andrew@webrtc.org56e4a052014-02-27 22:23:17 +0000685
Per Åhgren2507f8c2020-03-19 12:33:29 +0100686 // Native rates supported by the integer interfaces.
andrew@webrtc.orgddbb8a22014-04-22 21:00:04 +0000687 enum NativeRate {
andrew@webrtc.org56e4a052014-02-27 22:23:17 +0000688 kSampleRate8kHz = 8000,
689 kSampleRate16kHz = 16000,
aluebs@webrtc.org087da132014-11-17 23:01:23 +0000690 kSampleRate32kHz = 32000,
691 kSampleRate48kHz = 48000
andrew@webrtc.org56e4a052014-02-27 22:23:17 +0000692 };
andrew@webrtc.orgddbb8a22014-04-22 21:00:04 +0000693
kwibergd59d3bb2016-09-13 07:49:33 -0700694 // TODO(kwiberg): We currently need to support a compiler (Visual C++) that
695 // complains if we don't explicitly state the size of the array here. Remove
696 // the size when that's no longer the case.
697 static constexpr int kNativeSampleRatesHz[4] = {
698 kSampleRate8kHz, kSampleRate16kHz, kSampleRate32kHz, kSampleRate48kHz};
699 static constexpr size_t kNumNativeSampleRates =
700 arraysize(kNativeSampleRatesHz);
701 static constexpr int kMaxNativeSampleRateHz =
702 kNativeSampleRatesHz[kNumNativeSampleRates - 1];
Alejandro Luebscdfe20b2015-09-23 12:49:12 -0700703
Sam Zackrisson3bd444f2022-08-03 14:37:00 +0200704 // APM processes audio in chunks of about 10 ms. See GetFrameSize() for
705 // details.
Per Åhgren12dc2742020-12-08 09:40:35 +0100706 static constexpr int kChunkSizeMs = 10;
Sam Zackrisson3bd444f2022-08-03 14:37:00 +0200707
708 // Returns floor(sample_rate_hz/100): the number of samples per channel used
709 // as input and output to the audio processing module in calls to
710 // ProcessStream, ProcessReverseStream, AnalyzeReverseStream, and
711 // GetLinearAecOutput.
712 //
713 // This is exactly 10 ms for sample rates divisible by 100. For example:
714 // - 48000 Hz (480 samples per channel),
715 // - 44100 Hz (441 samples per channel),
716 // - 16000 Hz (160 samples per channel).
717 //
718 // Sample rates not divisible by 100 are received/produced in frames of
719 // approximately 10 ms. For example:
720 // - 22050 Hz (220 samples per channel, or ~9.98 ms per frame),
721 // - 11025 Hz (110 samples per channel, or ~9.98 ms per frame).
722 // These nondivisible sample rates yield lower audio quality compared to
723 // multiples of 100. Internal resampling to 10 ms frames causes a simulated
724 // clock drift effect which impacts the performance of (for example) echo
725 // cancellation.
726 static int GetFrameSize(int sample_rate_hz) { return sample_rate_hz / 100; }
niklase@google.com470e71d2011-07-07 08:21:25 +0000727};
728
Mirko Bonadei3d255302018-10-11 10:50:45 +0200729class RTC_EXPORT AudioProcessingBuilder {
Ivo Creusen5ec7e122017-12-22 11:35:59 +0100730 public:
731 AudioProcessingBuilder();
Alessio Bazzica20a9ac62021-10-14 10:55:08 +0200732 AudioProcessingBuilder(const AudioProcessingBuilder&) = delete;
733 AudioProcessingBuilder& operator=(const AudioProcessingBuilder&) = delete;
Ivo Creusen5ec7e122017-12-22 11:35:59 +0100734 ~AudioProcessingBuilder();
Alessio Bazzica20a9ac62021-10-14 10:55:08 +0200735
736 // Sets the APM configuration.
737 AudioProcessingBuilder& SetConfig(const AudioProcessing::Config& config) {
738 config_ = config;
739 return *this;
740 }
741
742 // Sets the echo controller factory to inject when APM is created.
Ivo Creusen5ec7e122017-12-22 11:35:59 +0100743 AudioProcessingBuilder& SetEchoControlFactory(
Per Åhgrencc73ed32020-04-26 23:56:17 +0200744 std::unique_ptr<EchoControlFactory> echo_control_factory) {
745 echo_control_factory_ = std::move(echo_control_factory);
746 return *this;
747 }
Alessio Bazzica20a9ac62021-10-14 10:55:08 +0200748
749 // Sets the capture post-processing sub-module to inject when APM is created.
Ivo Creusen5ec7e122017-12-22 11:35:59 +0100750 AudioProcessingBuilder& SetCapturePostProcessing(
Per Åhgrencc73ed32020-04-26 23:56:17 +0200751 std::unique_ptr<CustomProcessing> capture_post_processing) {
752 capture_post_processing_ = std::move(capture_post_processing);
753 return *this;
754 }
Alessio Bazzica20a9ac62021-10-14 10:55:08 +0200755
756 // Sets the render pre-processing sub-module to inject when APM is created.
Ivo Creusen5ec7e122017-12-22 11:35:59 +0100757 AudioProcessingBuilder& SetRenderPreProcessing(
Per Åhgrencc73ed32020-04-26 23:56:17 +0200758 std::unique_ptr<CustomProcessing> render_pre_processing) {
759 render_pre_processing_ = std::move(render_pre_processing);
760 return *this;
761 }
Alessio Bazzica20a9ac62021-10-14 10:55:08 +0200762
763 // Sets the echo detector to inject when APM is created.
Ivo Creusen09fa4b02018-01-11 16:08:54 +0100764 AudioProcessingBuilder& SetEchoDetector(
Per Åhgrencc73ed32020-04-26 23:56:17 +0200765 rtc::scoped_refptr<EchoDetector> echo_detector) {
766 echo_detector_ = std::move(echo_detector);
767 return *this;
768 }
Alessio Bazzica20a9ac62021-10-14 10:55:08 +0200769
770 // Sets the capture analyzer sub-module to inject when APM is created.
Valeriia Nemychnikovaf06eb572018-08-29 10:37:09 +0200771 AudioProcessingBuilder& SetCaptureAnalyzer(
Per Åhgrencc73ed32020-04-26 23:56:17 +0200772 std::unique_ptr<CustomAudioAnalyzer> capture_analyzer) {
773 capture_analyzer_ = std::move(capture_analyzer);
774 return *this;
775 }
Alessio Bazzica20a9ac62021-10-14 10:55:08 +0200776
777 // Creates an APM instance with the specified config or the default one if
778 // unspecified. Injects the specified components transferring the ownership
779 // to the newly created APM instance - i.e., except for the config, the
780 // builder is reset to its initial state.
Niels Möller4f776ac2021-07-02 11:30:54 +0200781 rtc::scoped_refptr<AudioProcessing> Create();
Ivo Creusen5ec7e122017-12-22 11:35:59 +0100782
783 private:
Alessio Bazzica20a9ac62021-10-14 10:55:08 +0200784 AudioProcessing::Config config_;
Ivo Creusen5ec7e122017-12-22 11:35:59 +0100785 std::unique_ptr<EchoControlFactory> echo_control_factory_;
786 std::unique_ptr<CustomProcessing> capture_post_processing_;
787 std::unique_ptr<CustomProcessing> render_pre_processing_;
Ivo Creusend1f970d2018-06-14 11:02:03 +0200788 rtc::scoped_refptr<EchoDetector> echo_detector_;
Valeriia Nemychnikovaf06eb572018-08-29 10:37:09 +0200789 std::unique_ptr<CustomAudioAnalyzer> capture_analyzer_;
Ivo Creusen5ec7e122017-12-22 11:35:59 +0100790};
791
Michael Graczyk86c6d332015-07-23 11:41:39 -0700792class StreamConfig {
793 public:
794 // sample_rate_hz: The sampling rate of the stream.
Henrik Lundin64253a92022-02-04 09:02:48 +0000795 // num_channels: The number of audio channels in the stream.
Alessio Bazzicac7d0e422022-02-04 17:06:55 +0100796 StreamConfig(int sample_rate_hz = 0, size_t num_channels = 0)
Michael Graczyk86c6d332015-07-23 11:41:39 -0700797 : sample_rate_hz_(sample_rate_hz),
798 num_channels_(num_channels),
Michael Graczyk86c6d332015-07-23 11:41:39 -0700799 num_frames_(calculate_frames(sample_rate_hz)) {}
800
801 void set_sample_rate_hz(int value) {
802 sample_rate_hz_ = value;
803 num_frames_ = calculate_frames(value);
804 }
Peter Kasting69558702016-01-12 16:26:35 -0800805 void set_num_channels(size_t value) { num_channels_ = value; }
Michael Graczyk86c6d332015-07-23 11:41:39 -0700806
807 int sample_rate_hz() const { return sample_rate_hz_; }
808
Henrik Lundin64253a92022-02-04 09:02:48 +0000809 // The number of channels in the stream.
Peter Kasting69558702016-01-12 16:26:35 -0800810 size_t num_channels() const { return num_channels_; }
Michael Graczyk86c6d332015-07-23 11:41:39 -0700811
Peter Kastingdce40cf2015-08-24 14:52:23 -0700812 size_t num_frames() const { return num_frames_; }
813 size_t num_samples() const { return num_channels_ * num_frames_; }
Michael Graczyk86c6d332015-07-23 11:41:39 -0700814
815 bool operator==(const StreamConfig& other) const {
816 return sample_rate_hz_ == other.sample_rate_hz_ &&
Henrik Lundin64253a92022-02-04 09:02:48 +0000817 num_channels_ == other.num_channels_;
Michael Graczyk86c6d332015-07-23 11:41:39 -0700818 }
819
820 bool operator!=(const StreamConfig& other) const { return !(*this == other); }
821
822 private:
Peter Kastingdce40cf2015-08-24 14:52:23 -0700823 static size_t calculate_frames(int sample_rate_hz) {
Sam Zackrisson3bd444f2022-08-03 14:37:00 +0200824 return static_cast<size_t>(AudioProcessing::GetFrameSize(sample_rate_hz));
Michael Graczyk86c6d332015-07-23 11:41:39 -0700825 }
826
827 int sample_rate_hz_;
Peter Kasting69558702016-01-12 16:26:35 -0800828 size_t num_channels_;
Peter Kastingdce40cf2015-08-24 14:52:23 -0700829 size_t num_frames_;
Michael Graczyk86c6d332015-07-23 11:41:39 -0700830};
831
832class ProcessingConfig {
833 public:
834 enum StreamName {
835 kInputStream,
836 kOutputStream,
ekmeyerson60d9b332015-08-14 10:35:55 -0700837 kReverseInputStream,
838 kReverseOutputStream,
Michael Graczyk86c6d332015-07-23 11:41:39 -0700839 kNumStreamNames,
840 };
841
842 const StreamConfig& input_stream() const {
843 return streams[StreamName::kInputStream];
844 }
845 const StreamConfig& output_stream() const {
846 return streams[StreamName::kOutputStream];
847 }
ekmeyerson60d9b332015-08-14 10:35:55 -0700848 const StreamConfig& reverse_input_stream() const {
849 return streams[StreamName::kReverseInputStream];
850 }
851 const StreamConfig& reverse_output_stream() const {
852 return streams[StreamName::kReverseOutputStream];
Michael Graczyk86c6d332015-07-23 11:41:39 -0700853 }
854
855 StreamConfig& input_stream() { return streams[StreamName::kInputStream]; }
856 StreamConfig& output_stream() { return streams[StreamName::kOutputStream]; }
ekmeyerson60d9b332015-08-14 10:35:55 -0700857 StreamConfig& reverse_input_stream() {
858 return streams[StreamName::kReverseInputStream];
859 }
860 StreamConfig& reverse_output_stream() {
861 return streams[StreamName::kReverseOutputStream];
862 }
Michael Graczyk86c6d332015-07-23 11:41:39 -0700863
864 bool operator==(const ProcessingConfig& other) const {
865 for (int i = 0; i < StreamName::kNumStreamNames; ++i) {
866 if (this->streams[i] != other.streams[i]) {
867 return false;
868 }
869 }
870 return true;
871 }
872
873 bool operator!=(const ProcessingConfig& other) const {
874 return !(*this == other);
875 }
876
877 StreamConfig streams[StreamName::kNumStreamNames];
878};
879
Valeriia Nemychnikovaf06eb572018-08-29 10:37:09 +0200880// Experimental interface for a custom analysis submodule.
881class CustomAudioAnalyzer {
882 public:
883 // (Re-) Initializes the submodule.
884 virtual void Initialize(int sample_rate_hz, int num_channels) = 0;
885 // Analyzes the given capture or render signal.
886 virtual void Analyze(const AudioBuffer* audio) = 0;
887 // Returns a string representation of the module state.
888 virtual std::string ToString() const = 0;
889
890 virtual ~CustomAudioAnalyzer() {}
891};
892
Alex Loiko5825aa62017-12-18 16:02:40 +0100893// Interface for a custom processing submodule.
894class CustomProcessing {
Sam Zackrisson0beac582017-09-25 12:04:02 +0200895 public:
896 // (Re-)Initializes the submodule.
897 virtual void Initialize(int sample_rate_hz, int num_channels) = 0;
898 // Processes the given capture or render signal.
899 virtual void Process(AudioBuffer* audio) = 0;
900 // Returns a string representation of the module state.
901 virtual std::string ToString() const = 0;
Alex Loiko73ec0192018-05-15 10:52:28 +0200902 // Handles RuntimeSettings. TODO(webrtc:9262): make pure virtual
903 // after updating dependencies.
904 virtual void SetRuntimeSetting(AudioProcessing::RuntimeSetting setting);
Sam Zackrisson0beac582017-09-25 12:04:02 +0200905
Alex Loiko5825aa62017-12-18 16:02:40 +0100906 virtual ~CustomProcessing() {}
Sam Zackrisson0beac582017-09-25 12:04:02 +0200907};
908
Ivo Creusen09fa4b02018-01-11 16:08:54 +0100909// Interface for an echo detector submodule.
Ivo Creusend1f970d2018-06-14 11:02:03 +0200910class EchoDetector : public rtc::RefCountInterface {
Ivo Creusen09fa4b02018-01-11 16:08:54 +0100911 public:
912 // (Re-)Initializes the submodule.
Ivo Creusen647ef092018-03-14 17:13:48 +0100913 virtual void Initialize(int capture_sample_rate_hz,
914 int num_capture_channels,
915 int render_sample_rate_hz,
916 int num_render_channels) = 0;
Ivo Creusen09fa4b02018-01-11 16:08:54 +0100917
Sam Zackrisson03cb7e52021-12-06 15:40:04 +0100918 // Analysis (not changing) of the first channel of the render signal.
Ivo Creusen09fa4b02018-01-11 16:08:54 +0100919 virtual void AnalyzeRenderAudio(rtc::ArrayView<const float> render_audio) = 0;
920
921 // Analysis (not changing) of the capture signal.
922 virtual void AnalyzeCaptureAudio(
923 rtc::ArrayView<const float> capture_audio) = 0;
924
Ivo Creusen09fa4b02018-01-11 16:08:54 +0100925 struct Metrics {
Ivo Creusenbb826c92020-04-29 14:34:48 +0200926 absl::optional<double> echo_likelihood;
927 absl::optional<double> echo_likelihood_recent_max;
Ivo Creusen09fa4b02018-01-11 16:08:54 +0100928 };
929
930 // Collect current metrics from the echo detector.
931 virtual Metrics GetMetrics() const = 0;
Ivo Creusen09fa4b02018-01-11 16:08:54 +0100932};
933
niklase@google.com470e71d2011-07-07 08:21:25 +0000934} // namespace webrtc
935
Mirko Bonadei92ea95e2017-09-15 06:47:31 +0200936#endif // MODULES_AUDIO_PROCESSING_INCLUDE_AUDIO_PROCESSING_H_