blob: 4a3a30baf8e26519847d403f9144786ad4d92c57 [file] [log] [blame]
niklase@google.com470e71d2011-07-07 08:21:25 +00001/*
andrew@webrtc.org648af742012-02-08 01:57:29 +00002 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
niklase@google.com470e71d2011-07-07 08:21:25 +00003 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020011#ifndef MODULES_AUDIO_PROCESSING_INCLUDE_AUDIO_PROCESSING_H_
12#define MODULES_AUDIO_PROCESSING_INCLUDE_AUDIO_PROCESSING_H_
niklase@google.com470e71d2011-07-07 08:21:25 +000013
Alejandro Luebscb3f9bd2015-10-29 18:21:34 -070014// MSVC++ requires this to be set before any other includes to get M_PI.
Patrik Höglund3ff90f12017-12-12 14:41:53 +010015#ifndef _USE_MATH_DEFINES
Alejandro Luebscb3f9bd2015-10-29 18:21:34 -070016#define _USE_MATH_DEFINES
Patrik Höglund3ff90f12017-12-12 14:41:53 +010017#endif
Alejandro Luebscb3f9bd2015-10-29 18:21:34 -070018
19#include <math.h>
andrew@webrtc.orgd72b3d62012-11-15 21:46:06 +000020#include <stddef.h> // size_t
henrikg@webrtc.org863b5362013-12-06 16:05:17 +000021#include <stdio.h> // FILE
peah8cee56f2017-08-24 22:36:53 -070022#include <string.h>
aluebs@webrtc.orgfb7a0392015-01-05 21:58:58 +000023#include <vector>
ajm@google.com22e65152011-07-18 18:03:01 +000024
Ivo Creusenae026092017-11-20 13:07:16 +010025#include "api/optional.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020026#include "modules/audio_processing/beamformer/array_util.h"
Ivo Creusen56d46092017-11-24 17:29:59 +010027#include "modules/audio_processing/include/audio_processing_statistics.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020028#include "modules/audio_processing/include/config.h"
29#include "rtc_base/arraysize.h"
Sam Zackrisson0beac582017-09-25 12:04:02 +020030#include "rtc_base/deprecation.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020031#include "rtc_base/platform_file.h"
32#include "rtc_base/refcount.h"
Mirko Bonadei71207422017-09-15 13:58:09 +020033#include "typedefs.h" // NOLINT(build/include)
niklase@google.com470e71d2011-07-07 08:21:25 +000034
35namespace webrtc {
36
peah50e21bd2016-03-05 08:39:21 -080037struct AecCore;
38
aleloi868f32f2017-05-23 07:20:05 -070039class AecDump;
Sam Zackrisson0beac582017-09-25 12:04:02 +020040class AudioBuffer;
niklase@google.com470e71d2011-07-07 08:21:25 +000041class AudioFrame;
Michael Graczykdfa36052015-03-25 16:37:27 -070042
Alejandro Luebsf4022ff2016-07-01 17:19:09 -070043class NonlinearBeamformer;
Michael Graczykdfa36052015-03-25 16:37:27 -070044
Michael Graczyk86c6d332015-07-23 11:41:39 -070045class StreamConfig;
46class ProcessingConfig;
47
niklase@google.com470e71d2011-07-07 08:21:25 +000048class EchoCancellation;
49class EchoControlMobile;
Gustaf Ullberg002ef282017-10-12 15:13:17 +020050class EchoControlFactory;
niklase@google.com470e71d2011-07-07 08:21:25 +000051class GainControl;
52class HighPassFilter;
53class LevelEstimator;
54class NoiseSuppression;
Alex Loiko5825aa62017-12-18 16:02:40 +010055class CustomProcessing;
niklase@google.com470e71d2011-07-07 08:21:25 +000056class VoiceDetection;
57
Alex Loiko5825aa62017-12-18 16:02:40 +010058// webrtc:8665, addedd temporarily to avoid breaking dependencies.
59typedef CustomProcessing PostProcessing;
60
Henrik Lundin441f6342015-06-09 16:03:13 +020061// Use to enable the extended filter mode in the AEC, along with robustness
62// measures around the reported system delays. It comes with a significant
63// increase in AEC complexity, but is much more robust to unreliable reported
64// delays.
andrew@webrtc.org6b1e2192013-09-25 23:46:20 +000065//
66// Detailed changes to the algorithm:
67// - The filter length is changed from 48 to 128 ms. This comes with tuning of
68// several parameters: i) filter adaptation stepsize and error threshold;
69// ii) non-linear processing smoothing and overdrive.
70// - Option to ignore the reported delays on platforms which we deem
71// sufficiently unreliable. See WEBRTC_UNTRUSTED_DELAY in echo_cancellation.c.
72// - Faster startup times by removing the excessive "startup phase" processing
73// of reported delays.
74// - Much more conservative adjustments to the far-end read pointer. We smooth
75// the delay difference more heavily, and back off from the difference more.
76// Adjustments force a readaptation of the filter, so they should be avoided
77// except when really necessary.
Henrik Lundin441f6342015-06-09 16:03:13 +020078struct ExtendedFilter {
79 ExtendedFilter() : enabled(false) {}
80 explicit ExtendedFilter(bool enabled) : enabled(enabled) {}
aluebs688e3082016-01-14 04:32:46 -080081 static const ConfigOptionID identifier = ConfigOptionID::kExtendedFilter;
Henrik Lundin441f6342015-06-09 16:03:13 +020082 bool enabled;
83};
andrew@webrtc.org6b1e2192013-09-25 23:46:20 +000084
peah0332c2d2016-04-15 11:23:33 -070085// Enables the refined linear filter adaptation in the echo canceller.
86// This configuration only applies to EchoCancellation and not
87// EchoControlMobile. It can be set in the constructor
88// or using AudioProcessing::SetExtraOptions().
89struct RefinedAdaptiveFilter {
90 RefinedAdaptiveFilter() : enabled(false) {}
91 explicit RefinedAdaptiveFilter(bool enabled) : enabled(enabled) {}
92 static const ConfigOptionID identifier =
93 ConfigOptionID::kAecRefinedAdaptiveFilter;
94 bool enabled;
95};
96
henrik.lundin366e9522015-07-03 00:50:05 -070097// Enables delay-agnostic echo cancellation. This feature relies on internally
98// estimated delays between the process and reverse streams, thus not relying
99// on reported system delays. This configuration only applies to
100// EchoCancellation and not EchoControlMobile. It can be set in the constructor
101// or using AudioProcessing::SetExtraOptions().
henrik.lundin0f133b92015-07-02 00:17:55 -0700102struct DelayAgnostic {
103 DelayAgnostic() : enabled(false) {}
104 explicit DelayAgnostic(bool enabled) : enabled(enabled) {}
aluebs688e3082016-01-14 04:32:46 -0800105 static const ConfigOptionID identifier = ConfigOptionID::kDelayAgnostic;
henrik.lundin0f133b92015-07-02 00:17:55 -0700106 bool enabled;
107};
bjornv@webrtc.org3f830722014-06-11 04:48:11 +0000108
Bjorn Volckeradc46c42015-04-15 11:42:40 +0200109// Use to enable experimental gain control (AGC). At startup the experimental
110// AGC moves the microphone volume up to |startup_min_volume| if the current
111// microphone volume is set too low. The value is clamped to its operating range
112// [12, 255]. Here, 255 maps to 100%.
113//
114// Must be provided through AudioProcessing::Create(Confg&).
Bjorn Volckerfb494512015-04-22 06:39:58 +0200115#if defined(WEBRTC_CHROMIUM_BUILD)
Bjorn Volckeradc46c42015-04-15 11:42:40 +0200116static const int kAgcStartupMinVolume = 85;
Bjorn Volckerfb494512015-04-22 06:39:58 +0200117#else
118static const int kAgcStartupMinVolume = 0;
119#endif // defined(WEBRTC_CHROMIUM_BUILD)
Henrik Lundine3a4da92017-11-06 11:42:21 +0100120static constexpr int kClippedLevelMin = 70;
andrew@webrtc.orgc7c7a532014-01-29 04:57:25 +0000121struct ExperimentalAgc {
henrik.lundinbd681b92016-12-05 09:08:42 -0800122 ExperimentalAgc() = default;
123 explicit ExperimentalAgc(bool enabled) : enabled(enabled) {}
Bjorn Volckeradc46c42015-04-15 11:42:40 +0200124 ExperimentalAgc(bool enabled, int startup_min_volume)
125 : enabled(enabled), startup_min_volume(startup_min_volume) {}
henrik.lundinbd681b92016-12-05 09:08:42 -0800126 ExperimentalAgc(bool enabled, int startup_min_volume, int clipped_level_min)
127 : enabled(enabled),
128 startup_min_volume(startup_min_volume),
129 clipped_level_min(clipped_level_min) {}
aluebs688e3082016-01-14 04:32:46 -0800130 static const ConfigOptionID identifier = ConfigOptionID::kExperimentalAgc;
henrik.lundinbd681b92016-12-05 09:08:42 -0800131 bool enabled = true;
132 int startup_min_volume = kAgcStartupMinVolume;
133 // Lowest microphone level that will be applied in response to clipping.
134 int clipped_level_min = kClippedLevelMin;
andrew@webrtc.org6b1e2192013-09-25 23:46:20 +0000135};
136
aluebs@webrtc.org9825afc2014-06-30 17:39:53 +0000137// Use to enable experimental noise suppression. It can be set in the
138// constructor or using AudioProcessing::SetExtraOptions().
139struct ExperimentalNs {
140 ExperimentalNs() : enabled(false) {}
141 explicit ExperimentalNs(bool enabled) : enabled(enabled) {}
aluebs688e3082016-01-14 04:32:46 -0800142 static const ConfigOptionID identifier = ConfigOptionID::kExperimentalNs;
aluebs@webrtc.org9825afc2014-06-30 17:39:53 +0000143 bool enabled;
144};
145
aluebs@webrtc.orgae643ce2014-12-19 19:57:34 +0000146// Use to enable beamforming. Must be provided through the constructor. It will
147// have no impact if used with AudioProcessing::SetExtraOptions().
148struct Beamforming {
aleloi5f099802016-08-25 00:45:31 -0700149 Beamforming();
150 Beamforming(bool enabled, const std::vector<Point>& array_geometry);
Alejandro Luebscb3f9bd2015-10-29 18:21:34 -0700151 Beamforming(bool enabled,
152 const std::vector<Point>& array_geometry,
aleloi5f099802016-08-25 00:45:31 -0700153 SphericalPointf target_direction);
154 ~Beamforming();
155
aluebs688e3082016-01-14 04:32:46 -0800156 static const ConfigOptionID identifier = ConfigOptionID::kBeamforming;
aluebs@webrtc.orgfb7a0392015-01-05 21:58:58 +0000157 const bool enabled;
158 const std::vector<Point> array_geometry;
Alejandro Luebscb3f9bd2015-10-29 18:21:34 -0700159 const SphericalPointf target_direction;
aluebs@webrtc.orgae643ce2014-12-19 19:57:34 +0000160};
161
Alejandro Luebsc9b0c262016-05-16 15:32:38 -0700162// Use to enable intelligibility enhancer in audio processing.
ekmeyerson60d9b332015-08-14 10:35:55 -0700163//
164// Note: If enabled and the reverse stream has more than one output channel,
165// the reverse stream will become an upmixed mono signal.
166struct Intelligibility {
167 Intelligibility() : enabled(false) {}
168 explicit Intelligibility(bool enabled) : enabled(enabled) {}
aluebs688e3082016-01-14 04:32:46 -0800169 static const ConfigOptionID identifier = ConfigOptionID::kIntelligibility;
ekmeyerson60d9b332015-08-14 10:35:55 -0700170 bool enabled;
171};
172
niklase@google.com470e71d2011-07-07 08:21:25 +0000173// The Audio Processing Module (APM) provides a collection of voice processing
174// components designed for real-time communications software.
175//
176// APM operates on two audio streams on a frame-by-frame basis. Frames of the
177// primary stream, on which all processing is applied, are passed to
aluebsb0319552016-03-17 20:39:53 -0700178// |ProcessStream()|. Frames of the reverse direction stream are passed to
179// |ProcessReverseStream()|. On the client-side, this will typically be the
180// near-end (capture) and far-end (render) streams, respectively. APM should be
181// placed in the signal chain as close to the audio hardware abstraction layer
182// (HAL) as possible.
niklase@google.com470e71d2011-07-07 08:21:25 +0000183//
184// On the server-side, the reverse stream will normally not be used, with
185// processing occurring on each incoming stream.
186//
187// Component interfaces follow a similar pattern and are accessed through
188// corresponding getters in APM. All components are disabled at create-time,
189// with default settings that are recommended for most situations. New settings
190// can be applied without enabling a component. Enabling a component triggers
191// memory allocation and initialization to allow it to start processing the
192// streams.
193//
194// Thread safety is provided with the following assumptions to reduce locking
195// overhead:
196// 1. The stream getters and setters are called from the same thread as
197// ProcessStream(). More precisely, stream functions are never called
198// concurrently with ProcessStream().
199// 2. Parameter getters are never called concurrently with the corresponding
200// setter.
201//
andrew@webrtc.orgddbb8a22014-04-22 21:00:04 +0000202// APM accepts only linear PCM audio data in chunks of 10 ms. The int16
203// interfaces use interleaved data, while the float interfaces use deinterleaved
204// data.
niklase@google.com470e71d2011-07-07 08:21:25 +0000205//
206// Usage example, omitting error checking:
207// AudioProcessing* apm = AudioProcessing::Create(0);
niklase@google.com470e71d2011-07-07 08:21:25 +0000208//
peah88ac8532016-09-12 16:47:25 -0700209// AudioProcessing::Config config;
210// config.level_controller.enabled = true;
peah8271d042016-11-22 07:24:52 -0800211// config.high_pass_filter.enabled = true;
peah88ac8532016-09-12 16:47:25 -0700212// apm->ApplyConfig(config)
213//
niklase@google.com470e71d2011-07-07 08:21:25 +0000214// apm->echo_cancellation()->enable_drift_compensation(false);
215// apm->echo_cancellation()->Enable(true);
216//
217// apm->noise_reduction()->set_level(kHighSuppression);
218// apm->noise_reduction()->Enable(true);
219//
220// apm->gain_control()->set_analog_level_limits(0, 255);
221// apm->gain_control()->set_mode(kAdaptiveAnalog);
222// apm->gain_control()->Enable(true);
223//
224// apm->voice_detection()->Enable(true);
225//
226// // Start a voice call...
227//
228// // ... Render frame arrives bound for the audio HAL ...
aluebsb0319552016-03-17 20:39:53 -0700229// apm->ProcessReverseStream(render_frame);
niklase@google.com470e71d2011-07-07 08:21:25 +0000230//
231// // ... Capture frame arrives from the audio HAL ...
232// // Call required set_stream_ functions.
233// apm->set_stream_delay_ms(delay_ms);
234// apm->gain_control()->set_stream_analog_level(analog_level);
235//
236// apm->ProcessStream(capture_frame);
237//
238// // Call required stream_ functions.
239// analog_level = apm->gain_control()->stream_analog_level();
240// has_voice = apm->stream_has_voice();
241//
242// // Repeate render and capture processing for the duration of the call...
243// // Start a new call...
244// apm->Initialize();
245//
246// // Close the application...
andrew@webrtc.orgf3930e92013-09-18 22:37:32 +0000247// delete apm;
niklase@google.com470e71d2011-07-07 08:21:25 +0000248//
peaha9cc40b2017-06-29 08:32:09 -0700249class AudioProcessing : public rtc::RefCountInterface {
niklase@google.com470e71d2011-07-07 08:21:25 +0000250 public:
peah88ac8532016-09-12 16:47:25 -0700251 // The struct below constitutes the new parameter scheme for the audio
252 // processing. It is being introduced gradually and until it is fully
253 // introduced, it is prone to change.
254 // TODO(peah): Remove this comment once the new config scheme is fully rolled
255 // out.
256 //
257 // The parameters and behavior of the audio processing module are controlled
258 // by changing the default values in the AudioProcessing::Config struct.
259 // The config is applied by passing the struct to the ApplyConfig method.
260 struct Config {
261 struct LevelController {
262 bool enabled = false;
peahc19f3122016-10-07 14:54:10 -0700263
264 // Sets the initial peak level to use inside the level controller in order
265 // to compute the signal gain. The unit for the peak level is dBFS and
266 // the allowed range is [-100, 0].
267 float initial_peak_level_dbfs = -6.0206f;
peah88ac8532016-09-12 16:47:25 -0700268 } level_controller;
ivoc9f4a4a02016-10-28 05:39:16 -0700269 struct ResidualEchoDetector {
ivocb829d9f2016-11-15 02:34:47 -0800270 bool enabled = true;
ivoc9f4a4a02016-10-28 05:39:16 -0700271 } residual_echo_detector;
peah8271d042016-11-22 07:24:52 -0800272
273 struct HighPassFilter {
274 bool enabled = false;
275 } high_pass_filter;
peahe0eae3c2016-12-14 01:16:23 -0800276
Gustaf Ullbergbd83b912017-10-18 12:32:42 +0200277 // Deprecated way of activating AEC3.
278 // TODO(gustaf): Remove when possible.
peahe0eae3c2016-12-14 01:16:23 -0800279 struct EchoCanceller3 {
280 bool enabled = false;
281 } echo_canceller3;
alessiob3ec96df2017-05-22 06:57:06 -0700282
283 // Enables the next generation AGC functionality. This feature replaces the
284 // standard methods of gain control in the previous AGC.
285 // The functionality is not yet activated in the code and turning this on
286 // does not yet have the desired behavior.
287 struct GainController2 {
288 bool enabled = false;
Alessio Bazzica270f7b52017-10-13 11:05:17 +0200289 float fixed_gain_db = 0.f;
alessiob3ec96df2017-05-22 06:57:06 -0700290 } gain_controller2;
peah8cee56f2017-08-24 22:36:53 -0700291
292 // Explicit copy assignment implementation to avoid issues with memory
293 // sanitizer complaints in case of self-assignment.
294 // TODO(peah): Add buildflag to ensure that this is only included for memory
295 // sanitizer builds.
296 Config& operator=(const Config& config) {
297 if (this != &config) {
298 memcpy(this, &config, sizeof(*this));
299 }
300 return *this;
301 }
peah88ac8532016-09-12 16:47:25 -0700302 };
303
Michael Graczyk86c6d332015-07-23 11:41:39 -0700304 // TODO(mgraczyk): Remove once all methods that use ChannelLayout are gone.
andrew@webrtc.org17e40642014-03-04 20:58:13 +0000305 enum ChannelLayout {
306 kMono,
307 // Left, right.
308 kStereo,
peah88ac8532016-09-12 16:47:25 -0700309 // Mono, keyboard, and mic.
andrew@webrtc.org17e40642014-03-04 20:58:13 +0000310 kMonoAndKeyboard,
peah88ac8532016-09-12 16:47:25 -0700311 // Left, right, keyboard, and mic.
andrew@webrtc.org17e40642014-03-04 20:58:13 +0000312 kStereoAndKeyboard
313 };
314
andrew@webrtc.org54744912014-02-05 06:30:29 +0000315 // Creates an APM instance. Use one instance for every primary audio stream
316 // requiring processing. On the client-side, this would typically be one
317 // instance for the near-end stream, and additional instances for each far-end
318 // stream which requires processing. On the server-side, this would typically
319 // be one instance for every incoming stream.
andrew@webrtc.orge84978f2014-01-25 02:09:06 +0000320 static AudioProcessing* Create();
andrew@webrtc.org54744912014-02-05 06:30:29 +0000321 // Allows passing in an optional configuration at create-time.
peah88ac8532016-09-12 16:47:25 -0700322 static AudioProcessing* Create(const webrtc::Config& config);
Alex Loiko5825aa62017-12-18 16:02:40 +0100323 // Deprecated. Use the Create below, with nullptr CustomProcessing.
Sam Zackrisson0beac582017-09-25 12:04:02 +0200324 RTC_DEPRECATED
peah88ac8532016-09-12 16:47:25 -0700325 static AudioProcessing* Create(const webrtc::Config& config,
Alejandro Luebsf4022ff2016-07-01 17:19:09 -0700326 NonlinearBeamformer* beamformer);
Alex Loiko5825aa62017-12-18 16:02:40 +0100327
328 // Will be deprecated and removed as part of webrtc:8665. Use the
329 // Create below, with nullptr CustomProcessing.
330 static AudioProcessing* Create(
331 const webrtc::Config& config,
332 std::unique_ptr<CustomProcessing> capture_post_processor,
333 std::unique_ptr<EchoControlFactory> echo_control_factory,
334 NonlinearBeamformer* beamformer);
335
Sam Zackrisson0beac582017-09-25 12:04:02 +0200336 // Allows passing in optional user-defined processing modules.
337 static AudioProcessing* Create(
338 const webrtc::Config& config,
Alex Loiko5825aa62017-12-18 16:02:40 +0100339 std::unique_ptr<CustomProcessing> capture_post_processor,
340 std::unique_ptr<CustomProcessing> render_pre_processor,
Gustaf Ullberg002ef282017-10-12 15:13:17 +0200341 std::unique_ptr<EchoControlFactory> echo_control_factory,
Sam Zackrisson0beac582017-09-25 12:04:02 +0200342 NonlinearBeamformer* beamformer);
peaha9cc40b2017-06-29 08:32:09 -0700343 ~AudioProcessing() override {}
niklase@google.com470e71d2011-07-07 08:21:25 +0000344
niklase@google.com470e71d2011-07-07 08:21:25 +0000345 // Initializes internal states, while retaining all user settings. This
346 // should be called before beginning to process a new audio stream. However,
347 // it is not necessary to call before processing the first stream after
andrew@webrtc.orgddbb8a22014-04-22 21:00:04 +0000348 // creation.
349 //
350 // It is also not necessary to call if the audio parameters (sample
andrew@webrtc.org60730cf2014-01-07 17:45:09 +0000351 // rate and number of channels) have changed. Passing updated parameters
aluebsb0319552016-03-17 20:39:53 -0700352 // directly to |ProcessStream()| and |ProcessReverseStream()| is permissible.
andrew@webrtc.orgddbb8a22014-04-22 21:00:04 +0000353 // If the parameters are known at init-time though, they may be provided.
niklase@google.com470e71d2011-07-07 08:21:25 +0000354 virtual int Initialize() = 0;
andrew@webrtc.orgddbb8a22014-04-22 21:00:04 +0000355
356 // The int16 interfaces require:
357 // - only |NativeRate|s be used
358 // - that the input, output and reverse rates must match
Michael Graczyk86c6d332015-07-23 11:41:39 -0700359 // - that |processing_config.output_stream()| matches
360 // |processing_config.input_stream()|.
andrew@webrtc.orgddbb8a22014-04-22 21:00:04 +0000361 //
Michael Graczyk86c6d332015-07-23 11:41:39 -0700362 // The float interfaces accept arbitrary rates and support differing input and
363 // output layouts, but the output must have either one channel or the same
364 // number of channels as the input.
365 virtual int Initialize(const ProcessingConfig& processing_config) = 0;
366
367 // Initialize with unpacked parameters. See Initialize() above for details.
368 //
369 // TODO(mgraczyk): Remove once clients are updated to use the new interface.
peahde65ddc2016-09-16 15:02:15 -0700370 virtual int Initialize(int capture_input_sample_rate_hz,
371 int capture_output_sample_rate_hz,
372 int render_sample_rate_hz,
373 ChannelLayout capture_input_layout,
374 ChannelLayout capture_output_layout,
375 ChannelLayout render_input_layout) = 0;
niklase@google.com470e71d2011-07-07 08:21:25 +0000376
peah88ac8532016-09-12 16:47:25 -0700377 // TODO(peah): This method is a temporary solution used to take control
378 // over the parameters in the audio processing module and is likely to change.
379 virtual void ApplyConfig(const Config& config) = 0;
380
andrew@webrtc.org61e596f2013-07-25 18:28:29 +0000381 // Pass down additional options which don't have explicit setters. This
382 // ensures the options are applied immediately.
peah88ac8532016-09-12 16:47:25 -0700383 virtual void SetExtraOptions(const webrtc::Config& config) = 0;
andrew@webrtc.org61e596f2013-07-25 18:28:29 +0000384
andrew@webrtc.orgddbb8a22014-04-22 21:00:04 +0000385 // TODO(ajm): Only intended for internal use. Make private and friend the
386 // necessary classes?
387 virtual int proc_sample_rate_hz() const = 0;
388 virtual int proc_split_sample_rate_hz() const = 0;
Peter Kasting69558702016-01-12 16:26:35 -0800389 virtual size_t num_input_channels() const = 0;
390 virtual size_t num_proc_channels() const = 0;
391 virtual size_t num_output_channels() const = 0;
392 virtual size_t num_reverse_channels() const = 0;
niklase@google.com470e71d2011-07-07 08:21:25 +0000393
andrew@webrtc.org17342e52014-02-12 22:28:31 +0000394 // Set to true when the output of AudioProcessing will be muted or in some
395 // other way not used. Ideally, the captured audio would still be processed,
396 // but some components may change behavior based on this information.
397 // Default false.
398 virtual void set_output_will_be_muted(bool muted) = 0;
andrew@webrtc.org17342e52014-02-12 22:28:31 +0000399
niklase@google.com470e71d2011-07-07 08:21:25 +0000400 // Processes a 10 ms |frame| of the primary audio stream. On the client-side,
401 // this is the near-end (or captured) audio.
402 //
403 // If needed for enabled functionality, any function with the set_stream_ tag
404 // must be called prior to processing the current frame. Any getter function
405 // with the stream_ tag which is needed should be called after processing.
406 //
andrew@webrtc.org63a50982012-05-02 23:56:37 +0000407 // The |sample_rate_hz_|, |num_channels_|, and |samples_per_channel_|
andrew@webrtc.org60730cf2014-01-07 17:45:09 +0000408 // members of |frame| must be valid. If changed from the previous call to this
409 // method, it will trigger an initialization.
niklase@google.com470e71d2011-07-07 08:21:25 +0000410 virtual int ProcessStream(AudioFrame* frame) = 0;
411
andrew@webrtc.org17e40642014-03-04 20:58:13 +0000412 // Accepts deinterleaved float audio with the range [-1, 1]. Each element
andrew@webrtc.orgddbb8a22014-04-22 21:00:04 +0000413 // of |src| points to a channel buffer, arranged according to
andrew@webrtc.org17e40642014-03-04 20:58:13 +0000414 // |input_layout|. At output, the channels will be arranged according to
andrew@webrtc.orgddbb8a22014-04-22 21:00:04 +0000415 // |output_layout| at |output_sample_rate_hz| in |dest|.
416 //
Michael Graczyk86c6d332015-07-23 11:41:39 -0700417 // The output layout must have one channel or as many channels as the input.
418 // |src| and |dest| may use the same memory, if desired.
419 //
420 // TODO(mgraczyk): Remove once clients are updated to use the new interface.
andrew@webrtc.orgddbb8a22014-04-22 21:00:04 +0000421 virtual int ProcessStream(const float* const* src,
Peter Kastingdce40cf2015-08-24 14:52:23 -0700422 size_t samples_per_channel,
andrew@webrtc.orgddbb8a22014-04-22 21:00:04 +0000423 int input_sample_rate_hz,
andrew@webrtc.org17e40642014-03-04 20:58:13 +0000424 ChannelLayout input_layout,
andrew@webrtc.orgddbb8a22014-04-22 21:00:04 +0000425 int output_sample_rate_hz,
426 ChannelLayout output_layout,
427 float* const* dest) = 0;
andrew@webrtc.org17e40642014-03-04 20:58:13 +0000428
Michael Graczyk86c6d332015-07-23 11:41:39 -0700429 // Accepts deinterleaved float audio with the range [-1, 1]. Each element of
430 // |src| points to a channel buffer, arranged according to |input_stream|. At
431 // output, the channels will be arranged according to |output_stream| in
432 // |dest|.
433 //
434 // The output must have one channel or as many channels as the input. |src|
435 // and |dest| may use the same memory, if desired.
436 virtual int ProcessStream(const float* const* src,
437 const StreamConfig& input_config,
438 const StreamConfig& output_config,
439 float* const* dest) = 0;
440
aluebsb0319552016-03-17 20:39:53 -0700441 // Processes a 10 ms |frame| of the reverse direction audio stream. The frame
442 // may be modified. On the client-side, this is the far-end (or to be
niklase@google.com470e71d2011-07-07 08:21:25 +0000443 // rendered) audio.
444 //
aluebsb0319552016-03-17 20:39:53 -0700445 // It is necessary to provide this if echo processing is enabled, as the
niklase@google.com470e71d2011-07-07 08:21:25 +0000446 // reverse stream forms the echo reference signal. It is recommended, but not
447 // necessary, to provide if gain control is enabled. On the server-side this
448 // typically will not be used. If you're not sure what to pass in here,
449 // chances are you don't need to use it.
450 //
andrew@webrtc.org63a50982012-05-02 23:56:37 +0000451 // The |sample_rate_hz_|, |num_channels_|, and |samples_per_channel_|
aluebsda116c42016-03-17 16:43:29 -0700452 // members of |frame| must be valid.
ekmeyerson60d9b332015-08-14 10:35:55 -0700453 virtual int ProcessReverseStream(AudioFrame* frame) = 0;
454
andrew@webrtc.org17e40642014-03-04 20:58:13 +0000455 // Accepts deinterleaved float audio with the range [-1, 1]. Each element
456 // of |data| points to a channel buffer, arranged according to |layout|.
Michael Graczyk86c6d332015-07-23 11:41:39 -0700457 // TODO(mgraczyk): Remove once clients are updated to use the new interface.
andrew@webrtc.org17e40642014-03-04 20:58:13 +0000458 virtual int AnalyzeReverseStream(const float* const* data,
Peter Kastingdce40cf2015-08-24 14:52:23 -0700459 size_t samples_per_channel,
peahde65ddc2016-09-16 15:02:15 -0700460 int sample_rate_hz,
andrew@webrtc.org17e40642014-03-04 20:58:13 +0000461 ChannelLayout layout) = 0;
462
Michael Graczyk86c6d332015-07-23 11:41:39 -0700463 // Accepts deinterleaved float audio with the range [-1, 1]. Each element of
464 // |data| points to a channel buffer, arranged according to |reverse_config|.
ekmeyerson60d9b332015-08-14 10:35:55 -0700465 virtual int ProcessReverseStream(const float* const* src,
peahde65ddc2016-09-16 15:02:15 -0700466 const StreamConfig& input_config,
467 const StreamConfig& output_config,
ekmeyerson60d9b332015-08-14 10:35:55 -0700468 float* const* dest) = 0;
Michael Graczyk86c6d332015-07-23 11:41:39 -0700469
niklase@google.com470e71d2011-07-07 08:21:25 +0000470 // This must be called if and only if echo processing is enabled.
471 //
aluebsb0319552016-03-17 20:39:53 -0700472 // Sets the |delay| in ms between ProcessReverseStream() receiving a far-end
niklase@google.com470e71d2011-07-07 08:21:25 +0000473 // frame and ProcessStream() receiving a near-end frame containing the
474 // corresponding echo. On the client-side this can be expressed as
475 // delay = (t_render - t_analyze) + (t_process - t_capture)
476 // where,
aluebsb0319552016-03-17 20:39:53 -0700477 // - t_analyze is the time a frame is passed to ProcessReverseStream() and
niklase@google.com470e71d2011-07-07 08:21:25 +0000478 // t_render is the time the first sample of the same frame is rendered by
479 // the audio hardware.
480 // - t_capture is the time the first sample of a frame is captured by the
alessiob13fc1802017-04-19 05:35:51 -0700481 // audio hardware and t_process is the time the same frame is passed to
niklase@google.com470e71d2011-07-07 08:21:25 +0000482 // ProcessStream().
483 virtual int set_stream_delay_ms(int delay) = 0;
484 virtual int stream_delay_ms() const = 0;
andrew@webrtc.org56e4a052014-02-27 22:23:17 +0000485 virtual bool was_stream_delay_set() const = 0;
niklase@google.com470e71d2011-07-07 08:21:25 +0000486
andrew@webrtc.org75dd2882014-02-11 20:52:30 +0000487 // Call to signal that a key press occurred (true) or did not occur (false)
488 // with this chunk of audio.
489 virtual void set_stream_key_pressed(bool key_pressed) = 0;
andrew@webrtc.org75dd2882014-02-11 20:52:30 +0000490
andrew@webrtc.org6f9f8172012-03-06 19:03:39 +0000491 // Sets a delay |offset| in ms to add to the values passed in through
492 // set_stream_delay_ms(). May be positive or negative.
493 //
494 // Note that this could cause an otherwise valid value passed to
495 // set_stream_delay_ms() to return an error.
496 virtual void set_delay_offset_ms(int offset) = 0;
497 virtual int delay_offset_ms() const = 0;
498
aleloi868f32f2017-05-23 07:20:05 -0700499 // Attaches provided webrtc::AecDump for recording debugging
500 // information. Log file and maximum file size logic is supposed to
501 // be handled by implementing instance of AecDump. Calling this
502 // method when another AecDump is attached resets the active AecDump
503 // with a new one. This causes the d-tor of the earlier AecDump to
504 // be called. The d-tor call may block until all pending logging
505 // tasks are completed.
Alex Loikobe767e02017-06-08 09:45:03 +0200506 virtual void AttachAecDump(std::unique_ptr<AecDump> aec_dump) = 0;
aleloi868f32f2017-05-23 07:20:05 -0700507
508 // If no AecDump is attached, this has no effect. If an AecDump is
509 // attached, it's destructor is called. The d-tor may block until
510 // all pending logging tasks are completed.
Alex Loikobe767e02017-06-08 09:45:03 +0200511 virtual void DetachAecDump() = 0;
aleloi868f32f2017-05-23 07:20:05 -0700512
Bjorn Volcker4e7aa432015-07-07 11:50:05 +0200513 // Use to send UMA histograms at end of a call. Note that all histogram
514 // specific member variables are reset.
515 virtual void UpdateHistogramsOnCallEnd() = 0;
516
ivoc3e9a5372016-10-28 07:55:33 -0700517 // TODO(ivoc): Remove when the calling code no longer uses the old Statistics
518 // API.
519 struct Statistic {
520 int instant = 0; // Instantaneous value.
521 int average = 0; // Long-term average.
522 int maximum = 0; // Long-term maximum.
523 int minimum = 0; // Long-term minimum.
524 };
525
526 struct Stat {
527 void Set(const Statistic& other) {
528 Set(other.instant, other.average, other.maximum, other.minimum);
529 }
530 void Set(float instant, float average, float maximum, float minimum) {
ivoc3e9a5372016-10-28 07:55:33 -0700531 instant_ = instant;
532 average_ = average;
533 maximum_ = maximum;
534 minimum_ = minimum;
535 }
536 float instant() const { return instant_; }
537 float average() const { return average_; }
538 float maximum() const { return maximum_; }
539 float minimum() const { return minimum_; }
540
541 private:
542 float instant_ = 0.0f; // Instantaneous value.
543 float average_ = 0.0f; // Long-term average.
544 float maximum_ = 0.0f; // Long-term maximum.
545 float minimum_ = 0.0f; // Long-term minimum.
546 };
547
548 struct AudioProcessingStatistics {
ivoc4e477a12017-01-15 08:29:46 -0800549 AudioProcessingStatistics();
550 AudioProcessingStatistics(const AudioProcessingStatistics& other);
551 ~AudioProcessingStatistics();
ivocd0a151c2016-11-02 09:14:37 -0700552
ivoc3e9a5372016-10-28 07:55:33 -0700553 // AEC Statistics.
554 // RERL = ERL + ERLE
555 Stat residual_echo_return_loss;
556 // ERL = 10log_10(P_far / P_echo)
557 Stat echo_return_loss;
558 // ERLE = 10log_10(P_echo / P_out)
559 Stat echo_return_loss_enhancement;
560 // (Pre non-linear processing suppression) A_NLP = 10log_10(P_echo / P_a)
561 Stat a_nlp;
562 // Fraction of time that the AEC linear filter is divergent, in a 1-second
563 // non-overlapped aggregation window.
ivocd0a151c2016-11-02 09:14:37 -0700564 float divergent_filter_fraction = -1.0f;
ivoc3e9a5372016-10-28 07:55:33 -0700565
566 // The delay metrics consists of the delay median and standard deviation. It
567 // also consists of the fraction of delay estimates that can make the echo
568 // cancellation perform poorly. The values are aggregated until the first
569 // call to |GetStatistics()| and afterwards aggregated and updated every
570 // second. Note that if there are several clients pulling metrics from
571 // |GetStatistics()| during a session the first call from any of them will
572 // change to one second aggregation window for all.
ivocd0a151c2016-11-02 09:14:37 -0700573 int delay_median = -1;
574 int delay_standard_deviation = -1;
575 float fraction_poor_delays = -1.0f;
ivoc3e9a5372016-10-28 07:55:33 -0700576
ivoc4e477a12017-01-15 08:29:46 -0800577 // Residual echo detector likelihood.
ivocd0a151c2016-11-02 09:14:37 -0700578 float residual_echo_likelihood = -1.0f;
ivoc4e477a12017-01-15 08:29:46 -0800579 // Maximum residual echo likelihood from the last time period.
580 float residual_echo_likelihood_recent_max = -1.0f;
ivoc3e9a5372016-10-28 07:55:33 -0700581 };
582
583 // TODO(ivoc): Make this pure virtual when all subclasses have been updated.
584 virtual AudioProcessingStatistics GetStatistics() const;
585
Ivo Creusenae026092017-11-20 13:07:16 +0100586 // This returns the stats as optionals and it will replace the regular
587 // GetStatistics.
588 virtual AudioProcessingStats GetStatistics(bool has_remote_tracks) const;
589
niklase@google.com470e71d2011-07-07 08:21:25 +0000590 // These provide access to the component interfaces and should never return
591 // NULL. The pointers will be valid for the lifetime of the APM instance.
592 // The memory for these objects is entirely managed internally.
593 virtual EchoCancellation* echo_cancellation() const = 0;
594 virtual EchoControlMobile* echo_control_mobile() const = 0;
595 virtual GainControl* gain_control() const = 0;
peah8271d042016-11-22 07:24:52 -0800596 // TODO(peah): Deprecate this API call.
niklase@google.com470e71d2011-07-07 08:21:25 +0000597 virtual HighPassFilter* high_pass_filter() const = 0;
598 virtual LevelEstimator* level_estimator() const = 0;
599 virtual NoiseSuppression* noise_suppression() const = 0;
600 virtual VoiceDetection* voice_detection() const = 0;
601
henrik.lundinadf06352017-04-05 05:48:24 -0700602 // Returns the last applied configuration.
henrik.lundin77492862017-04-06 23:28:09 -0700603 virtual AudioProcessing::Config GetConfig() const = 0;
henrik.lundinadf06352017-04-05 05:48:24 -0700604
andrew@webrtc.org648af742012-02-08 01:57:29 +0000605 enum Error {
606 // Fatal errors.
niklase@google.com470e71d2011-07-07 08:21:25 +0000607 kNoError = 0,
608 kUnspecifiedError = -1,
609 kCreationFailedError = -2,
610 kUnsupportedComponentError = -3,
611 kUnsupportedFunctionError = -4,
612 kNullPointerError = -5,
613 kBadParameterError = -6,
614 kBadSampleRateError = -7,
615 kBadDataLengthError = -8,
616 kBadNumberChannelsError = -9,
617 kFileError = -10,
618 kStreamParameterNotSetError = -11,
andrew@webrtc.org648af742012-02-08 01:57:29 +0000619 kNotEnabledError = -12,
niklase@google.com470e71d2011-07-07 08:21:25 +0000620
andrew@webrtc.org648af742012-02-08 01:57:29 +0000621 // Warnings are non-fatal.
niklase@google.com470e71d2011-07-07 08:21:25 +0000622 // This results when a set_stream_ parameter is out of range. Processing
623 // will continue, but the parameter may have been truncated.
andrew@webrtc.org648af742012-02-08 01:57:29 +0000624 kBadStreamParameterWarning = -13
niklase@google.com470e71d2011-07-07 08:21:25 +0000625 };
andrew@webrtc.org56e4a052014-02-27 22:23:17 +0000626
andrew@webrtc.orgddbb8a22014-04-22 21:00:04 +0000627 enum NativeRate {
andrew@webrtc.org56e4a052014-02-27 22:23:17 +0000628 kSampleRate8kHz = 8000,
629 kSampleRate16kHz = 16000,
aluebs@webrtc.org087da132014-11-17 23:01:23 +0000630 kSampleRate32kHz = 32000,
631 kSampleRate48kHz = 48000
andrew@webrtc.org56e4a052014-02-27 22:23:17 +0000632 };
andrew@webrtc.orgddbb8a22014-04-22 21:00:04 +0000633
kwibergd59d3bb2016-09-13 07:49:33 -0700634 // TODO(kwiberg): We currently need to support a compiler (Visual C++) that
635 // complains if we don't explicitly state the size of the array here. Remove
636 // the size when that's no longer the case.
637 static constexpr int kNativeSampleRatesHz[4] = {
638 kSampleRate8kHz, kSampleRate16kHz, kSampleRate32kHz, kSampleRate48kHz};
639 static constexpr size_t kNumNativeSampleRates =
640 arraysize(kNativeSampleRatesHz);
641 static constexpr int kMaxNativeSampleRateHz =
642 kNativeSampleRatesHz[kNumNativeSampleRates - 1];
Alejandro Luebscdfe20b2015-09-23 12:49:12 -0700643
andrew@webrtc.orgddbb8a22014-04-22 21:00:04 +0000644 static const int kChunkSizeMs = 10;
niklase@google.com470e71d2011-07-07 08:21:25 +0000645};
646
Michael Graczyk86c6d332015-07-23 11:41:39 -0700647class StreamConfig {
648 public:
649 // sample_rate_hz: The sampling rate of the stream.
650 //
651 // num_channels: The number of audio channels in the stream, excluding the
652 // keyboard channel if it is present. When passing a
653 // StreamConfig with an array of arrays T*[N],
654 //
655 // N == {num_channels + 1 if has_keyboard
656 // {num_channels if !has_keyboard
657 //
658 // has_keyboard: True if the stream has a keyboard channel. When has_keyboard
659 // is true, the last channel in any corresponding list of
660 // channels is the keyboard channel.
661 StreamConfig(int sample_rate_hz = 0,
Peter Kasting69558702016-01-12 16:26:35 -0800662 size_t num_channels = 0,
Michael Graczyk86c6d332015-07-23 11:41:39 -0700663 bool has_keyboard = false)
664 : sample_rate_hz_(sample_rate_hz),
665 num_channels_(num_channels),
666 has_keyboard_(has_keyboard),
667 num_frames_(calculate_frames(sample_rate_hz)) {}
668
669 void set_sample_rate_hz(int value) {
670 sample_rate_hz_ = value;
671 num_frames_ = calculate_frames(value);
672 }
Peter Kasting69558702016-01-12 16:26:35 -0800673 void set_num_channels(size_t value) { num_channels_ = value; }
Michael Graczyk86c6d332015-07-23 11:41:39 -0700674 void set_has_keyboard(bool value) { has_keyboard_ = value; }
675
676 int sample_rate_hz() const { return sample_rate_hz_; }
677
678 // The number of channels in the stream, not including the keyboard channel if
679 // present.
Peter Kasting69558702016-01-12 16:26:35 -0800680 size_t num_channels() const { return num_channels_; }
Michael Graczyk86c6d332015-07-23 11:41:39 -0700681
682 bool has_keyboard() const { return has_keyboard_; }
Peter Kastingdce40cf2015-08-24 14:52:23 -0700683 size_t num_frames() const { return num_frames_; }
684 size_t num_samples() const { return num_channels_ * num_frames_; }
Michael Graczyk86c6d332015-07-23 11:41:39 -0700685
686 bool operator==(const StreamConfig& other) const {
687 return sample_rate_hz_ == other.sample_rate_hz_ &&
688 num_channels_ == other.num_channels_ &&
689 has_keyboard_ == other.has_keyboard_;
690 }
691
692 bool operator!=(const StreamConfig& other) const { return !(*this == other); }
693
694 private:
Peter Kastingdce40cf2015-08-24 14:52:23 -0700695 static size_t calculate_frames(int sample_rate_hz) {
696 return static_cast<size_t>(
697 AudioProcessing::kChunkSizeMs * sample_rate_hz / 1000);
Michael Graczyk86c6d332015-07-23 11:41:39 -0700698 }
699
700 int sample_rate_hz_;
Peter Kasting69558702016-01-12 16:26:35 -0800701 size_t num_channels_;
Michael Graczyk86c6d332015-07-23 11:41:39 -0700702 bool has_keyboard_;
Peter Kastingdce40cf2015-08-24 14:52:23 -0700703 size_t num_frames_;
Michael Graczyk86c6d332015-07-23 11:41:39 -0700704};
705
706class ProcessingConfig {
707 public:
708 enum StreamName {
709 kInputStream,
710 kOutputStream,
ekmeyerson60d9b332015-08-14 10:35:55 -0700711 kReverseInputStream,
712 kReverseOutputStream,
Michael Graczyk86c6d332015-07-23 11:41:39 -0700713 kNumStreamNames,
714 };
715
716 const StreamConfig& input_stream() const {
717 return streams[StreamName::kInputStream];
718 }
719 const StreamConfig& output_stream() const {
720 return streams[StreamName::kOutputStream];
721 }
ekmeyerson60d9b332015-08-14 10:35:55 -0700722 const StreamConfig& reverse_input_stream() const {
723 return streams[StreamName::kReverseInputStream];
724 }
725 const StreamConfig& reverse_output_stream() const {
726 return streams[StreamName::kReverseOutputStream];
Michael Graczyk86c6d332015-07-23 11:41:39 -0700727 }
728
729 StreamConfig& input_stream() { return streams[StreamName::kInputStream]; }
730 StreamConfig& output_stream() { return streams[StreamName::kOutputStream]; }
ekmeyerson60d9b332015-08-14 10:35:55 -0700731 StreamConfig& reverse_input_stream() {
732 return streams[StreamName::kReverseInputStream];
733 }
734 StreamConfig& reverse_output_stream() {
735 return streams[StreamName::kReverseOutputStream];
736 }
Michael Graczyk86c6d332015-07-23 11:41:39 -0700737
738 bool operator==(const ProcessingConfig& other) const {
739 for (int i = 0; i < StreamName::kNumStreamNames; ++i) {
740 if (this->streams[i] != other.streams[i]) {
741 return false;
742 }
743 }
744 return true;
745 }
746
747 bool operator!=(const ProcessingConfig& other) const {
748 return !(*this == other);
749 }
750
751 StreamConfig streams[StreamName::kNumStreamNames];
752};
753
niklase@google.com470e71d2011-07-07 08:21:25 +0000754// The acoustic echo cancellation (AEC) component provides better performance
755// than AECM but also requires more processing power and is dependent on delay
756// stability and reporting accuracy. As such it is well-suited and recommended
757// for PC and IP phone applications.
758//
759// Not recommended to be enabled on the server-side.
760class EchoCancellation {
761 public:
762 // EchoCancellation and EchoControlMobile may not be enabled simultaneously.
763 // Enabling one will disable the other.
764 virtual int Enable(bool enable) = 0;
765 virtual bool is_enabled() const = 0;
766
767 // Differences in clock speed on the primary and reverse streams can impact
768 // the AEC performance. On the client-side, this could be seen when different
769 // render and capture devices are used, particularly with webcams.
770 //
771 // This enables a compensation mechanism, and requires that
andrew@webrtc.orgddbb8a22014-04-22 21:00:04 +0000772 // set_stream_drift_samples() be called.
niklase@google.com470e71d2011-07-07 08:21:25 +0000773 virtual int enable_drift_compensation(bool enable) = 0;
774 virtual bool is_drift_compensation_enabled() const = 0;
775
niklase@google.com470e71d2011-07-07 08:21:25 +0000776 // Sets the difference between the number of samples rendered and captured by
777 // the audio devices since the last call to |ProcessStream()|. Must be called
andrew@webrtc.org6be1e932013-03-01 18:47:28 +0000778 // if drift compensation is enabled, prior to |ProcessStream()|.
779 virtual void set_stream_drift_samples(int drift) = 0;
niklase@google.com470e71d2011-07-07 08:21:25 +0000780 virtual int stream_drift_samples() const = 0;
781
782 enum SuppressionLevel {
783 kLowSuppression,
784 kModerateSuppression,
785 kHighSuppression
786 };
787
788 // Sets the aggressiveness of the suppressor. A higher level trades off
789 // double-talk performance for increased echo suppression.
790 virtual int set_suppression_level(SuppressionLevel level) = 0;
791 virtual SuppressionLevel suppression_level() const = 0;
792
793 // Returns false if the current frame almost certainly contains no echo
794 // and true if it _might_ contain echo.
795 virtual bool stream_has_echo() const = 0;
796
797 // Enables the computation of various echo metrics. These are obtained
798 // through |GetMetrics()|.
799 virtual int enable_metrics(bool enable) = 0;
800 virtual bool are_metrics_enabled() const = 0;
801
802 // Each statistic is reported in dB.
803 // P_far: Far-end (render) signal power.
804 // P_echo: Near-end (capture) echo signal power.
805 // P_out: Signal power at the output of the AEC.
806 // P_a: Internal signal power at the point before the AEC's non-linear
807 // processor.
808 struct Metrics {
809 // RERL = ERL + ERLE
810 AudioProcessing::Statistic residual_echo_return_loss;
811
812 // ERL = 10log_10(P_far / P_echo)
813 AudioProcessing::Statistic echo_return_loss;
814
815 // ERLE = 10log_10(P_echo / P_out)
816 AudioProcessing::Statistic echo_return_loss_enhancement;
817
818 // (Pre non-linear processing suppression) A_NLP = 10log_10(P_echo / P_a)
819 AudioProcessing::Statistic a_nlp;
minyue50453372016-04-07 06:36:43 -0700820
minyue38156552016-05-03 14:42:41 -0700821 // Fraction of time that the AEC linear filter is divergent, in a 1-second
minyue50453372016-04-07 06:36:43 -0700822 // non-overlapped aggregation window.
823 float divergent_filter_fraction;
niklase@google.com470e71d2011-07-07 08:21:25 +0000824 };
825
ivoc3e9a5372016-10-28 07:55:33 -0700826 // Deprecated. Use GetStatistics on the AudioProcessing interface instead.
niklase@google.com470e71d2011-07-07 08:21:25 +0000827 // TODO(ajm): discuss the metrics update period.
828 virtual int GetMetrics(Metrics* metrics) = 0;
829
bjornv@google.com1ba3dbe2011-10-03 08:18:10 +0000830 // Enables computation and logging of delay values. Statistics are obtained
831 // through |GetDelayMetrics()|.
832 virtual int enable_delay_logging(bool enable) = 0;
833 virtual bool is_delay_logging_enabled() const = 0;
834
835 // The delay metrics consists of the delay |median| and the delay standard
bjornv@webrtc.orgb1786db2015-02-03 06:06:26 +0000836 // deviation |std|. It also consists of the fraction of delay estimates
837 // |fraction_poor_delays| that can make the echo cancellation perform poorly.
838 // The values are aggregated until the first call to |GetDelayMetrics()| and
839 // afterwards aggregated and updated every second.
840 // Note that if there are several clients pulling metrics from
841 // |GetDelayMetrics()| during a session the first call from any of them will
842 // change to one second aggregation window for all.
ivoc3e9a5372016-10-28 07:55:33 -0700843 // Deprecated. Use GetStatistics on the AudioProcessing interface instead.
bjornv@google.com1ba3dbe2011-10-03 08:18:10 +0000844 virtual int GetDelayMetrics(int* median, int* std) = 0;
ivoc3e9a5372016-10-28 07:55:33 -0700845 // Deprecated. Use GetStatistics on the AudioProcessing interface instead.
bjornv@webrtc.orgb1786db2015-02-03 06:06:26 +0000846 virtual int GetDelayMetrics(int* median, int* std,
847 float* fraction_poor_delays) = 0;
bjornv@google.com1ba3dbe2011-10-03 08:18:10 +0000848
bjornv@webrtc.org91d11b32013-03-05 16:53:09 +0000849 // Returns a pointer to the low level AEC component. In case of multiple
850 // channels, the pointer to the first one is returned. A NULL pointer is
851 // returned when the AEC component is disabled or has not been initialized
852 // successfully.
853 virtual struct AecCore* aec_core() const = 0;
854
niklase@google.com470e71d2011-07-07 08:21:25 +0000855 protected:
andrew@webrtc.orgd72b3d62012-11-15 21:46:06 +0000856 virtual ~EchoCancellation() {}
niklase@google.com470e71d2011-07-07 08:21:25 +0000857};
858
859// The acoustic echo control for mobile (AECM) component is a low complexity
860// robust option intended for use on mobile devices.
861//
862// Not recommended to be enabled on the server-side.
863class EchoControlMobile {
864 public:
865 // EchoCancellation and EchoControlMobile may not be enabled simultaneously.
866 // Enabling one will disable the other.
867 virtual int Enable(bool enable) = 0;
868 virtual bool is_enabled() const = 0;
869
870 // Recommended settings for particular audio routes. In general, the louder
871 // the echo is expected to be, the higher this value should be set. The
872 // preferred setting may vary from device to device.
873 enum RoutingMode {
874 kQuietEarpieceOrHeadset,
875 kEarpiece,
876 kLoudEarpiece,
877 kSpeakerphone,
878 kLoudSpeakerphone
879 };
880
881 // Sets echo control appropriate for the audio routing |mode| on the device.
882 // It can and should be updated during a call if the audio routing changes.
883 virtual int set_routing_mode(RoutingMode mode) = 0;
884 virtual RoutingMode routing_mode() const = 0;
885
886 // Comfort noise replaces suppressed background noise to maintain a
887 // consistent signal level.
888 virtual int enable_comfort_noise(bool enable) = 0;
889 virtual bool is_comfort_noise_enabled() const = 0;
890
bjornv@google.comc4b939c2011-07-13 08:09:56 +0000891 // A typical use case is to initialize the component with an echo path from a
ajm@google.com22e65152011-07-18 18:03:01 +0000892 // previous call. The echo path is retrieved using |GetEchoPath()|, typically
893 // at the end of a call. The data can then be stored for later use as an
894 // initializer before the next call, using |SetEchoPath()|.
895 //
bjornv@google.comc4b939c2011-07-13 08:09:56 +0000896 // Controlling the echo path this way requires the data |size_bytes| to match
897 // the internal echo path size. This size can be acquired using
898 // |echo_path_size_bytes()|. |SetEchoPath()| causes an entire reset, worth
ajm@google.com22e65152011-07-18 18:03:01 +0000899 // noting if it is to be called during an ongoing call.
900 //
901 // It is possible that version incompatibilities may result in a stored echo
902 // path of the incorrect size. In this case, the stored path should be
903 // discarded.
904 virtual int SetEchoPath(const void* echo_path, size_t size_bytes) = 0;
905 virtual int GetEchoPath(void* echo_path, size_t size_bytes) const = 0;
906
907 // The returned path size is guaranteed not to change for the lifetime of
908 // the application.
909 static size_t echo_path_size_bytes();
bjornv@google.comc4b939c2011-07-13 08:09:56 +0000910
niklase@google.com470e71d2011-07-07 08:21:25 +0000911 protected:
andrew@webrtc.orgd72b3d62012-11-15 21:46:06 +0000912 virtual ~EchoControlMobile() {}
niklase@google.com470e71d2011-07-07 08:21:25 +0000913};
914
Gustaf Ullbergc5222982017-10-05 10:25:05 +0200915// Interface for an acoustic echo cancellation (AEC) submodule.
916class EchoControl {
917 public:
918 // Analysis (not changing) of the render signal.
919 virtual void AnalyzeRender(AudioBuffer* render) = 0;
920
921 // Analysis (not changing) of the capture signal.
922 virtual void AnalyzeCapture(AudioBuffer* capture) = 0;
923
924 // Processes the capture signal in order to remove the echo.
925 virtual void ProcessCapture(AudioBuffer* capture, bool echo_path_change) = 0;
926
Gustaf Ullberg332150d2017-11-22 14:17:39 +0100927 struct Metrics {
928 double echo_return_loss;
929 double echo_return_loss_enhancement;
Per Åhgren83c4a022017-11-27 12:07:09 +0100930 int delay_ms;
Gustaf Ullberg332150d2017-11-22 14:17:39 +0100931 };
932
933 // Collect current metrics from the echo controller.
934 virtual Metrics GetMetrics() const = 0;
935
Gustaf Ullbergc5222982017-10-05 10:25:05 +0200936 virtual ~EchoControl() {}
937};
938
Gustaf Ullberg002ef282017-10-12 15:13:17 +0200939// Interface for a factory that creates EchoControllers.
940class EchoControlFactory {
941 public:
942 virtual std::unique_ptr<EchoControl> Create(int sample_rate_hz) = 0;
943 virtual ~EchoControlFactory() = default;
944};
945
niklase@google.com470e71d2011-07-07 08:21:25 +0000946// The automatic gain control (AGC) component brings the signal to an
947// appropriate range. This is done by applying a digital gain directly and, in
948// the analog mode, prescribing an analog gain to be applied at the audio HAL.
949//
950// Recommended to be enabled on the client-side.
951class GainControl {
952 public:
953 virtual int Enable(bool enable) = 0;
954 virtual bool is_enabled() const = 0;
955
956 // When an analog mode is set, this must be called prior to |ProcessStream()|
957 // to pass the current analog level from the audio HAL. Must be within the
958 // range provided to |set_analog_level_limits()|.
959 virtual int set_stream_analog_level(int level) = 0;
960
961 // When an analog mode is set, this should be called after |ProcessStream()|
962 // to obtain the recommended new analog level for the audio HAL. It is the
963 // users responsibility to apply this level.
964 virtual int stream_analog_level() = 0;
965
966 enum Mode {
967 // Adaptive mode intended for use if an analog volume control is available
968 // on the capture device. It will require the user to provide coupling
969 // between the OS mixer controls and AGC through the |stream_analog_level()|
970 // functions.
971 //
972 // It consists of an analog gain prescription for the audio device and a
973 // digital compression stage.
974 kAdaptiveAnalog,
975
976 // Adaptive mode intended for situations in which an analog volume control
977 // is unavailable. It operates in a similar fashion to the adaptive analog
978 // mode, but with scaling instead applied in the digital domain. As with
979 // the analog mode, it additionally uses a digital compression stage.
980 kAdaptiveDigital,
981
982 // Fixed mode which enables only the digital compression stage also used by
983 // the two adaptive modes.
984 //
985 // It is distinguished from the adaptive modes by considering only a
986 // short time-window of the input signal. It applies a fixed gain through
987 // most of the input level range, and compresses (gradually reduces gain
988 // with increasing level) the input signal at higher levels. This mode is
989 // preferred on embedded devices where the capture signal level is
990 // predictable, so that a known gain can be applied.
991 kFixedDigital
992 };
993
994 virtual int set_mode(Mode mode) = 0;
995 virtual Mode mode() const = 0;
996
997 // Sets the target peak |level| (or envelope) of the AGC in dBFs (decibels
998 // from digital full-scale). The convention is to use positive values. For
999 // instance, passing in a value of 3 corresponds to -3 dBFs, or a target
1000 // level 3 dB below full-scale. Limited to [0, 31].
1001 //
1002 // TODO(ajm): use a negative value here instead, if/when VoE will similarly
1003 // update its interface.
1004 virtual int set_target_level_dbfs(int level) = 0;
1005 virtual int target_level_dbfs() const = 0;
1006
1007 // Sets the maximum |gain| the digital compression stage may apply, in dB. A
1008 // higher number corresponds to greater compression, while a value of 0 will
1009 // leave the signal uncompressed. Limited to [0, 90].
1010 virtual int set_compression_gain_db(int gain) = 0;
1011 virtual int compression_gain_db() const = 0;
1012
1013 // When enabled, the compression stage will hard limit the signal to the
1014 // target level. Otherwise, the signal will be compressed but not limited
1015 // above the target level.
1016 virtual int enable_limiter(bool enable) = 0;
1017 virtual bool is_limiter_enabled() const = 0;
1018
1019 // Sets the |minimum| and |maximum| analog levels of the audio capture device.
1020 // Must be set if and only if an analog mode is used. Limited to [0, 65535].
1021 virtual int set_analog_level_limits(int minimum,
1022 int maximum) = 0;
1023 virtual int analog_level_minimum() const = 0;
1024 virtual int analog_level_maximum() const = 0;
1025
1026 // Returns true if the AGC has detected a saturation event (period where the
1027 // signal reaches digital full-scale) in the current frame and the analog
1028 // level cannot be reduced.
1029 //
1030 // This could be used as an indicator to reduce or disable analog mic gain at
1031 // the audio HAL.
1032 virtual bool stream_is_saturated() const = 0;
1033
1034 protected:
andrew@webrtc.orgd72b3d62012-11-15 21:46:06 +00001035 virtual ~GainControl() {}
niklase@google.com470e71d2011-07-07 08:21:25 +00001036};
peah8271d042016-11-22 07:24:52 -08001037// TODO(peah): Remove this interface.
niklase@google.com470e71d2011-07-07 08:21:25 +00001038// A filtering component which removes DC offset and low-frequency noise.
1039// Recommended to be enabled on the client-side.
1040class HighPassFilter {
1041 public:
1042 virtual int Enable(bool enable) = 0;
1043 virtual bool is_enabled() const = 0;
1044
andrew@webrtc.orgd72b3d62012-11-15 21:46:06 +00001045 virtual ~HighPassFilter() {}
niklase@google.com470e71d2011-07-07 08:21:25 +00001046};
1047
1048// An estimation component used to retrieve level metrics.
1049class LevelEstimator {
1050 public:
1051 virtual int Enable(bool enable) = 0;
1052 virtual bool is_enabled() const = 0;
1053
andrew@webrtc.org755b04a2011-11-15 16:57:56 +00001054 // Returns the root mean square (RMS) level in dBFs (decibels from digital
1055 // full-scale), or alternately dBov. It is computed over all primary stream
1056 // frames since the last call to RMS(). The returned value is positive but
1057 // should be interpreted as negative. It is constrained to [0, 127].
1058 //
andrew@webrtc.org382c0c22014-05-05 18:22:21 +00001059 // The computation follows: https://tools.ietf.org/html/rfc6465
andrew@webrtc.org755b04a2011-11-15 16:57:56 +00001060 // with the intent that it can provide the RTP audio level indication.
1061 //
1062 // Frames passed to ProcessStream() with an |_energy| of zero are considered
1063 // to have been muted. The RMS of the frame will be interpreted as -127.
1064 virtual int RMS() = 0;
niklase@google.com470e71d2011-07-07 08:21:25 +00001065
1066 protected:
andrew@webrtc.orgd72b3d62012-11-15 21:46:06 +00001067 virtual ~LevelEstimator() {}
niklase@google.com470e71d2011-07-07 08:21:25 +00001068};
1069
1070// The noise suppression (NS) component attempts to remove noise while
1071// retaining speech. Recommended to be enabled on the client-side.
1072//
1073// Recommended to be enabled on the client-side.
1074class NoiseSuppression {
1075 public:
1076 virtual int Enable(bool enable) = 0;
1077 virtual bool is_enabled() const = 0;
1078
1079 // Determines the aggressiveness of the suppression. Increasing the level
1080 // will reduce the noise level at the expense of a higher speech distortion.
1081 enum Level {
1082 kLow,
1083 kModerate,
1084 kHigh,
1085 kVeryHigh
1086 };
1087
1088 virtual int set_level(Level level) = 0;
1089 virtual Level level() const = 0;
1090
bjornv@webrtc.org08329f42012-07-12 21:00:43 +00001091 // Returns the internally computed prior speech probability of current frame
1092 // averaged over output channels. This is not supported in fixed point, for
1093 // which |kUnsupportedFunctionError| is returned.
1094 virtual float speech_probability() const = 0;
1095
Alejandro Luebsfa639f02016-02-09 11:24:32 -08001096 // Returns the noise estimate per frequency bin averaged over all channels.
1097 virtual std::vector<float> NoiseEstimate() = 0;
1098
niklase@google.com470e71d2011-07-07 08:21:25 +00001099 protected:
andrew@webrtc.orgd72b3d62012-11-15 21:46:06 +00001100 virtual ~NoiseSuppression() {}
niklase@google.com470e71d2011-07-07 08:21:25 +00001101};
1102
Alex Loiko5825aa62017-12-18 16:02:40 +01001103// Interface for a custom processing submodule.
1104class CustomProcessing {
Sam Zackrisson0beac582017-09-25 12:04:02 +02001105 public:
1106 // (Re-)Initializes the submodule.
1107 virtual void Initialize(int sample_rate_hz, int num_channels) = 0;
1108 // Processes the given capture or render signal.
1109 virtual void Process(AudioBuffer* audio) = 0;
1110 // Returns a string representation of the module state.
1111 virtual std::string ToString() const = 0;
1112
Alex Loiko5825aa62017-12-18 16:02:40 +01001113 virtual ~CustomProcessing() {}
Sam Zackrisson0beac582017-09-25 12:04:02 +02001114};
1115
niklase@google.com470e71d2011-07-07 08:21:25 +00001116// The voice activity detection (VAD) component analyzes the stream to
1117// determine if voice is present. A facility is also provided to pass in an
1118// external VAD decision.
andrew@webrtc.orged083d42011-09-19 15:28:51 +00001119//
1120// In addition to |stream_has_voice()| the VAD decision is provided through the
andrew@webrtc.org63a50982012-05-02 23:56:37 +00001121// |AudioFrame| passed to |ProcessStream()|. The |vad_activity_| member will be
andrew@webrtc.orged083d42011-09-19 15:28:51 +00001122// modified to reflect the current decision.
niklase@google.com470e71d2011-07-07 08:21:25 +00001123class VoiceDetection {
1124 public:
1125 virtual int Enable(bool enable) = 0;
1126 virtual bool is_enabled() const = 0;
1127
1128 // Returns true if voice is detected in the current frame. Should be called
1129 // after |ProcessStream()|.
1130 virtual bool stream_has_voice() const = 0;
1131
1132 // Some of the APM functionality requires a VAD decision. In the case that
1133 // a decision is externally available for the current frame, it can be passed
1134 // in here, before |ProcessStream()| is called.
1135 //
1136 // VoiceDetection does _not_ need to be enabled to use this. If it happens to
1137 // be enabled, detection will be skipped for any frame in which an external
1138 // VAD decision is provided.
1139 virtual int set_stream_has_voice(bool has_voice) = 0;
1140
1141 // Specifies the likelihood that a frame will be declared to contain voice.
1142 // A higher value makes it more likely that speech will not be clipped, at
1143 // the expense of more noise being detected as voice.
1144 enum Likelihood {
1145 kVeryLowLikelihood,
1146 kLowLikelihood,
1147 kModerateLikelihood,
1148 kHighLikelihood
1149 };
1150
1151 virtual int set_likelihood(Likelihood likelihood) = 0;
1152 virtual Likelihood likelihood() const = 0;
1153
1154 // Sets the |size| of the frames in ms on which the VAD will operate. Larger
1155 // frames will improve detection accuracy, but reduce the frequency of
1156 // updates.
1157 //
1158 // This does not impact the size of frames passed to |ProcessStream()|.
1159 virtual int set_frame_size_ms(int size) = 0;
1160 virtual int frame_size_ms() const = 0;
1161
1162 protected:
andrew@webrtc.orgd72b3d62012-11-15 21:46:06 +00001163 virtual ~VoiceDetection() {}
niklase@google.com470e71d2011-07-07 08:21:25 +00001164};
Gustaf Ullbergbd83b912017-10-18 12:32:42 +02001165
1166// Configuration struct for EchoCanceller3
1167struct EchoCanceller3Config {
1168 struct Delay {
1169 size_t default_delay = 5;
Per Åhgren38e2d952017-11-17 14:54:28 +01001170 size_t down_sampling_factor = 4;
1171 size_t num_filters = 4;
Per Åhgren8ba58612017-12-01 23:01:44 +01001172 size_t api_call_jitter_blocks = 26;
1173 size_t min_echo_path_delay_blocks = 5;
Gustaf Ullbergbd83b912017-10-18 12:32:42 +02001174 } delay;
1175
Per Åhgren09a718a2017-12-11 22:28:45 +01001176 struct Filter {
1177 size_t length_blocks = 12;
Per Åhgren019008b2017-12-18 11:38:39 +01001178 float shadow_rate = 0.1f;
1179 float leakage_converged = 0.005f;
1180 float leakage_diverged = 0.05f;
1181 float error_floor = 0.001f;
1182 float main_noise_gate = 20075344.f;
1183 float shadow_noise_gate = 20075344.f;
Per Åhgren09a718a2017-12-11 22:28:45 +01001184 } filter;
1185
Gustaf Ullbergbd83b912017-10-18 12:32:42 +02001186 struct Erle {
1187 float min = 1.f;
1188 float max_l = 8.f;
1189 float max_h = 1.5f;
1190 } erle;
1191
1192 struct EpStrength {
1193 float lf = 10.f;
1194 float mf = 10.f;
1195 float hf = 10.f;
1196 float default_len = 0.f;
1197 bool echo_can_saturate = true;
1198 bool bounded_erl = false;
1199 } ep_strength;
1200
1201 struct Mask {
1202 float m1 = 0.01f;
1203 float m2 = 0.0001f;
1204 float m3 = 0.01f;
1205 float m4 = 0.1f;
1206 float m5 = 0.3f;
1207 float m6 = 0.0001f;
1208 float m7 = 0.01f;
1209 float m8 = 0.0001f;
1210 float m9 = 0.1f;
1211 } gain_mask;
1212
1213 struct EchoAudibility {
1214 float low_render_limit = 4 * 64.f;
1215 float normal_render_limit = 64.f;
1216 } echo_audibility;
1217
1218 struct RenderLevels {
1219 float active_render_limit = 100.f;
1220 float poor_excitation_render_limit = 150.f;
1221 } render_levels;
1222
1223 struct GainUpdates {
1224 struct GainChanges {
1225 float max_inc;
1226 float max_dec;
1227 float rate_inc;
1228 float rate_dec;
1229 float min_inc;
1230 float min_dec;
1231 };
1232
1233 GainChanges low_noise = {3.f, 3.f, 1.5f, 1.5f, 1.5f, 1.5f};
1234 GainChanges normal = {2.f, 2.f, 1.5f, 1.5f, 1.2f, 1.2f};
Per Åhgren63b494d2017-12-06 11:32:38 +01001235 GainChanges saturation = {1.2f, 1.2f, 1.5f, 1.5f, 1.f, 1.f};
Gustaf Ullbergbd83b912017-10-18 12:32:42 +02001236 GainChanges nonlinear = {1.5f, 1.5f, 1.2f, 1.2f, 1.1f, 1.1f};
1237
1238 float floor_first_increase = 0.0001f;
1239 } gain_updates;
1240};
1241
1242class EchoCanceller3Factory : public EchoControlFactory {
1243 public:
1244 EchoCanceller3Factory();
1245 EchoCanceller3Factory(const EchoCanceller3Config& config);
1246 std::unique_ptr<EchoControl> Create(int sample_rate_hz) override;
1247
1248 private:
1249 EchoCanceller3Config config_;
1250};
niklase@google.com470e71d2011-07-07 08:21:25 +00001251} // namespace webrtc
1252
Mirko Bonadei92ea95e2017-09-15 06:47:31 +02001253#endif // MODULES_AUDIO_PROCESSING_INCLUDE_AUDIO_PROCESSING_H_