blob: d02b30d90b923a135ac7c49f61d1b2c74d6606ce [file] [log] [blame]
niklase@google.com470e71d2011-07-07 08:21:25 +00001/*
andrew@webrtc.org648af742012-02-08 01:57:29 +00002 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
niklase@google.com470e71d2011-07-07 08:21:25 +00003 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
andrew@webrtc.orgd72b3d62012-11-15 21:46:06 +000011#ifndef WEBRTC_MODULES_AUDIO_PROCESSING_INCLUDE_AUDIO_PROCESSING_H_
12#define WEBRTC_MODULES_AUDIO_PROCESSING_INCLUDE_AUDIO_PROCESSING_H_
niklase@google.com470e71d2011-07-07 08:21:25 +000013
andrew@webrtc.orgd72b3d62012-11-15 21:46:06 +000014#include <stddef.h> // size_t
henrikg@webrtc.org863b5362013-12-06 16:05:17 +000015#include <stdio.h> // FILE
ajm@google.com22e65152011-07-18 18:03:01 +000016
andrew@webrtc.org61e596f2013-07-25 18:28:29 +000017#include "webrtc/common.h"
andrew@webrtc.orgd72b3d62012-11-15 21:46:06 +000018#include "webrtc/modules/interface/module.h"
19#include "webrtc/typedefs.h"
niklase@google.com470e71d2011-07-07 08:21:25 +000020
bjornv@webrtc.org91d11b32013-03-05 16:53:09 +000021struct AecCore;
22
niklase@google.com470e71d2011-07-07 08:21:25 +000023namespace webrtc {
24
25class AudioFrame;
26class EchoCancellation;
27class EchoControlMobile;
28class GainControl;
29class HighPassFilter;
30class LevelEstimator;
31class NoiseSuppression;
32class VoiceDetection;
33
andrew@webrtc.org6b1e2192013-09-25 23:46:20 +000034// Use to enable the delay correction feature. This now engages an extended
35// filter mode in the AEC, along with robustness measures around the reported
36// system delays. It comes with a significant increase in AEC complexity, but is
37// much more robust to unreliable reported delays.
38//
39// Detailed changes to the algorithm:
40// - The filter length is changed from 48 to 128 ms. This comes with tuning of
41// several parameters: i) filter adaptation stepsize and error threshold;
42// ii) non-linear processing smoothing and overdrive.
43// - Option to ignore the reported delays on platforms which we deem
44// sufficiently unreliable. See WEBRTC_UNTRUSTED_DELAY in echo_cancellation.c.
45// - Faster startup times by removing the excessive "startup phase" processing
46// of reported delays.
47// - Much more conservative adjustments to the far-end read pointer. We smooth
48// the delay difference more heavily, and back off from the difference more.
49// Adjustments force a readaptation of the filter, so they should be avoided
50// except when really necessary.
51struct DelayCorrection {
52 DelayCorrection() : enabled(false) {}
andrew@webrtc.orgc7c7a532014-01-29 04:57:25 +000053 explicit DelayCorrection(bool enabled) : enabled(enabled) {}
54 bool enabled;
55};
andrew@webrtc.org6b1e2192013-09-25 23:46:20 +000056
andrew@webrtc.orgc7c7a532014-01-29 04:57:25 +000057// Must be provided through AudioProcessing::Create(Confg&). It will have no
58// impact if used with AudioProcessing::SetExtraOptions().
59struct ExperimentalAgc {
60 ExperimentalAgc() : enabled(true) {}
61 explicit ExperimentalAgc(bool enabled) : enabled(enabled) {}
andrew@webrtc.org6b1e2192013-09-25 23:46:20 +000062 bool enabled;
63};
64
niklase@google.com470e71d2011-07-07 08:21:25 +000065// The Audio Processing Module (APM) provides a collection of voice processing
66// components designed for real-time communications software.
67//
68// APM operates on two audio streams on a frame-by-frame basis. Frames of the
69// primary stream, on which all processing is applied, are passed to
70// |ProcessStream()|. Frames of the reverse direction stream, which are used for
71// analysis by some components, are passed to |AnalyzeReverseStream()|. On the
72// client-side, this will typically be the near-end (capture) and far-end
73// (render) streams, respectively. APM should be placed in the signal chain as
74// close to the audio hardware abstraction layer (HAL) as possible.
75//
76// On the server-side, the reverse stream will normally not be used, with
77// processing occurring on each incoming stream.
78//
79// Component interfaces follow a similar pattern and are accessed through
80// corresponding getters in APM. All components are disabled at create-time,
81// with default settings that are recommended for most situations. New settings
82// can be applied without enabling a component. Enabling a component triggers
83// memory allocation and initialization to allow it to start processing the
84// streams.
85//
86// Thread safety is provided with the following assumptions to reduce locking
87// overhead:
88// 1. The stream getters and setters are called from the same thread as
89// ProcessStream(). More precisely, stream functions are never called
90// concurrently with ProcessStream().
91// 2. Parameter getters are never called concurrently with the corresponding
92// setter.
93//
94// APM accepts only 16-bit linear PCM audio data in frames of 10 ms. Multiple
95// channels should be interleaved.
96//
97// Usage example, omitting error checking:
98// AudioProcessing* apm = AudioProcessing::Create(0);
niklase@google.com470e71d2011-07-07 08:21:25 +000099//
100// apm->high_pass_filter()->Enable(true);
101//
102// apm->echo_cancellation()->enable_drift_compensation(false);
103// apm->echo_cancellation()->Enable(true);
104//
105// apm->noise_reduction()->set_level(kHighSuppression);
106// apm->noise_reduction()->Enable(true);
107//
108// apm->gain_control()->set_analog_level_limits(0, 255);
109// apm->gain_control()->set_mode(kAdaptiveAnalog);
110// apm->gain_control()->Enable(true);
111//
112// apm->voice_detection()->Enable(true);
113//
114// // Start a voice call...
115//
116// // ... Render frame arrives bound for the audio HAL ...
117// apm->AnalyzeReverseStream(render_frame);
118//
119// // ... Capture frame arrives from the audio HAL ...
120// // Call required set_stream_ functions.
121// apm->set_stream_delay_ms(delay_ms);
122// apm->gain_control()->set_stream_analog_level(analog_level);
123//
124// apm->ProcessStream(capture_frame);
125//
126// // Call required stream_ functions.
127// analog_level = apm->gain_control()->stream_analog_level();
128// has_voice = apm->stream_has_voice();
129//
130// // Repeate render and capture processing for the duration of the call...
131// // Start a new call...
132// apm->Initialize();
133//
134// // Close the application...
andrew@webrtc.orgf3930e92013-09-18 22:37:32 +0000135// delete apm;
niklase@google.com470e71d2011-07-07 08:21:25 +0000136//
137class AudioProcessing : public Module {
138 public:
andrew@webrtc.org54744912014-02-05 06:30:29 +0000139 // Creates an APM instance. Use one instance for every primary audio stream
140 // requiring processing. On the client-side, this would typically be one
141 // instance for the near-end stream, and additional instances for each far-end
142 // stream which requires processing. On the server-side, this would typically
143 // be one instance for every incoming stream.
andrew@webrtc.orge84978f2014-01-25 02:09:06 +0000144 static AudioProcessing* Create();
andrew@webrtc.org54744912014-02-05 06:30:29 +0000145 // Allows passing in an optional configuration at create-time.
andrew@webrtc.orge84978f2014-01-25 02:09:06 +0000146 static AudioProcessing* Create(const Config& config);
147 // TODO(ajm): Deprecated; remove all calls to it.
niklase@google.com470e71d2011-07-07 08:21:25 +0000148 static AudioProcessing* Create(int id);
andrew@webrtc.orgd72b3d62012-11-15 21:46:06 +0000149 virtual ~AudioProcessing() {}
niklase@google.com470e71d2011-07-07 08:21:25 +0000150
niklase@google.com470e71d2011-07-07 08:21:25 +0000151 // Initializes internal states, while retaining all user settings. This
152 // should be called before beginning to process a new audio stream. However,
153 // it is not necessary to call before processing the first stream after
andrew@webrtc.org60730cf2014-01-07 17:45:09 +0000154 // creation. It is also not necessary to call if the audio parameters (sample
155 // rate and number of channels) have changed. Passing updated parameters
156 // directly to |ProcessStream()| and |AnalyzeReverseStream()| is permissible.
niklase@google.com470e71d2011-07-07 08:21:25 +0000157 virtual int Initialize() = 0;
158
andrew@webrtc.org61e596f2013-07-25 18:28:29 +0000159 // Pass down additional options which don't have explicit setters. This
160 // ensures the options are applied immediately.
161 virtual void SetExtraOptions(const Config& config) = 0;
162
aluebs@webrtc.org0b72f582013-11-19 15:17:51 +0000163 virtual int EnableExperimentalNs(bool enable) = 0;
164 virtual bool experimental_ns_enabled() const = 0;
165
andrew@webrtc.org60730cf2014-01-07 17:45:09 +0000166 // DEPRECATED: It is now possible to modify the sample rate directly in a call
167 // to |ProcessStream|.
niklase@google.com470e71d2011-07-07 08:21:25 +0000168 // Sets the sample |rate| in Hz for both the primary and reverse audio
169 // streams. 8000, 16000 or 32000 Hz are permitted.
170 virtual int set_sample_rate_hz(int rate) = 0;
171 virtual int sample_rate_hz() const = 0;
172
andrew@webrtc.org60730cf2014-01-07 17:45:09 +0000173 // DEPRECATED: It is now possible to modify the number of channels directly in
174 // a call to |ProcessStream|.
niklase@google.com470e71d2011-07-07 08:21:25 +0000175 // Sets the number of channels for the primary audio stream. Input frames must
176 // contain a number of channels given by |input_channels|, while output frames
177 // will be returned with number of channels given by |output_channels|.
178 virtual int set_num_channels(int input_channels, int output_channels) = 0;
179 virtual int num_input_channels() const = 0;
180 virtual int num_output_channels() const = 0;
181
andrew@webrtc.org60730cf2014-01-07 17:45:09 +0000182 // DEPRECATED: It is now possible to modify the number of channels directly in
183 // a call to |AnalyzeReverseStream|.
niklase@google.com470e71d2011-07-07 08:21:25 +0000184 // Sets the number of channels for the reverse audio stream. Input frames must
185 // contain a number of channels given by |channels|.
186 virtual int set_num_reverse_channels(int channels) = 0;
187 virtual int num_reverse_channels() const = 0;
188
andrew@webrtc.org17342e52014-02-12 22:28:31 +0000189 // Set to true when the output of AudioProcessing will be muted or in some
190 // other way not used. Ideally, the captured audio would still be processed,
191 // but some components may change behavior based on this information.
192 // Default false.
193 virtual void set_output_will_be_muted(bool muted) = 0;
194 virtual bool output_will_be_muted() const = 0;
195
niklase@google.com470e71d2011-07-07 08:21:25 +0000196 // Processes a 10 ms |frame| of the primary audio stream. On the client-side,
197 // this is the near-end (or captured) audio.
198 //
199 // If needed for enabled functionality, any function with the set_stream_ tag
200 // must be called prior to processing the current frame. Any getter function
201 // with the stream_ tag which is needed should be called after processing.
202 //
andrew@webrtc.org63a50982012-05-02 23:56:37 +0000203 // The |sample_rate_hz_|, |num_channels_|, and |samples_per_channel_|
andrew@webrtc.org60730cf2014-01-07 17:45:09 +0000204 // members of |frame| must be valid. If changed from the previous call to this
205 // method, it will trigger an initialization.
niklase@google.com470e71d2011-07-07 08:21:25 +0000206 virtual int ProcessStream(AudioFrame* frame) = 0;
207
208 // Analyzes a 10 ms |frame| of the reverse direction audio stream. The frame
209 // will not be modified. On the client-side, this is the far-end (or to be
210 // rendered) audio.
211 //
212 // It is only necessary to provide this if echo processing is enabled, as the
213 // reverse stream forms the echo reference signal. It is recommended, but not
214 // necessary, to provide if gain control is enabled. On the server-side this
215 // typically will not be used. If you're not sure what to pass in here,
216 // chances are you don't need to use it.
217 //
andrew@webrtc.org63a50982012-05-02 23:56:37 +0000218 // The |sample_rate_hz_|, |num_channels_|, and |samples_per_channel_|
andrew@webrtc.org60730cf2014-01-07 17:45:09 +0000219 // members of |frame| must be valid. |sample_rate_hz_| must correspond to
220 // |sample_rate_hz()|
niklase@google.com470e71d2011-07-07 08:21:25 +0000221 //
222 // TODO(ajm): add const to input; requires an implementation fix.
223 virtual int AnalyzeReverseStream(AudioFrame* frame) = 0;
224
225 // This must be called if and only if echo processing is enabled.
226 //
227 // Sets the |delay| in ms between AnalyzeReverseStream() receiving a far-end
228 // frame and ProcessStream() receiving a near-end frame containing the
229 // corresponding echo. On the client-side this can be expressed as
230 // delay = (t_render - t_analyze) + (t_process - t_capture)
231 // where,
232 // - t_analyze is the time a frame is passed to AnalyzeReverseStream() and
233 // t_render is the time the first sample of the same frame is rendered by
234 // the audio hardware.
235 // - t_capture is the time the first sample of a frame is captured by the
236 // audio hardware and t_pull is the time the same frame is passed to
237 // ProcessStream().
238 virtual int set_stream_delay_ms(int delay) = 0;
239 virtual int stream_delay_ms() const = 0;
240
andrew@webrtc.org75dd2882014-02-11 20:52:30 +0000241 // Call to signal that a key press occurred (true) or did not occur (false)
242 // with this chunk of audio.
243 virtual void set_stream_key_pressed(bool key_pressed) = 0;
244 virtual bool stream_key_pressed() const = 0;
245
andrew@webrtc.org6f9f8172012-03-06 19:03:39 +0000246 // Sets a delay |offset| in ms to add to the values passed in through
247 // set_stream_delay_ms(). May be positive or negative.
248 //
249 // Note that this could cause an otherwise valid value passed to
250 // set_stream_delay_ms() to return an error.
251 virtual void set_delay_offset_ms(int offset) = 0;
252 virtual int delay_offset_ms() const = 0;
253
niklase@google.com470e71d2011-07-07 08:21:25 +0000254 // Starts recording debugging information to a file specified by |filename|,
255 // a NULL-terminated string. If there is an ongoing recording, the old file
256 // will be closed, and recording will continue in the newly specified file.
257 // An already existing file will be overwritten without warning.
andrew@webrtc.org5ae19de2011-12-13 22:59:33 +0000258 static const size_t kMaxFilenameSize = 1024;
niklase@google.com470e71d2011-07-07 08:21:25 +0000259 virtual int StartDebugRecording(const char filename[kMaxFilenameSize]) = 0;
260
henrikg@webrtc.org863b5362013-12-06 16:05:17 +0000261 // Same as above but uses an existing file handle. Takes ownership
262 // of |handle| and closes it at StopDebugRecording().
263 virtual int StartDebugRecording(FILE* handle) = 0;
264
niklase@google.com470e71d2011-07-07 08:21:25 +0000265 // Stops recording debugging information, and closes the file. Recording
266 // cannot be resumed in the same file (without overwriting it).
267 virtual int StopDebugRecording() = 0;
268
269 // These provide access to the component interfaces and should never return
270 // NULL. The pointers will be valid for the lifetime of the APM instance.
271 // The memory for these objects is entirely managed internally.
272 virtual EchoCancellation* echo_cancellation() const = 0;
273 virtual EchoControlMobile* echo_control_mobile() const = 0;
274 virtual GainControl* gain_control() const = 0;
275 virtual HighPassFilter* high_pass_filter() const = 0;
276 virtual LevelEstimator* level_estimator() const = 0;
277 virtual NoiseSuppression* noise_suppression() const = 0;
278 virtual VoiceDetection* voice_detection() const = 0;
279
280 struct Statistic {
281 int instant; // Instantaneous value.
282 int average; // Long-term average.
283 int maximum; // Long-term maximum.
284 int minimum; // Long-term minimum.
285 };
286
andrew@webrtc.org648af742012-02-08 01:57:29 +0000287 enum Error {
288 // Fatal errors.
niklase@google.com470e71d2011-07-07 08:21:25 +0000289 kNoError = 0,
290 kUnspecifiedError = -1,
291 kCreationFailedError = -2,
292 kUnsupportedComponentError = -3,
293 kUnsupportedFunctionError = -4,
294 kNullPointerError = -5,
295 kBadParameterError = -6,
296 kBadSampleRateError = -7,
297 kBadDataLengthError = -8,
298 kBadNumberChannelsError = -9,
299 kFileError = -10,
300 kStreamParameterNotSetError = -11,
andrew@webrtc.org648af742012-02-08 01:57:29 +0000301 kNotEnabledError = -12,
niklase@google.com470e71d2011-07-07 08:21:25 +0000302
andrew@webrtc.org648af742012-02-08 01:57:29 +0000303 // Warnings are non-fatal.
niklase@google.com470e71d2011-07-07 08:21:25 +0000304 // This results when a set_stream_ parameter is out of range. Processing
305 // will continue, but the parameter may have been truncated.
andrew@webrtc.org648af742012-02-08 01:57:29 +0000306 kBadStreamParameterWarning = -13
niklase@google.com470e71d2011-07-07 08:21:25 +0000307 };
308
309 // Inherited from Module.
pbos@webrtc.org91620802013-08-02 11:44:11 +0000310 virtual int32_t TimeUntilNextProcess() OVERRIDE;
311 virtual int32_t Process() OVERRIDE;
niklase@google.com470e71d2011-07-07 08:21:25 +0000312};
313
314// The acoustic echo cancellation (AEC) component provides better performance
315// than AECM but also requires more processing power and is dependent on delay
316// stability and reporting accuracy. As such it is well-suited and recommended
317// for PC and IP phone applications.
318//
319// Not recommended to be enabled on the server-side.
320class EchoCancellation {
321 public:
322 // EchoCancellation and EchoControlMobile may not be enabled simultaneously.
323 // Enabling one will disable the other.
324 virtual int Enable(bool enable) = 0;
325 virtual bool is_enabled() const = 0;
326
327 // Differences in clock speed on the primary and reverse streams can impact
328 // the AEC performance. On the client-side, this could be seen when different
329 // render and capture devices are used, particularly with webcams.
330 //
331 // This enables a compensation mechanism, and requires that
332 // |set_device_sample_rate_hz()| and |set_stream_drift_samples()| be called.
333 virtual int enable_drift_compensation(bool enable) = 0;
334 virtual bool is_drift_compensation_enabled() const = 0;
335
336 // Provides the sampling rate of the audio devices. It is assumed the render
337 // and capture devices use the same nominal sample rate. Required if and only
338 // if drift compensation is enabled.
339 virtual int set_device_sample_rate_hz(int rate) = 0;
340 virtual int device_sample_rate_hz() const = 0;
341
342 // Sets the difference between the number of samples rendered and captured by
343 // the audio devices since the last call to |ProcessStream()|. Must be called
andrew@webrtc.org6be1e932013-03-01 18:47:28 +0000344 // if drift compensation is enabled, prior to |ProcessStream()|.
345 virtual void set_stream_drift_samples(int drift) = 0;
niklase@google.com470e71d2011-07-07 08:21:25 +0000346 virtual int stream_drift_samples() const = 0;
347
348 enum SuppressionLevel {
349 kLowSuppression,
350 kModerateSuppression,
351 kHighSuppression
352 };
353
354 // Sets the aggressiveness of the suppressor. A higher level trades off
355 // double-talk performance for increased echo suppression.
356 virtual int set_suppression_level(SuppressionLevel level) = 0;
357 virtual SuppressionLevel suppression_level() const = 0;
358
359 // Returns false if the current frame almost certainly contains no echo
360 // and true if it _might_ contain echo.
361 virtual bool stream_has_echo() const = 0;
362
363 // Enables the computation of various echo metrics. These are obtained
364 // through |GetMetrics()|.
365 virtual int enable_metrics(bool enable) = 0;
366 virtual bool are_metrics_enabled() const = 0;
367
368 // Each statistic is reported in dB.
369 // P_far: Far-end (render) signal power.
370 // P_echo: Near-end (capture) echo signal power.
371 // P_out: Signal power at the output of the AEC.
372 // P_a: Internal signal power at the point before the AEC's non-linear
373 // processor.
374 struct Metrics {
375 // RERL = ERL + ERLE
376 AudioProcessing::Statistic residual_echo_return_loss;
377
378 // ERL = 10log_10(P_far / P_echo)
379 AudioProcessing::Statistic echo_return_loss;
380
381 // ERLE = 10log_10(P_echo / P_out)
382 AudioProcessing::Statistic echo_return_loss_enhancement;
383
384 // (Pre non-linear processing suppression) A_NLP = 10log_10(P_echo / P_a)
385 AudioProcessing::Statistic a_nlp;
386 };
387
388 // TODO(ajm): discuss the metrics update period.
389 virtual int GetMetrics(Metrics* metrics) = 0;
390
bjornv@google.com1ba3dbe2011-10-03 08:18:10 +0000391 // Enables computation and logging of delay values. Statistics are obtained
392 // through |GetDelayMetrics()|.
393 virtual int enable_delay_logging(bool enable) = 0;
394 virtual bool is_delay_logging_enabled() const = 0;
395
396 // The delay metrics consists of the delay |median| and the delay standard
397 // deviation |std|. The values are averaged over the time period since the
398 // last call to |GetDelayMetrics()|.
399 virtual int GetDelayMetrics(int* median, int* std) = 0;
400
bjornv@webrtc.org91d11b32013-03-05 16:53:09 +0000401 // Returns a pointer to the low level AEC component. In case of multiple
402 // channels, the pointer to the first one is returned. A NULL pointer is
403 // returned when the AEC component is disabled or has not been initialized
404 // successfully.
405 virtual struct AecCore* aec_core() const = 0;
406
niklase@google.com470e71d2011-07-07 08:21:25 +0000407 protected:
andrew@webrtc.orgd72b3d62012-11-15 21:46:06 +0000408 virtual ~EchoCancellation() {}
niklase@google.com470e71d2011-07-07 08:21:25 +0000409};
410
411// The acoustic echo control for mobile (AECM) component is a low complexity
412// robust option intended for use on mobile devices.
413//
414// Not recommended to be enabled on the server-side.
415class EchoControlMobile {
416 public:
417 // EchoCancellation and EchoControlMobile may not be enabled simultaneously.
418 // Enabling one will disable the other.
419 virtual int Enable(bool enable) = 0;
420 virtual bool is_enabled() const = 0;
421
422 // Recommended settings for particular audio routes. In general, the louder
423 // the echo is expected to be, the higher this value should be set. The
424 // preferred setting may vary from device to device.
425 enum RoutingMode {
426 kQuietEarpieceOrHeadset,
427 kEarpiece,
428 kLoudEarpiece,
429 kSpeakerphone,
430 kLoudSpeakerphone
431 };
432
433 // Sets echo control appropriate for the audio routing |mode| on the device.
434 // It can and should be updated during a call if the audio routing changes.
435 virtual int set_routing_mode(RoutingMode mode) = 0;
436 virtual RoutingMode routing_mode() const = 0;
437
438 // Comfort noise replaces suppressed background noise to maintain a
439 // consistent signal level.
440 virtual int enable_comfort_noise(bool enable) = 0;
441 virtual bool is_comfort_noise_enabled() const = 0;
442
bjornv@google.comc4b939c2011-07-13 08:09:56 +0000443 // A typical use case is to initialize the component with an echo path from a
ajm@google.com22e65152011-07-18 18:03:01 +0000444 // previous call. The echo path is retrieved using |GetEchoPath()|, typically
445 // at the end of a call. The data can then be stored for later use as an
446 // initializer before the next call, using |SetEchoPath()|.
447 //
bjornv@google.comc4b939c2011-07-13 08:09:56 +0000448 // Controlling the echo path this way requires the data |size_bytes| to match
449 // the internal echo path size. This size can be acquired using
450 // |echo_path_size_bytes()|. |SetEchoPath()| causes an entire reset, worth
ajm@google.com22e65152011-07-18 18:03:01 +0000451 // noting if it is to be called during an ongoing call.
452 //
453 // It is possible that version incompatibilities may result in a stored echo
454 // path of the incorrect size. In this case, the stored path should be
455 // discarded.
456 virtual int SetEchoPath(const void* echo_path, size_t size_bytes) = 0;
457 virtual int GetEchoPath(void* echo_path, size_t size_bytes) const = 0;
458
459 // The returned path size is guaranteed not to change for the lifetime of
460 // the application.
461 static size_t echo_path_size_bytes();
bjornv@google.comc4b939c2011-07-13 08:09:56 +0000462
niklase@google.com470e71d2011-07-07 08:21:25 +0000463 protected:
andrew@webrtc.orgd72b3d62012-11-15 21:46:06 +0000464 virtual ~EchoControlMobile() {}
niklase@google.com470e71d2011-07-07 08:21:25 +0000465};
466
467// The automatic gain control (AGC) component brings the signal to an
468// appropriate range. This is done by applying a digital gain directly and, in
469// the analog mode, prescribing an analog gain to be applied at the audio HAL.
470//
471// Recommended to be enabled on the client-side.
472class GainControl {
473 public:
474 virtual int Enable(bool enable) = 0;
475 virtual bool is_enabled() const = 0;
476
477 // When an analog mode is set, this must be called prior to |ProcessStream()|
478 // to pass the current analog level from the audio HAL. Must be within the
479 // range provided to |set_analog_level_limits()|.
480 virtual int set_stream_analog_level(int level) = 0;
481
482 // When an analog mode is set, this should be called after |ProcessStream()|
483 // to obtain the recommended new analog level for the audio HAL. It is the
484 // users responsibility to apply this level.
485 virtual int stream_analog_level() = 0;
486
487 enum Mode {
488 // Adaptive mode intended for use if an analog volume control is available
489 // on the capture device. It will require the user to provide coupling
490 // between the OS mixer controls and AGC through the |stream_analog_level()|
491 // functions.
492 //
493 // It consists of an analog gain prescription for the audio device and a
494 // digital compression stage.
495 kAdaptiveAnalog,
496
497 // Adaptive mode intended for situations in which an analog volume control
498 // is unavailable. It operates in a similar fashion to the adaptive analog
499 // mode, but with scaling instead applied in the digital domain. As with
500 // the analog mode, it additionally uses a digital compression stage.
501 kAdaptiveDigital,
502
503 // Fixed mode which enables only the digital compression stage also used by
504 // the two adaptive modes.
505 //
506 // It is distinguished from the adaptive modes by considering only a
507 // short time-window of the input signal. It applies a fixed gain through
508 // most of the input level range, and compresses (gradually reduces gain
509 // with increasing level) the input signal at higher levels. This mode is
510 // preferred on embedded devices where the capture signal level is
511 // predictable, so that a known gain can be applied.
512 kFixedDigital
513 };
514
515 virtual int set_mode(Mode mode) = 0;
516 virtual Mode mode() const = 0;
517
518 // Sets the target peak |level| (or envelope) of the AGC in dBFs (decibels
519 // from digital full-scale). The convention is to use positive values. For
520 // instance, passing in a value of 3 corresponds to -3 dBFs, or a target
521 // level 3 dB below full-scale. Limited to [0, 31].
522 //
523 // TODO(ajm): use a negative value here instead, if/when VoE will similarly
524 // update its interface.
525 virtual int set_target_level_dbfs(int level) = 0;
526 virtual int target_level_dbfs() const = 0;
527
528 // Sets the maximum |gain| the digital compression stage may apply, in dB. A
529 // higher number corresponds to greater compression, while a value of 0 will
530 // leave the signal uncompressed. Limited to [0, 90].
531 virtual int set_compression_gain_db(int gain) = 0;
532 virtual int compression_gain_db() const = 0;
533
534 // When enabled, the compression stage will hard limit the signal to the
535 // target level. Otherwise, the signal will be compressed but not limited
536 // above the target level.
537 virtual int enable_limiter(bool enable) = 0;
538 virtual bool is_limiter_enabled() const = 0;
539
540 // Sets the |minimum| and |maximum| analog levels of the audio capture device.
541 // Must be set if and only if an analog mode is used. Limited to [0, 65535].
542 virtual int set_analog_level_limits(int minimum,
543 int maximum) = 0;
544 virtual int analog_level_minimum() const = 0;
545 virtual int analog_level_maximum() const = 0;
546
547 // Returns true if the AGC has detected a saturation event (period where the
548 // signal reaches digital full-scale) in the current frame and the analog
549 // level cannot be reduced.
550 //
551 // This could be used as an indicator to reduce or disable analog mic gain at
552 // the audio HAL.
553 virtual bool stream_is_saturated() const = 0;
554
555 protected:
andrew@webrtc.orgd72b3d62012-11-15 21:46:06 +0000556 virtual ~GainControl() {}
niklase@google.com470e71d2011-07-07 08:21:25 +0000557};
558
559// A filtering component which removes DC offset and low-frequency noise.
560// Recommended to be enabled on the client-side.
561class HighPassFilter {
562 public:
563 virtual int Enable(bool enable) = 0;
564 virtual bool is_enabled() const = 0;
565
566 protected:
andrew@webrtc.orgd72b3d62012-11-15 21:46:06 +0000567 virtual ~HighPassFilter() {}
niklase@google.com470e71d2011-07-07 08:21:25 +0000568};
569
570// An estimation component used to retrieve level metrics.
571class LevelEstimator {
572 public:
573 virtual int Enable(bool enable) = 0;
574 virtual bool is_enabled() const = 0;
575
andrew@webrtc.org755b04a2011-11-15 16:57:56 +0000576 // Returns the root mean square (RMS) level in dBFs (decibels from digital
577 // full-scale), or alternately dBov. It is computed over all primary stream
578 // frames since the last call to RMS(). The returned value is positive but
579 // should be interpreted as negative. It is constrained to [0, 127].
580 //
581 // The computation follows:
582 // http://tools.ietf.org/html/draft-ietf-avtext-client-to-mixer-audio-level-05
583 // with the intent that it can provide the RTP audio level indication.
584 //
585 // Frames passed to ProcessStream() with an |_energy| of zero are considered
586 // to have been muted. The RMS of the frame will be interpreted as -127.
587 virtual int RMS() = 0;
niklase@google.com470e71d2011-07-07 08:21:25 +0000588
589 protected:
andrew@webrtc.orgd72b3d62012-11-15 21:46:06 +0000590 virtual ~LevelEstimator() {}
niklase@google.com470e71d2011-07-07 08:21:25 +0000591};
592
593// The noise suppression (NS) component attempts to remove noise while
594// retaining speech. Recommended to be enabled on the client-side.
595//
596// Recommended to be enabled on the client-side.
597class NoiseSuppression {
598 public:
599 virtual int Enable(bool enable) = 0;
600 virtual bool is_enabled() const = 0;
601
602 // Determines the aggressiveness of the suppression. Increasing the level
603 // will reduce the noise level at the expense of a higher speech distortion.
604 enum Level {
605 kLow,
606 kModerate,
607 kHigh,
608 kVeryHigh
609 };
610
611 virtual int set_level(Level level) = 0;
612 virtual Level level() const = 0;
613
bjornv@webrtc.org08329f42012-07-12 21:00:43 +0000614 // Returns the internally computed prior speech probability of current frame
615 // averaged over output channels. This is not supported in fixed point, for
616 // which |kUnsupportedFunctionError| is returned.
617 virtual float speech_probability() const = 0;
618
niklase@google.com470e71d2011-07-07 08:21:25 +0000619 protected:
andrew@webrtc.orgd72b3d62012-11-15 21:46:06 +0000620 virtual ~NoiseSuppression() {}
niklase@google.com470e71d2011-07-07 08:21:25 +0000621};
622
623// The voice activity detection (VAD) component analyzes the stream to
624// determine if voice is present. A facility is also provided to pass in an
625// external VAD decision.
andrew@webrtc.orged083d42011-09-19 15:28:51 +0000626//
627// In addition to |stream_has_voice()| the VAD decision is provided through the
andrew@webrtc.org63a50982012-05-02 23:56:37 +0000628// |AudioFrame| passed to |ProcessStream()|. The |vad_activity_| member will be
andrew@webrtc.orged083d42011-09-19 15:28:51 +0000629// modified to reflect the current decision.
niklase@google.com470e71d2011-07-07 08:21:25 +0000630class VoiceDetection {
631 public:
632 virtual int Enable(bool enable) = 0;
633 virtual bool is_enabled() const = 0;
634
635 // Returns true if voice is detected in the current frame. Should be called
636 // after |ProcessStream()|.
637 virtual bool stream_has_voice() const = 0;
638
639 // Some of the APM functionality requires a VAD decision. In the case that
640 // a decision is externally available for the current frame, it can be passed
641 // in here, before |ProcessStream()| is called.
642 //
643 // VoiceDetection does _not_ need to be enabled to use this. If it happens to
644 // be enabled, detection will be skipped for any frame in which an external
645 // VAD decision is provided.
646 virtual int set_stream_has_voice(bool has_voice) = 0;
647
648 // Specifies the likelihood that a frame will be declared to contain voice.
649 // A higher value makes it more likely that speech will not be clipped, at
650 // the expense of more noise being detected as voice.
651 enum Likelihood {
652 kVeryLowLikelihood,
653 kLowLikelihood,
654 kModerateLikelihood,
655 kHighLikelihood
656 };
657
658 virtual int set_likelihood(Likelihood likelihood) = 0;
659 virtual Likelihood likelihood() const = 0;
660
661 // Sets the |size| of the frames in ms on which the VAD will operate. Larger
662 // frames will improve detection accuracy, but reduce the frequency of
663 // updates.
664 //
665 // This does not impact the size of frames passed to |ProcessStream()|.
666 virtual int set_frame_size_ms(int size) = 0;
667 virtual int frame_size_ms() const = 0;
668
669 protected:
andrew@webrtc.orgd72b3d62012-11-15 21:46:06 +0000670 virtual ~VoiceDetection() {}
niklase@google.com470e71d2011-07-07 08:21:25 +0000671};
672} // namespace webrtc
673
andrew@webrtc.orgd72b3d62012-11-15 21:46:06 +0000674#endif // WEBRTC_MODULES_AUDIO_PROCESSING_INCLUDE_AUDIO_PROCESSING_H_