niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 1 | /* |
andrew@webrtc.org | 648af74 | 2012-02-08 01:57:29 +0000 | [diff] [blame] | 2 | * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 3 | * |
| 4 | * Use of this source code is governed by a BSD-style license |
| 5 | * that can be found in the LICENSE file in the root of the source |
| 6 | * tree. An additional intellectual property rights grant can be found |
| 7 | * in the file PATENTS. All contributing project authors may |
| 8 | * be found in the AUTHORS file in the root of the source tree. |
| 9 | */ |
| 10 | |
andrew@webrtc.org | d72b3d6 | 2012-11-15 21:46:06 +0000 | [diff] [blame] | 11 | #ifndef WEBRTC_MODULES_AUDIO_PROCESSING_INCLUDE_AUDIO_PROCESSING_H_ |
| 12 | #define WEBRTC_MODULES_AUDIO_PROCESSING_INCLUDE_AUDIO_PROCESSING_H_ |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 13 | |
andrew@webrtc.org | d72b3d6 | 2012-11-15 21:46:06 +0000 | [diff] [blame] | 14 | #include <stddef.h> // size_t |
henrikg@webrtc.org | 863b536 | 2013-12-06 16:05:17 +0000 | [diff] [blame] | 15 | #include <stdio.h> // FILE |
ajm@google.com | 22e6515 | 2011-07-18 18:03:01 +0000 | [diff] [blame] | 16 | |
andrew@webrtc.org | 61e596f | 2013-07-25 18:28:29 +0000 | [diff] [blame] | 17 | #include "webrtc/common.h" |
andrew@webrtc.org | d72b3d6 | 2012-11-15 21:46:06 +0000 | [diff] [blame] | 18 | #include "webrtc/modules/interface/module.h" |
| 19 | #include "webrtc/typedefs.h" |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 20 | |
bjornv@webrtc.org | 91d11b3 | 2013-03-05 16:53:09 +0000 | [diff] [blame] | 21 | struct AecCore; |
| 22 | |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 23 | namespace webrtc { |
| 24 | |
| 25 | class AudioFrame; |
| 26 | class EchoCancellation; |
| 27 | class EchoControlMobile; |
| 28 | class GainControl; |
| 29 | class HighPassFilter; |
| 30 | class LevelEstimator; |
| 31 | class NoiseSuppression; |
| 32 | class VoiceDetection; |
| 33 | |
andrew@webrtc.org | 6b1e219 | 2013-09-25 23:46:20 +0000 | [diff] [blame] | 34 | // Use to enable the delay correction feature. This now engages an extended |
| 35 | // filter mode in the AEC, along with robustness measures around the reported |
| 36 | // system delays. It comes with a significant increase in AEC complexity, but is |
| 37 | // much more robust to unreliable reported delays. |
| 38 | // |
| 39 | // Detailed changes to the algorithm: |
| 40 | // - The filter length is changed from 48 to 128 ms. This comes with tuning of |
| 41 | // several parameters: i) filter adaptation stepsize and error threshold; |
| 42 | // ii) non-linear processing smoothing and overdrive. |
| 43 | // - Option to ignore the reported delays on platforms which we deem |
| 44 | // sufficiently unreliable. See WEBRTC_UNTRUSTED_DELAY in echo_cancellation.c. |
| 45 | // - Faster startup times by removing the excessive "startup phase" processing |
| 46 | // of reported delays. |
| 47 | // - Much more conservative adjustments to the far-end read pointer. We smooth |
| 48 | // the delay difference more heavily, and back off from the difference more. |
| 49 | // Adjustments force a readaptation of the filter, so they should be avoided |
| 50 | // except when really necessary. |
| 51 | struct DelayCorrection { |
| 52 | DelayCorrection() : enabled(false) {} |
andrew@webrtc.org | c7c7a53 | 2014-01-29 04:57:25 +0000 | [diff] [blame^] | 53 | explicit DelayCorrection(bool enabled) : enabled(enabled) {} |
| 54 | bool enabled; |
| 55 | }; |
andrew@webrtc.org | 6b1e219 | 2013-09-25 23:46:20 +0000 | [diff] [blame] | 56 | |
andrew@webrtc.org | c7c7a53 | 2014-01-29 04:57:25 +0000 | [diff] [blame^] | 57 | // Must be provided through AudioProcessing::Create(Confg&). It will have no |
| 58 | // impact if used with AudioProcessing::SetExtraOptions(). |
| 59 | struct ExperimentalAgc { |
| 60 | ExperimentalAgc() : enabled(true) {} |
| 61 | explicit ExperimentalAgc(bool enabled) : enabled(enabled) {} |
andrew@webrtc.org | 6b1e219 | 2013-09-25 23:46:20 +0000 | [diff] [blame] | 62 | bool enabled; |
| 63 | }; |
| 64 | |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 65 | // The Audio Processing Module (APM) provides a collection of voice processing |
| 66 | // components designed for real-time communications software. |
| 67 | // |
| 68 | // APM operates on two audio streams on a frame-by-frame basis. Frames of the |
| 69 | // primary stream, on which all processing is applied, are passed to |
| 70 | // |ProcessStream()|. Frames of the reverse direction stream, which are used for |
| 71 | // analysis by some components, are passed to |AnalyzeReverseStream()|. On the |
| 72 | // client-side, this will typically be the near-end (capture) and far-end |
| 73 | // (render) streams, respectively. APM should be placed in the signal chain as |
| 74 | // close to the audio hardware abstraction layer (HAL) as possible. |
| 75 | // |
| 76 | // On the server-side, the reverse stream will normally not be used, with |
| 77 | // processing occurring on each incoming stream. |
| 78 | // |
| 79 | // Component interfaces follow a similar pattern and are accessed through |
| 80 | // corresponding getters in APM. All components are disabled at create-time, |
| 81 | // with default settings that are recommended for most situations. New settings |
| 82 | // can be applied without enabling a component. Enabling a component triggers |
| 83 | // memory allocation and initialization to allow it to start processing the |
| 84 | // streams. |
| 85 | // |
| 86 | // Thread safety is provided with the following assumptions to reduce locking |
| 87 | // overhead: |
| 88 | // 1. The stream getters and setters are called from the same thread as |
| 89 | // ProcessStream(). More precisely, stream functions are never called |
| 90 | // concurrently with ProcessStream(). |
| 91 | // 2. Parameter getters are never called concurrently with the corresponding |
| 92 | // setter. |
| 93 | // |
| 94 | // APM accepts only 16-bit linear PCM audio data in frames of 10 ms. Multiple |
| 95 | // channels should be interleaved. |
| 96 | // |
| 97 | // Usage example, omitting error checking: |
| 98 | // AudioProcessing* apm = AudioProcessing::Create(0); |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 99 | // |
| 100 | // apm->high_pass_filter()->Enable(true); |
| 101 | // |
| 102 | // apm->echo_cancellation()->enable_drift_compensation(false); |
| 103 | // apm->echo_cancellation()->Enable(true); |
| 104 | // |
| 105 | // apm->noise_reduction()->set_level(kHighSuppression); |
| 106 | // apm->noise_reduction()->Enable(true); |
| 107 | // |
| 108 | // apm->gain_control()->set_analog_level_limits(0, 255); |
| 109 | // apm->gain_control()->set_mode(kAdaptiveAnalog); |
| 110 | // apm->gain_control()->Enable(true); |
| 111 | // |
| 112 | // apm->voice_detection()->Enable(true); |
| 113 | // |
| 114 | // // Start a voice call... |
| 115 | // |
| 116 | // // ... Render frame arrives bound for the audio HAL ... |
| 117 | // apm->AnalyzeReverseStream(render_frame); |
| 118 | // |
| 119 | // // ... Capture frame arrives from the audio HAL ... |
| 120 | // // Call required set_stream_ functions. |
| 121 | // apm->set_stream_delay_ms(delay_ms); |
| 122 | // apm->gain_control()->set_stream_analog_level(analog_level); |
| 123 | // |
| 124 | // apm->ProcessStream(capture_frame); |
| 125 | // |
| 126 | // // Call required stream_ functions. |
| 127 | // analog_level = apm->gain_control()->stream_analog_level(); |
| 128 | // has_voice = apm->stream_has_voice(); |
| 129 | // |
| 130 | // // Repeate render and capture processing for the duration of the call... |
| 131 | // // Start a new call... |
| 132 | // apm->Initialize(); |
| 133 | // |
| 134 | // // Close the application... |
andrew@webrtc.org | f3930e9 | 2013-09-18 22:37:32 +0000 | [diff] [blame] | 135 | // delete apm; |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 136 | // |
| 137 | class AudioProcessing : public Module { |
| 138 | public: |
| 139 | // Creates a APM instance, with identifier |id|. Use one instance for every |
| 140 | // primary audio stream requiring processing. On the client-side, this would |
| 141 | // typically be one instance for the near-end stream, and additional instances |
| 142 | // for each far-end stream which requires processing. On the server-side, |
| 143 | // this would typically be one instance for every incoming stream. |
andrew@webrtc.org | e84978f | 2014-01-25 02:09:06 +0000 | [diff] [blame] | 144 | static AudioProcessing* Create(); |
| 145 | static AudioProcessing* Create(const Config& config); |
| 146 | // TODO(ajm): Deprecated; remove all calls to it. |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 147 | static AudioProcessing* Create(int id); |
andrew@webrtc.org | d72b3d6 | 2012-11-15 21:46:06 +0000 | [diff] [blame] | 148 | virtual ~AudioProcessing() {} |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 149 | |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 150 | // Initializes internal states, while retaining all user settings. This |
| 151 | // should be called before beginning to process a new audio stream. However, |
| 152 | // it is not necessary to call before processing the first stream after |
andrew@webrtc.org | 60730cf | 2014-01-07 17:45:09 +0000 | [diff] [blame] | 153 | // creation. It is also not necessary to call if the audio parameters (sample |
| 154 | // rate and number of channels) have changed. Passing updated parameters |
| 155 | // directly to |ProcessStream()| and |AnalyzeReverseStream()| is permissible. |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 156 | virtual int Initialize() = 0; |
| 157 | |
andrew@webrtc.org | 61e596f | 2013-07-25 18:28:29 +0000 | [diff] [blame] | 158 | // Pass down additional options which don't have explicit setters. This |
| 159 | // ensures the options are applied immediately. |
| 160 | virtual void SetExtraOptions(const Config& config) = 0; |
| 161 | |
aluebs@webrtc.org | 0b72f58 | 2013-11-19 15:17:51 +0000 | [diff] [blame] | 162 | virtual int EnableExperimentalNs(bool enable) = 0; |
| 163 | virtual bool experimental_ns_enabled() const = 0; |
| 164 | |
andrew@webrtc.org | 60730cf | 2014-01-07 17:45:09 +0000 | [diff] [blame] | 165 | // DEPRECATED: It is now possible to modify the sample rate directly in a call |
| 166 | // to |ProcessStream|. |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 167 | // Sets the sample |rate| in Hz for both the primary and reverse audio |
| 168 | // streams. 8000, 16000 or 32000 Hz are permitted. |
| 169 | virtual int set_sample_rate_hz(int rate) = 0; |
| 170 | virtual int sample_rate_hz() const = 0; |
| 171 | |
andrew@webrtc.org | 60730cf | 2014-01-07 17:45:09 +0000 | [diff] [blame] | 172 | // DEPRECATED: It is now possible to modify the number of channels directly in |
| 173 | // a call to |ProcessStream|. |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 174 | // Sets the number of channels for the primary audio stream. Input frames must |
| 175 | // contain a number of channels given by |input_channels|, while output frames |
| 176 | // will be returned with number of channels given by |output_channels|. |
| 177 | virtual int set_num_channels(int input_channels, int output_channels) = 0; |
| 178 | virtual int num_input_channels() const = 0; |
| 179 | virtual int num_output_channels() const = 0; |
| 180 | |
andrew@webrtc.org | 60730cf | 2014-01-07 17:45:09 +0000 | [diff] [blame] | 181 | // DEPRECATED: It is now possible to modify the number of channels directly in |
| 182 | // a call to |AnalyzeReverseStream|. |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 183 | // Sets the number of channels for the reverse audio stream. Input frames must |
| 184 | // contain a number of channels given by |channels|. |
| 185 | virtual int set_num_reverse_channels(int channels) = 0; |
| 186 | virtual int num_reverse_channels() const = 0; |
| 187 | |
| 188 | // Processes a 10 ms |frame| of the primary audio stream. On the client-side, |
| 189 | // this is the near-end (or captured) audio. |
| 190 | // |
| 191 | // If needed for enabled functionality, any function with the set_stream_ tag |
| 192 | // must be called prior to processing the current frame. Any getter function |
| 193 | // with the stream_ tag which is needed should be called after processing. |
| 194 | // |
andrew@webrtc.org | 63a5098 | 2012-05-02 23:56:37 +0000 | [diff] [blame] | 195 | // The |sample_rate_hz_|, |num_channels_|, and |samples_per_channel_| |
andrew@webrtc.org | 60730cf | 2014-01-07 17:45:09 +0000 | [diff] [blame] | 196 | // members of |frame| must be valid. If changed from the previous call to this |
| 197 | // method, it will trigger an initialization. |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 198 | virtual int ProcessStream(AudioFrame* frame) = 0; |
| 199 | |
| 200 | // Analyzes a 10 ms |frame| of the reverse direction audio stream. The frame |
| 201 | // will not be modified. On the client-side, this is the far-end (or to be |
| 202 | // rendered) audio. |
| 203 | // |
| 204 | // It is only necessary to provide this if echo processing is enabled, as the |
| 205 | // reverse stream forms the echo reference signal. It is recommended, but not |
| 206 | // necessary, to provide if gain control is enabled. On the server-side this |
| 207 | // typically will not be used. If you're not sure what to pass in here, |
| 208 | // chances are you don't need to use it. |
| 209 | // |
andrew@webrtc.org | 63a5098 | 2012-05-02 23:56:37 +0000 | [diff] [blame] | 210 | // The |sample_rate_hz_|, |num_channels_|, and |samples_per_channel_| |
andrew@webrtc.org | 60730cf | 2014-01-07 17:45:09 +0000 | [diff] [blame] | 211 | // members of |frame| must be valid. |sample_rate_hz_| must correspond to |
| 212 | // |sample_rate_hz()| |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 213 | // |
| 214 | // TODO(ajm): add const to input; requires an implementation fix. |
| 215 | virtual int AnalyzeReverseStream(AudioFrame* frame) = 0; |
| 216 | |
| 217 | // This must be called if and only if echo processing is enabled. |
| 218 | // |
| 219 | // Sets the |delay| in ms between AnalyzeReverseStream() receiving a far-end |
| 220 | // frame and ProcessStream() receiving a near-end frame containing the |
| 221 | // corresponding echo. On the client-side this can be expressed as |
| 222 | // delay = (t_render - t_analyze) + (t_process - t_capture) |
| 223 | // where, |
| 224 | // - t_analyze is the time a frame is passed to AnalyzeReverseStream() and |
| 225 | // t_render is the time the first sample of the same frame is rendered by |
| 226 | // the audio hardware. |
| 227 | // - t_capture is the time the first sample of a frame is captured by the |
| 228 | // audio hardware and t_pull is the time the same frame is passed to |
| 229 | // ProcessStream(). |
| 230 | virtual int set_stream_delay_ms(int delay) = 0; |
| 231 | virtual int stream_delay_ms() const = 0; |
| 232 | |
andrew@webrtc.org | 6f9f817 | 2012-03-06 19:03:39 +0000 | [diff] [blame] | 233 | // Sets a delay |offset| in ms to add to the values passed in through |
| 234 | // set_stream_delay_ms(). May be positive or negative. |
| 235 | // |
| 236 | // Note that this could cause an otherwise valid value passed to |
| 237 | // set_stream_delay_ms() to return an error. |
| 238 | virtual void set_delay_offset_ms(int offset) = 0; |
| 239 | virtual int delay_offset_ms() const = 0; |
| 240 | |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 241 | // Starts recording debugging information to a file specified by |filename|, |
| 242 | // a NULL-terminated string. If there is an ongoing recording, the old file |
| 243 | // will be closed, and recording will continue in the newly specified file. |
| 244 | // An already existing file will be overwritten without warning. |
andrew@webrtc.org | 5ae19de | 2011-12-13 22:59:33 +0000 | [diff] [blame] | 245 | static const size_t kMaxFilenameSize = 1024; |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 246 | virtual int StartDebugRecording(const char filename[kMaxFilenameSize]) = 0; |
| 247 | |
henrikg@webrtc.org | 863b536 | 2013-12-06 16:05:17 +0000 | [diff] [blame] | 248 | // Same as above but uses an existing file handle. Takes ownership |
| 249 | // of |handle| and closes it at StopDebugRecording(). |
| 250 | virtual int StartDebugRecording(FILE* handle) = 0; |
| 251 | |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 252 | // Stops recording debugging information, and closes the file. Recording |
| 253 | // cannot be resumed in the same file (without overwriting it). |
| 254 | virtual int StopDebugRecording() = 0; |
| 255 | |
| 256 | // These provide access to the component interfaces and should never return |
| 257 | // NULL. The pointers will be valid for the lifetime of the APM instance. |
| 258 | // The memory for these objects is entirely managed internally. |
| 259 | virtual EchoCancellation* echo_cancellation() const = 0; |
| 260 | virtual EchoControlMobile* echo_control_mobile() const = 0; |
| 261 | virtual GainControl* gain_control() const = 0; |
| 262 | virtual HighPassFilter* high_pass_filter() const = 0; |
| 263 | virtual LevelEstimator* level_estimator() const = 0; |
| 264 | virtual NoiseSuppression* noise_suppression() const = 0; |
| 265 | virtual VoiceDetection* voice_detection() const = 0; |
| 266 | |
| 267 | struct Statistic { |
| 268 | int instant; // Instantaneous value. |
| 269 | int average; // Long-term average. |
| 270 | int maximum; // Long-term maximum. |
| 271 | int minimum; // Long-term minimum. |
| 272 | }; |
| 273 | |
andrew@webrtc.org | 648af74 | 2012-02-08 01:57:29 +0000 | [diff] [blame] | 274 | enum Error { |
| 275 | // Fatal errors. |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 276 | kNoError = 0, |
| 277 | kUnspecifiedError = -1, |
| 278 | kCreationFailedError = -2, |
| 279 | kUnsupportedComponentError = -3, |
| 280 | kUnsupportedFunctionError = -4, |
| 281 | kNullPointerError = -5, |
| 282 | kBadParameterError = -6, |
| 283 | kBadSampleRateError = -7, |
| 284 | kBadDataLengthError = -8, |
| 285 | kBadNumberChannelsError = -9, |
| 286 | kFileError = -10, |
| 287 | kStreamParameterNotSetError = -11, |
andrew@webrtc.org | 648af74 | 2012-02-08 01:57:29 +0000 | [diff] [blame] | 288 | kNotEnabledError = -12, |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 289 | |
andrew@webrtc.org | 648af74 | 2012-02-08 01:57:29 +0000 | [diff] [blame] | 290 | // Warnings are non-fatal. |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 291 | // This results when a set_stream_ parameter is out of range. Processing |
| 292 | // will continue, but the parameter may have been truncated. |
andrew@webrtc.org | 648af74 | 2012-02-08 01:57:29 +0000 | [diff] [blame] | 293 | kBadStreamParameterWarning = -13 |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 294 | }; |
| 295 | |
| 296 | // Inherited from Module. |
pbos@webrtc.org | 9162080 | 2013-08-02 11:44:11 +0000 | [diff] [blame] | 297 | virtual int32_t TimeUntilNextProcess() OVERRIDE; |
| 298 | virtual int32_t Process() OVERRIDE; |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 299 | }; |
| 300 | |
| 301 | // The acoustic echo cancellation (AEC) component provides better performance |
| 302 | // than AECM but also requires more processing power and is dependent on delay |
| 303 | // stability and reporting accuracy. As such it is well-suited and recommended |
| 304 | // for PC and IP phone applications. |
| 305 | // |
| 306 | // Not recommended to be enabled on the server-side. |
| 307 | class EchoCancellation { |
| 308 | public: |
| 309 | // EchoCancellation and EchoControlMobile may not be enabled simultaneously. |
| 310 | // Enabling one will disable the other. |
| 311 | virtual int Enable(bool enable) = 0; |
| 312 | virtual bool is_enabled() const = 0; |
| 313 | |
| 314 | // Differences in clock speed on the primary and reverse streams can impact |
| 315 | // the AEC performance. On the client-side, this could be seen when different |
| 316 | // render and capture devices are used, particularly with webcams. |
| 317 | // |
| 318 | // This enables a compensation mechanism, and requires that |
| 319 | // |set_device_sample_rate_hz()| and |set_stream_drift_samples()| be called. |
| 320 | virtual int enable_drift_compensation(bool enable) = 0; |
| 321 | virtual bool is_drift_compensation_enabled() const = 0; |
| 322 | |
| 323 | // Provides the sampling rate of the audio devices. It is assumed the render |
| 324 | // and capture devices use the same nominal sample rate. Required if and only |
| 325 | // if drift compensation is enabled. |
| 326 | virtual int set_device_sample_rate_hz(int rate) = 0; |
| 327 | virtual int device_sample_rate_hz() const = 0; |
| 328 | |
| 329 | // Sets the difference between the number of samples rendered and captured by |
| 330 | // the audio devices since the last call to |ProcessStream()|. Must be called |
andrew@webrtc.org | 6be1e93 | 2013-03-01 18:47:28 +0000 | [diff] [blame] | 331 | // if drift compensation is enabled, prior to |ProcessStream()|. |
| 332 | virtual void set_stream_drift_samples(int drift) = 0; |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 333 | virtual int stream_drift_samples() const = 0; |
| 334 | |
| 335 | enum SuppressionLevel { |
| 336 | kLowSuppression, |
| 337 | kModerateSuppression, |
| 338 | kHighSuppression |
| 339 | }; |
| 340 | |
| 341 | // Sets the aggressiveness of the suppressor. A higher level trades off |
| 342 | // double-talk performance for increased echo suppression. |
| 343 | virtual int set_suppression_level(SuppressionLevel level) = 0; |
| 344 | virtual SuppressionLevel suppression_level() const = 0; |
| 345 | |
| 346 | // Returns false if the current frame almost certainly contains no echo |
| 347 | // and true if it _might_ contain echo. |
| 348 | virtual bool stream_has_echo() const = 0; |
| 349 | |
| 350 | // Enables the computation of various echo metrics. These are obtained |
| 351 | // through |GetMetrics()|. |
| 352 | virtual int enable_metrics(bool enable) = 0; |
| 353 | virtual bool are_metrics_enabled() const = 0; |
| 354 | |
| 355 | // Each statistic is reported in dB. |
| 356 | // P_far: Far-end (render) signal power. |
| 357 | // P_echo: Near-end (capture) echo signal power. |
| 358 | // P_out: Signal power at the output of the AEC. |
| 359 | // P_a: Internal signal power at the point before the AEC's non-linear |
| 360 | // processor. |
| 361 | struct Metrics { |
| 362 | // RERL = ERL + ERLE |
| 363 | AudioProcessing::Statistic residual_echo_return_loss; |
| 364 | |
| 365 | // ERL = 10log_10(P_far / P_echo) |
| 366 | AudioProcessing::Statistic echo_return_loss; |
| 367 | |
| 368 | // ERLE = 10log_10(P_echo / P_out) |
| 369 | AudioProcessing::Statistic echo_return_loss_enhancement; |
| 370 | |
| 371 | // (Pre non-linear processing suppression) A_NLP = 10log_10(P_echo / P_a) |
| 372 | AudioProcessing::Statistic a_nlp; |
| 373 | }; |
| 374 | |
| 375 | // TODO(ajm): discuss the metrics update period. |
| 376 | virtual int GetMetrics(Metrics* metrics) = 0; |
| 377 | |
bjornv@google.com | 1ba3dbe | 2011-10-03 08:18:10 +0000 | [diff] [blame] | 378 | // Enables computation and logging of delay values. Statistics are obtained |
| 379 | // through |GetDelayMetrics()|. |
| 380 | virtual int enable_delay_logging(bool enable) = 0; |
| 381 | virtual bool is_delay_logging_enabled() const = 0; |
| 382 | |
| 383 | // The delay metrics consists of the delay |median| and the delay standard |
| 384 | // deviation |std|. The values are averaged over the time period since the |
| 385 | // last call to |GetDelayMetrics()|. |
| 386 | virtual int GetDelayMetrics(int* median, int* std) = 0; |
| 387 | |
bjornv@webrtc.org | 91d11b3 | 2013-03-05 16:53:09 +0000 | [diff] [blame] | 388 | // Returns a pointer to the low level AEC component. In case of multiple |
| 389 | // channels, the pointer to the first one is returned. A NULL pointer is |
| 390 | // returned when the AEC component is disabled or has not been initialized |
| 391 | // successfully. |
| 392 | virtual struct AecCore* aec_core() const = 0; |
| 393 | |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 394 | protected: |
andrew@webrtc.org | d72b3d6 | 2012-11-15 21:46:06 +0000 | [diff] [blame] | 395 | virtual ~EchoCancellation() {} |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 396 | }; |
| 397 | |
| 398 | // The acoustic echo control for mobile (AECM) component is a low complexity |
| 399 | // robust option intended for use on mobile devices. |
| 400 | // |
| 401 | // Not recommended to be enabled on the server-side. |
| 402 | class EchoControlMobile { |
| 403 | public: |
| 404 | // EchoCancellation and EchoControlMobile may not be enabled simultaneously. |
| 405 | // Enabling one will disable the other. |
| 406 | virtual int Enable(bool enable) = 0; |
| 407 | virtual bool is_enabled() const = 0; |
| 408 | |
| 409 | // Recommended settings for particular audio routes. In general, the louder |
| 410 | // the echo is expected to be, the higher this value should be set. The |
| 411 | // preferred setting may vary from device to device. |
| 412 | enum RoutingMode { |
| 413 | kQuietEarpieceOrHeadset, |
| 414 | kEarpiece, |
| 415 | kLoudEarpiece, |
| 416 | kSpeakerphone, |
| 417 | kLoudSpeakerphone |
| 418 | }; |
| 419 | |
| 420 | // Sets echo control appropriate for the audio routing |mode| on the device. |
| 421 | // It can and should be updated during a call if the audio routing changes. |
| 422 | virtual int set_routing_mode(RoutingMode mode) = 0; |
| 423 | virtual RoutingMode routing_mode() const = 0; |
| 424 | |
| 425 | // Comfort noise replaces suppressed background noise to maintain a |
| 426 | // consistent signal level. |
| 427 | virtual int enable_comfort_noise(bool enable) = 0; |
| 428 | virtual bool is_comfort_noise_enabled() const = 0; |
| 429 | |
bjornv@google.com | c4b939c | 2011-07-13 08:09:56 +0000 | [diff] [blame] | 430 | // A typical use case is to initialize the component with an echo path from a |
ajm@google.com | 22e6515 | 2011-07-18 18:03:01 +0000 | [diff] [blame] | 431 | // previous call. The echo path is retrieved using |GetEchoPath()|, typically |
| 432 | // at the end of a call. The data can then be stored for later use as an |
| 433 | // initializer before the next call, using |SetEchoPath()|. |
| 434 | // |
bjornv@google.com | c4b939c | 2011-07-13 08:09:56 +0000 | [diff] [blame] | 435 | // Controlling the echo path this way requires the data |size_bytes| to match |
| 436 | // the internal echo path size. This size can be acquired using |
| 437 | // |echo_path_size_bytes()|. |SetEchoPath()| causes an entire reset, worth |
ajm@google.com | 22e6515 | 2011-07-18 18:03:01 +0000 | [diff] [blame] | 438 | // noting if it is to be called during an ongoing call. |
| 439 | // |
| 440 | // It is possible that version incompatibilities may result in a stored echo |
| 441 | // path of the incorrect size. In this case, the stored path should be |
| 442 | // discarded. |
| 443 | virtual int SetEchoPath(const void* echo_path, size_t size_bytes) = 0; |
| 444 | virtual int GetEchoPath(void* echo_path, size_t size_bytes) const = 0; |
| 445 | |
| 446 | // The returned path size is guaranteed not to change for the lifetime of |
| 447 | // the application. |
| 448 | static size_t echo_path_size_bytes(); |
bjornv@google.com | c4b939c | 2011-07-13 08:09:56 +0000 | [diff] [blame] | 449 | |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 450 | protected: |
andrew@webrtc.org | d72b3d6 | 2012-11-15 21:46:06 +0000 | [diff] [blame] | 451 | virtual ~EchoControlMobile() {} |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 452 | }; |
| 453 | |
| 454 | // The automatic gain control (AGC) component brings the signal to an |
| 455 | // appropriate range. This is done by applying a digital gain directly and, in |
| 456 | // the analog mode, prescribing an analog gain to be applied at the audio HAL. |
| 457 | // |
| 458 | // Recommended to be enabled on the client-side. |
| 459 | class GainControl { |
| 460 | public: |
| 461 | virtual int Enable(bool enable) = 0; |
| 462 | virtual bool is_enabled() const = 0; |
| 463 | |
| 464 | // When an analog mode is set, this must be called prior to |ProcessStream()| |
| 465 | // to pass the current analog level from the audio HAL. Must be within the |
| 466 | // range provided to |set_analog_level_limits()|. |
| 467 | virtual int set_stream_analog_level(int level) = 0; |
| 468 | |
| 469 | // When an analog mode is set, this should be called after |ProcessStream()| |
| 470 | // to obtain the recommended new analog level for the audio HAL. It is the |
| 471 | // users responsibility to apply this level. |
| 472 | virtual int stream_analog_level() = 0; |
| 473 | |
| 474 | enum Mode { |
| 475 | // Adaptive mode intended for use if an analog volume control is available |
| 476 | // on the capture device. It will require the user to provide coupling |
| 477 | // between the OS mixer controls and AGC through the |stream_analog_level()| |
| 478 | // functions. |
| 479 | // |
| 480 | // It consists of an analog gain prescription for the audio device and a |
| 481 | // digital compression stage. |
| 482 | kAdaptiveAnalog, |
| 483 | |
| 484 | // Adaptive mode intended for situations in which an analog volume control |
| 485 | // is unavailable. It operates in a similar fashion to the adaptive analog |
| 486 | // mode, but with scaling instead applied in the digital domain. As with |
| 487 | // the analog mode, it additionally uses a digital compression stage. |
| 488 | kAdaptiveDigital, |
| 489 | |
| 490 | // Fixed mode which enables only the digital compression stage also used by |
| 491 | // the two adaptive modes. |
| 492 | // |
| 493 | // It is distinguished from the adaptive modes by considering only a |
| 494 | // short time-window of the input signal. It applies a fixed gain through |
| 495 | // most of the input level range, and compresses (gradually reduces gain |
| 496 | // with increasing level) the input signal at higher levels. This mode is |
| 497 | // preferred on embedded devices where the capture signal level is |
| 498 | // predictable, so that a known gain can be applied. |
| 499 | kFixedDigital |
| 500 | }; |
| 501 | |
| 502 | virtual int set_mode(Mode mode) = 0; |
| 503 | virtual Mode mode() const = 0; |
| 504 | |
| 505 | // Sets the target peak |level| (or envelope) of the AGC in dBFs (decibels |
| 506 | // from digital full-scale). The convention is to use positive values. For |
| 507 | // instance, passing in a value of 3 corresponds to -3 dBFs, or a target |
| 508 | // level 3 dB below full-scale. Limited to [0, 31]. |
| 509 | // |
| 510 | // TODO(ajm): use a negative value here instead, if/when VoE will similarly |
| 511 | // update its interface. |
| 512 | virtual int set_target_level_dbfs(int level) = 0; |
| 513 | virtual int target_level_dbfs() const = 0; |
| 514 | |
| 515 | // Sets the maximum |gain| the digital compression stage may apply, in dB. A |
| 516 | // higher number corresponds to greater compression, while a value of 0 will |
| 517 | // leave the signal uncompressed. Limited to [0, 90]. |
| 518 | virtual int set_compression_gain_db(int gain) = 0; |
| 519 | virtual int compression_gain_db() const = 0; |
| 520 | |
| 521 | // When enabled, the compression stage will hard limit the signal to the |
| 522 | // target level. Otherwise, the signal will be compressed but not limited |
| 523 | // above the target level. |
| 524 | virtual int enable_limiter(bool enable) = 0; |
| 525 | virtual bool is_limiter_enabled() const = 0; |
| 526 | |
| 527 | // Sets the |minimum| and |maximum| analog levels of the audio capture device. |
| 528 | // Must be set if and only if an analog mode is used. Limited to [0, 65535]. |
| 529 | virtual int set_analog_level_limits(int minimum, |
| 530 | int maximum) = 0; |
| 531 | virtual int analog_level_minimum() const = 0; |
| 532 | virtual int analog_level_maximum() const = 0; |
| 533 | |
| 534 | // Returns true if the AGC has detected a saturation event (period where the |
| 535 | // signal reaches digital full-scale) in the current frame and the analog |
| 536 | // level cannot be reduced. |
| 537 | // |
| 538 | // This could be used as an indicator to reduce or disable analog mic gain at |
| 539 | // the audio HAL. |
| 540 | virtual bool stream_is_saturated() const = 0; |
| 541 | |
| 542 | protected: |
andrew@webrtc.org | d72b3d6 | 2012-11-15 21:46:06 +0000 | [diff] [blame] | 543 | virtual ~GainControl() {} |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 544 | }; |
| 545 | |
| 546 | // A filtering component which removes DC offset and low-frequency noise. |
| 547 | // Recommended to be enabled on the client-side. |
| 548 | class HighPassFilter { |
| 549 | public: |
| 550 | virtual int Enable(bool enable) = 0; |
| 551 | virtual bool is_enabled() const = 0; |
| 552 | |
| 553 | protected: |
andrew@webrtc.org | d72b3d6 | 2012-11-15 21:46:06 +0000 | [diff] [blame] | 554 | virtual ~HighPassFilter() {} |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 555 | }; |
| 556 | |
| 557 | // An estimation component used to retrieve level metrics. |
| 558 | class LevelEstimator { |
| 559 | public: |
| 560 | virtual int Enable(bool enable) = 0; |
| 561 | virtual bool is_enabled() const = 0; |
| 562 | |
andrew@webrtc.org | 755b04a | 2011-11-15 16:57:56 +0000 | [diff] [blame] | 563 | // Returns the root mean square (RMS) level in dBFs (decibels from digital |
| 564 | // full-scale), or alternately dBov. It is computed over all primary stream |
| 565 | // frames since the last call to RMS(). The returned value is positive but |
| 566 | // should be interpreted as negative. It is constrained to [0, 127]. |
| 567 | // |
| 568 | // The computation follows: |
| 569 | // http://tools.ietf.org/html/draft-ietf-avtext-client-to-mixer-audio-level-05 |
| 570 | // with the intent that it can provide the RTP audio level indication. |
| 571 | // |
| 572 | // Frames passed to ProcessStream() with an |_energy| of zero are considered |
| 573 | // to have been muted. The RMS of the frame will be interpreted as -127. |
| 574 | virtual int RMS() = 0; |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 575 | |
| 576 | protected: |
andrew@webrtc.org | d72b3d6 | 2012-11-15 21:46:06 +0000 | [diff] [blame] | 577 | virtual ~LevelEstimator() {} |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 578 | }; |
| 579 | |
| 580 | // The noise suppression (NS) component attempts to remove noise while |
| 581 | // retaining speech. Recommended to be enabled on the client-side. |
| 582 | // |
| 583 | // Recommended to be enabled on the client-side. |
| 584 | class NoiseSuppression { |
| 585 | public: |
| 586 | virtual int Enable(bool enable) = 0; |
| 587 | virtual bool is_enabled() const = 0; |
| 588 | |
| 589 | // Determines the aggressiveness of the suppression. Increasing the level |
| 590 | // will reduce the noise level at the expense of a higher speech distortion. |
| 591 | enum Level { |
| 592 | kLow, |
| 593 | kModerate, |
| 594 | kHigh, |
| 595 | kVeryHigh |
| 596 | }; |
| 597 | |
| 598 | virtual int set_level(Level level) = 0; |
| 599 | virtual Level level() const = 0; |
| 600 | |
bjornv@webrtc.org | 08329f4 | 2012-07-12 21:00:43 +0000 | [diff] [blame] | 601 | // Returns the internally computed prior speech probability of current frame |
| 602 | // averaged over output channels. This is not supported in fixed point, for |
| 603 | // which |kUnsupportedFunctionError| is returned. |
| 604 | virtual float speech_probability() const = 0; |
| 605 | |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 606 | protected: |
andrew@webrtc.org | d72b3d6 | 2012-11-15 21:46:06 +0000 | [diff] [blame] | 607 | virtual ~NoiseSuppression() {} |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 608 | }; |
| 609 | |
| 610 | // The voice activity detection (VAD) component analyzes the stream to |
| 611 | // determine if voice is present. A facility is also provided to pass in an |
| 612 | // external VAD decision. |
andrew@webrtc.org | ed083d4 | 2011-09-19 15:28:51 +0000 | [diff] [blame] | 613 | // |
| 614 | // In addition to |stream_has_voice()| the VAD decision is provided through the |
andrew@webrtc.org | 63a5098 | 2012-05-02 23:56:37 +0000 | [diff] [blame] | 615 | // |AudioFrame| passed to |ProcessStream()|. The |vad_activity_| member will be |
andrew@webrtc.org | ed083d4 | 2011-09-19 15:28:51 +0000 | [diff] [blame] | 616 | // modified to reflect the current decision. |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 617 | class VoiceDetection { |
| 618 | public: |
| 619 | virtual int Enable(bool enable) = 0; |
| 620 | virtual bool is_enabled() const = 0; |
| 621 | |
| 622 | // Returns true if voice is detected in the current frame. Should be called |
| 623 | // after |ProcessStream()|. |
| 624 | virtual bool stream_has_voice() const = 0; |
| 625 | |
| 626 | // Some of the APM functionality requires a VAD decision. In the case that |
| 627 | // a decision is externally available for the current frame, it can be passed |
| 628 | // in here, before |ProcessStream()| is called. |
| 629 | // |
| 630 | // VoiceDetection does _not_ need to be enabled to use this. If it happens to |
| 631 | // be enabled, detection will be skipped for any frame in which an external |
| 632 | // VAD decision is provided. |
| 633 | virtual int set_stream_has_voice(bool has_voice) = 0; |
| 634 | |
| 635 | // Specifies the likelihood that a frame will be declared to contain voice. |
| 636 | // A higher value makes it more likely that speech will not be clipped, at |
| 637 | // the expense of more noise being detected as voice. |
| 638 | enum Likelihood { |
| 639 | kVeryLowLikelihood, |
| 640 | kLowLikelihood, |
| 641 | kModerateLikelihood, |
| 642 | kHighLikelihood |
| 643 | }; |
| 644 | |
| 645 | virtual int set_likelihood(Likelihood likelihood) = 0; |
| 646 | virtual Likelihood likelihood() const = 0; |
| 647 | |
| 648 | // Sets the |size| of the frames in ms on which the VAD will operate. Larger |
| 649 | // frames will improve detection accuracy, but reduce the frequency of |
| 650 | // updates. |
| 651 | // |
| 652 | // This does not impact the size of frames passed to |ProcessStream()|. |
| 653 | virtual int set_frame_size_ms(int size) = 0; |
| 654 | virtual int frame_size_ms() const = 0; |
| 655 | |
| 656 | protected: |
andrew@webrtc.org | d72b3d6 | 2012-11-15 21:46:06 +0000 | [diff] [blame] | 657 | virtual ~VoiceDetection() {} |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 658 | }; |
| 659 | } // namespace webrtc |
| 660 | |
andrew@webrtc.org | d72b3d6 | 2012-11-15 21:46:06 +0000 | [diff] [blame] | 661 | #endif // WEBRTC_MODULES_AUDIO_PROCESSING_INCLUDE_AUDIO_PROCESSING_H_ |