blob: 26b20206ac3915274d232e41137099ae5eac98e7 [file] [log] [blame]
niklase@google.com470e71d2011-07-07 08:21:25 +00001/*
andrew@webrtc.org40654032012-01-30 20:51:15 +00002 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
niklase@google.com470e71d2011-07-07 08:21:25 +00003 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
andrew@webrtc.org78693fe2013-03-01 16:36:19 +000011#include "webrtc/modules/audio_processing/audio_processing_impl.h"
niklase@google.com470e71d2011-07-07 08:21:25 +000012
ajm@google.com808e0e02011-08-03 21:08:51 +000013#include <assert.h>
niklase@google.com470e71d2011-07-07 08:21:25 +000014
andrew@webrtc.org60730cf2014-01-07 17:45:09 +000015#include "webrtc/common_audio/signal_processing/include/signal_processing_library.h"
andrew@webrtc.org78693fe2013-03-01 16:36:19 +000016#include "webrtc/modules/audio_processing/audio_buffer.h"
andrew@webrtc.org56e4a052014-02-27 22:23:17 +000017#include "webrtc/modules/audio_processing/echo_cancellation_impl.h"
andrew@webrtc.org78693fe2013-03-01 16:36:19 +000018#include "webrtc/modules/audio_processing/echo_control_mobile_impl.h"
19#include "webrtc/modules/audio_processing/gain_control_impl.h"
20#include "webrtc/modules/audio_processing/high_pass_filter_impl.h"
21#include "webrtc/modules/audio_processing/level_estimator_impl.h"
22#include "webrtc/modules/audio_processing/noise_suppression_impl.h"
23#include "webrtc/modules/audio_processing/processing_component.h"
andrew@webrtc.org78693fe2013-03-01 16:36:19 +000024#include "webrtc/modules/audio_processing/voice_detection_impl.h"
25#include "webrtc/modules/interface/module_common_types.h"
andrew@webrtc.org60730cf2014-01-07 17:45:09 +000026#include "webrtc/system_wrappers/interface/compile_assert.h"
andrew@webrtc.org78693fe2013-03-01 16:36:19 +000027#include "webrtc/system_wrappers/interface/critical_section_wrapper.h"
28#include "webrtc/system_wrappers/interface/file_wrapper.h"
29#include "webrtc/system_wrappers/interface/logging.h"
andrew@webrtc.org7bf26462011-12-03 00:03:31 +000030
31#ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP
32// Files generated at build-time by the protobuf compiler.
leozwang@webrtc.orga3736342012-03-16 21:36:00 +000033#ifdef WEBRTC_ANDROID_PLATFORM_BUILD
leozwang@webrtc.org534e4952012-10-22 21:21:52 +000034#include "external/webrtc/webrtc/modules/audio_processing/debug.pb.h"
leozwang@google.comce9bfbb2011-08-03 23:34:31 +000035#else
ajm@google.com808e0e02011-08-03 21:08:51 +000036#include "webrtc/audio_processing/debug.pb.h"
leozwang@google.comce9bfbb2011-08-03 23:34:31 +000037#endif
andrew@webrtc.org7bf26462011-12-03 00:03:31 +000038#endif // WEBRTC_AUDIOPROC_DEBUG_DUMP
niklase@google.com470e71d2011-07-07 08:21:25 +000039
andrew@webrtc.org60730cf2014-01-07 17:45:09 +000040static const int kChunkSizeMs = 10;
41
42#define RETURN_ON_ERR(expr) \
43 do { \
44 int err = expr; \
45 if (err != kNoError) { \
46 return err; \
47 } \
48 } while (0)
49
niklase@google.com470e71d2011-07-07 08:21:25 +000050namespace webrtc {
andrew@webrtc.org60730cf2014-01-07 17:45:09 +000051
52// Throughout webrtc, it's assumed that success is represented by zero.
53COMPILE_ASSERT(AudioProcessing::kNoError == 0, no_error_must_be_zero);
54
niklase@google.com470e71d2011-07-07 08:21:25 +000055AudioProcessing* AudioProcessing::Create(int id) {
andrew@webrtc.orge84978f2014-01-25 02:09:06 +000056 return Create();
57}
58
59AudioProcessing* AudioProcessing::Create() {
60 Config config;
61 return Create(config);
62}
63
64AudioProcessing* AudioProcessing::Create(const Config& config) {
65 AudioProcessingImpl* apm = new AudioProcessingImpl(config);
niklase@google.com470e71d2011-07-07 08:21:25 +000066 if (apm->Initialize() != kNoError) {
67 delete apm;
68 apm = NULL;
69 }
70
71 return apm;
72}
73
andrew@webrtc.orge84978f2014-01-25 02:09:06 +000074AudioProcessingImpl::AudioProcessingImpl(const Config& config)
andrew@webrtc.org60730cf2014-01-07 17:45:09 +000075 : echo_cancellation_(NULL),
niklase@google.com470e71d2011-07-07 08:21:25 +000076 echo_control_mobile_(NULL),
77 gain_control_(NULL),
78 high_pass_filter_(NULL),
79 level_estimator_(NULL),
80 noise_suppression_(NULL),
81 voice_detection_(NULL),
niklase@google.com470e71d2011-07-07 08:21:25 +000082 crit_(CriticalSectionWrapper::CreateCriticalSection()),
83 render_audio_(NULL),
84 capture_audio_(NULL),
andrew@webrtc.org7bf26462011-12-03 00:03:31 +000085#ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP
86 debug_file_(FileWrapper::Create()),
87 event_msg_(new audioproc::Event()),
88#endif
niklase@google.com470e71d2011-07-07 08:21:25 +000089 sample_rate_hz_(kSampleRate16kHz),
90 split_sample_rate_hz_(kSampleRate16kHz),
andrew@webrtc.org60730cf2014-01-07 17:45:09 +000091 samples_per_channel_(kChunkSizeMs * sample_rate_hz_ / 1000),
niklase@google.com470e71d2011-07-07 08:21:25 +000092 stream_delay_ms_(0),
andrew@webrtc.org6f9f8172012-03-06 19:03:39 +000093 delay_offset_ms_(0),
niklase@google.com470e71d2011-07-07 08:21:25 +000094 was_stream_delay_set_(false),
ajm@google.com808e0e02011-08-03 21:08:51 +000095 num_reverse_channels_(1),
96 num_input_channels_(1),
andrew@webrtc.org07b59502014-02-12 16:41:13 +000097 num_output_channels_(1),
andrew@webrtc.org38bf2492014-02-13 17:43:44 +000098 output_will_be_muted_(false),
andrew@webrtc.org07b59502014-02-12 16:41:13 +000099 key_pressed_(false) {
andrew@webrtc.org56e4a052014-02-27 22:23:17 +0000100 echo_cancellation_ = new EchoCancellationImpl(this, crit_);
niklase@google.com470e71d2011-07-07 08:21:25 +0000101 component_list_.push_back(echo_cancellation_);
102
andrew@webrtc.org56e4a052014-02-27 22:23:17 +0000103 echo_control_mobile_ = new EchoControlMobileImpl(this, crit_);
niklase@google.com470e71d2011-07-07 08:21:25 +0000104 component_list_.push_back(echo_control_mobile_);
105
andrew@webrtc.org56e4a052014-02-27 22:23:17 +0000106 gain_control_ = new GainControlImpl(this, crit_);
niklase@google.com470e71d2011-07-07 08:21:25 +0000107 component_list_.push_back(gain_control_);
108
andrew@webrtc.org56e4a052014-02-27 22:23:17 +0000109 high_pass_filter_ = new HighPassFilterImpl(this, crit_);
niklase@google.com470e71d2011-07-07 08:21:25 +0000110 component_list_.push_back(high_pass_filter_);
111
andrew@webrtc.org56e4a052014-02-27 22:23:17 +0000112 level_estimator_ = new LevelEstimatorImpl(this, crit_);
niklase@google.com470e71d2011-07-07 08:21:25 +0000113 component_list_.push_back(level_estimator_);
114
andrew@webrtc.org56e4a052014-02-27 22:23:17 +0000115 noise_suppression_ = new NoiseSuppressionImpl(this, crit_);
niklase@google.com470e71d2011-07-07 08:21:25 +0000116 component_list_.push_back(noise_suppression_);
117
andrew@webrtc.org56e4a052014-02-27 22:23:17 +0000118 voice_detection_ = new VoiceDetectionImpl(this, crit_);
niklase@google.com470e71d2011-07-07 08:21:25 +0000119 component_list_.push_back(voice_detection_);
andrew@webrtc.orge84978f2014-01-25 02:09:06 +0000120
121 SetExtraOptions(config);
niklase@google.com470e71d2011-07-07 08:21:25 +0000122}
123
124AudioProcessingImpl::~AudioProcessingImpl() {
andrew@webrtc.org81865342012-10-27 00:28:27 +0000125 {
126 CriticalSectionScoped crit_scoped(crit_);
127 while (!component_list_.empty()) {
128 ProcessingComponent* component = component_list_.front();
129 component->Destroy();
130 delete component;
131 component_list_.pop_front();
132 }
niklase@google.com470e71d2011-07-07 08:21:25 +0000133
andrew@webrtc.org7bf26462011-12-03 00:03:31 +0000134#ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP
andrew@webrtc.org81865342012-10-27 00:28:27 +0000135 if (debug_file_->Open()) {
136 debug_file_->CloseFile();
137 }
andrew@webrtc.org7bf26462011-12-03 00:03:31 +0000138#endif
ajm@google.com808e0e02011-08-03 21:08:51 +0000139
andrew@webrtc.org81865342012-10-27 00:28:27 +0000140 if (render_audio_) {
141 delete render_audio_;
142 render_audio_ = NULL;
143 }
144
145 if (capture_audio_) {
146 delete capture_audio_;
147 capture_audio_ = NULL;
148 }
niklase@google.com470e71d2011-07-07 08:21:25 +0000149 }
150
andrew@webrtc.org16cfbe22012-08-29 16:58:25 +0000151 delete crit_;
152 crit_ = NULL;
niklase@google.com470e71d2011-07-07 08:21:25 +0000153}
154
niklase@google.com470e71d2011-07-07 08:21:25 +0000155int AudioProcessingImpl::split_sample_rate_hz() const {
156 return split_sample_rate_hz_;
157}
158
159int AudioProcessingImpl::Initialize() {
andrew@webrtc.org40654032012-01-30 20:51:15 +0000160 CriticalSectionScoped crit_scoped(crit_);
niklase@google.com470e71d2011-07-07 08:21:25 +0000161 return InitializeLocked();
162}
163
164int AudioProcessingImpl::InitializeLocked() {
165 if (render_audio_ != NULL) {
166 delete render_audio_;
167 render_audio_ = NULL;
168 }
169
170 if (capture_audio_ != NULL) {
171 delete capture_audio_;
172 capture_audio_ = NULL;
173 }
174
ajm@google.com808e0e02011-08-03 21:08:51 +0000175 render_audio_ = new AudioBuffer(num_reverse_channels_,
niklase@google.com470e71d2011-07-07 08:21:25 +0000176 samples_per_channel_);
ajm@google.com808e0e02011-08-03 21:08:51 +0000177 capture_audio_ = new AudioBuffer(num_input_channels_,
niklase@google.com470e71d2011-07-07 08:21:25 +0000178 samples_per_channel_);
179
niklase@google.com470e71d2011-07-07 08:21:25 +0000180 // Initialize all components.
181 std::list<ProcessingComponent*>::iterator it;
andrew@webrtc.org81865342012-10-27 00:28:27 +0000182 for (it = component_list_.begin(); it != component_list_.end(); ++it) {
niklase@google.com470e71d2011-07-07 08:21:25 +0000183 int err = (*it)->Initialize();
184 if (err != kNoError) {
185 return err;
186 }
187 }
188
andrew@webrtc.org7bf26462011-12-03 00:03:31 +0000189#ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP
ajm@google.com808e0e02011-08-03 21:08:51 +0000190 if (debug_file_->Open()) {
191 int err = WriteInitMessage();
192 if (err != kNoError) {
193 return err;
194 }
195 }
andrew@webrtc.org7bf26462011-12-03 00:03:31 +0000196#endif
ajm@google.com808e0e02011-08-03 21:08:51 +0000197
niklase@google.com470e71d2011-07-07 08:21:25 +0000198 return kNoError;
199}
200
andrew@webrtc.org61e596f2013-07-25 18:28:29 +0000201void AudioProcessingImpl::SetExtraOptions(const Config& config) {
andrew@webrtc.orge84978f2014-01-25 02:09:06 +0000202 CriticalSectionScoped crit_scoped(crit_);
andrew@webrtc.org61e596f2013-07-25 18:28:29 +0000203 std::list<ProcessingComponent*>::iterator it;
204 for (it = component_list_.begin(); it != component_list_.end(); ++it)
205 (*it)->SetExtraOptions(config);
206}
207
aluebs@webrtc.org0b72f582013-11-19 15:17:51 +0000208int AudioProcessingImpl::EnableExperimentalNs(bool enable) {
209 return kNoError;
210}
211
niklase@google.com470e71d2011-07-07 08:21:25 +0000212int AudioProcessingImpl::set_sample_rate_hz(int rate) {
andrew@webrtc.org40654032012-01-30 20:51:15 +0000213 CriticalSectionScoped crit_scoped(crit_);
andrew@webrtc.org81865342012-10-27 00:28:27 +0000214 if (rate == sample_rate_hz_) {
215 return kNoError;
216 }
niklase@google.com470e71d2011-07-07 08:21:25 +0000217 if (rate != kSampleRate8kHz &&
218 rate != kSampleRate16kHz &&
219 rate != kSampleRate32kHz) {
220 return kBadParameterError;
221 }
andrew@webrtc.org78693fe2013-03-01 16:36:19 +0000222 if (echo_control_mobile_->is_enabled() && rate > kSampleRate16kHz) {
223 LOG(LS_ERROR) << "AECM only supports 16 kHz or lower sample rates";
224 return kUnsupportedComponentError;
225 }
niklase@google.com470e71d2011-07-07 08:21:25 +0000226
227 sample_rate_hz_ = rate;
228 samples_per_channel_ = rate / 100;
229
230 if (sample_rate_hz_ == kSampleRate32kHz) {
231 split_sample_rate_hz_ = kSampleRate16kHz;
232 } else {
233 split_sample_rate_hz_ = sample_rate_hz_;
234 }
235
236 return InitializeLocked();
237}
238
239int AudioProcessingImpl::sample_rate_hz() const {
henrika@webrtc.org19da7192013-04-05 14:34:57 +0000240 CriticalSectionScoped crit_scoped(crit_);
niklase@google.com470e71d2011-07-07 08:21:25 +0000241 return sample_rate_hz_;
242}
243
244int AudioProcessingImpl::set_num_reverse_channels(int channels) {
andrew@webrtc.org40654032012-01-30 20:51:15 +0000245 CriticalSectionScoped crit_scoped(crit_);
andrew@webrtc.org81865342012-10-27 00:28:27 +0000246 if (channels == num_reverse_channels_) {
247 return kNoError;
248 }
niklase@google.com470e71d2011-07-07 08:21:25 +0000249 // Only stereo supported currently.
250 if (channels > 2 || channels < 1) {
251 return kBadParameterError;
252 }
253
ajm@google.com808e0e02011-08-03 21:08:51 +0000254 num_reverse_channels_ = channels;
niklase@google.com470e71d2011-07-07 08:21:25 +0000255
256 return InitializeLocked();
257}
258
259int AudioProcessingImpl::num_reverse_channels() const {
ajm@google.com808e0e02011-08-03 21:08:51 +0000260 return num_reverse_channels_;
niklase@google.com470e71d2011-07-07 08:21:25 +0000261}
262
263int AudioProcessingImpl::set_num_channels(
264 int input_channels,
265 int output_channels) {
andrew@webrtc.org40654032012-01-30 20:51:15 +0000266 CriticalSectionScoped crit_scoped(crit_);
andrew@webrtc.org81865342012-10-27 00:28:27 +0000267 if (input_channels == num_input_channels_ &&
268 output_channels == num_output_channels_) {
269 return kNoError;
270 }
niklase@google.com470e71d2011-07-07 08:21:25 +0000271 if (output_channels > input_channels) {
272 return kBadParameterError;
273 }
niklase@google.com470e71d2011-07-07 08:21:25 +0000274 // Only stereo supported currently.
andrew@webrtc.org81865342012-10-27 00:28:27 +0000275 if (input_channels > 2 || input_channels < 1 ||
276 output_channels > 2 || output_channels < 1) {
niklase@google.com470e71d2011-07-07 08:21:25 +0000277 return kBadParameterError;
278 }
279
ajm@google.com808e0e02011-08-03 21:08:51 +0000280 num_input_channels_ = input_channels;
281 num_output_channels_ = output_channels;
niklase@google.com470e71d2011-07-07 08:21:25 +0000282
283 return InitializeLocked();
284}
285
286int AudioProcessingImpl::num_input_channels() const {
ajm@google.com808e0e02011-08-03 21:08:51 +0000287 return num_input_channels_;
niklase@google.com470e71d2011-07-07 08:21:25 +0000288}
289
290int AudioProcessingImpl::num_output_channels() const {
ajm@google.com808e0e02011-08-03 21:08:51 +0000291 return num_output_channels_;
niklase@google.com470e71d2011-07-07 08:21:25 +0000292}
293
andrew@webrtc.org17342e52014-02-12 22:28:31 +0000294void AudioProcessingImpl::set_output_will_be_muted(bool muted) {
295 output_will_be_muted_ = muted;
296}
297
298bool AudioProcessingImpl::output_will_be_muted() const {
299 return output_will_be_muted_;
300}
301
andrew@webrtc.org60730cf2014-01-07 17:45:09 +0000302int AudioProcessingImpl::MaybeInitializeLocked(int sample_rate_hz,
303 int num_input_channels, int num_output_channels, int num_reverse_channels) {
304 if (sample_rate_hz == sample_rate_hz_ &&
305 num_input_channels == num_input_channels_ &&
306 num_output_channels == num_output_channels_ &&
307 num_reverse_channels == num_reverse_channels_) {
308 return kNoError;
309 }
310
311 if (sample_rate_hz != kSampleRate8kHz &&
312 sample_rate_hz != kSampleRate16kHz &&
313 sample_rate_hz != kSampleRate32kHz) {
314 return kBadSampleRateError;
315 }
316 if (num_output_channels > num_input_channels) {
317 return kBadNumberChannelsError;
318 }
319 // Only mono and stereo supported currently.
320 if (num_input_channels > 2 || num_input_channels < 1 ||
321 num_output_channels > 2 || num_output_channels < 1 ||
322 num_reverse_channels > 2 || num_reverse_channels < 1) {
323 return kBadNumberChannelsError;
324 }
325 if (echo_control_mobile_->is_enabled() && sample_rate_hz > kSampleRate16kHz) {
326 LOG(LS_ERROR) << "AECM only supports 16 or 8 kHz sample rates";
327 return kUnsupportedComponentError;
328 }
329
330 sample_rate_hz_ = sample_rate_hz;
331 samples_per_channel_ = kChunkSizeMs * sample_rate_hz / 1000;
332 num_input_channels_ = num_input_channels;
333 num_output_channels_ = num_output_channels;
334 num_reverse_channels_ = num_reverse_channels;
335
336 if (sample_rate_hz_ == kSampleRate32kHz) {
337 split_sample_rate_hz_ = kSampleRate16kHz;
338 } else {
339 split_sample_rate_hz_ = sample_rate_hz_;
340 }
341
342 return InitializeLocked();
343}
344
niklase@google.com470e71d2011-07-07 08:21:25 +0000345int AudioProcessingImpl::ProcessStream(AudioFrame* frame) {
andrew@webrtc.org40654032012-01-30 20:51:15 +0000346 CriticalSectionScoped crit_scoped(crit_);
niklase@google.com470e71d2011-07-07 08:21:25 +0000347 int err = kNoError;
348
349 if (frame == NULL) {
350 return kNullPointerError;
351 }
andrew@webrtc.org60730cf2014-01-07 17:45:09 +0000352 // TODO(ajm): We now always set the output channels equal to the input
353 // channels here. Remove the ability to downmix entirely.
354 RETURN_ON_ERR(MaybeInitializeLocked(frame->sample_rate_hz_,
355 frame->num_channels_, frame->num_channels_, num_reverse_channels_));
andrew@webrtc.org63a50982012-05-02 23:56:37 +0000356 if (frame->samples_per_channel_ != samples_per_channel_) {
niklase@google.com470e71d2011-07-07 08:21:25 +0000357 return kBadDataLengthError;
358 }
359
andrew@webrtc.org7bf26462011-12-03 00:03:31 +0000360#ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP
niklase@google.com470e71d2011-07-07 08:21:25 +0000361 if (debug_file_->Open()) {
ajm@google.com808e0e02011-08-03 21:08:51 +0000362 event_msg_->set_type(audioproc::Event::STREAM);
363 audioproc::Stream* msg = event_msg_->mutable_stream();
andrew@webrtc.org755b04a2011-11-15 16:57:56 +0000364 const size_t data_size = sizeof(int16_t) *
andrew@webrtc.org63a50982012-05-02 23:56:37 +0000365 frame->samples_per_channel_ *
366 frame->num_channels_;
367 msg->set_input_data(frame->data_, data_size);
ajm@google.com808e0e02011-08-03 21:08:51 +0000368 msg->set_delay(stream_delay_ms_);
369 msg->set_drift(echo_cancellation_->stream_drift_samples());
370 msg->set_level(gain_control_->stream_analog_level());
andrew@webrtc.orgce8e0772014-02-12 15:28:30 +0000371 msg->set_keypress(key_pressed_);
niklase@google.com470e71d2011-07-07 08:21:25 +0000372 }
andrew@webrtc.org7bf26462011-12-03 00:03:31 +0000373#endif
niklase@google.com470e71d2011-07-07 08:21:25 +0000374
375 capture_audio_->DeinterleaveFrom(frame);
376
377 // TODO(ajm): experiment with mixing and AEC placement.
ajm@google.com808e0e02011-08-03 21:08:51 +0000378 if (num_output_channels_ < num_input_channels_) {
379 capture_audio_->Mix(num_output_channels_);
andrew@webrtc.org63a50982012-05-02 23:56:37 +0000380 frame->num_channels_ = num_output_channels_;
niklase@google.com470e71d2011-07-07 08:21:25 +0000381 }
382
andrew@webrtc.org369166a2012-04-24 18:38:03 +0000383 bool data_processed = is_data_processed();
384 if (analysis_needed(data_processed)) {
andrew@webrtc.org755b04a2011-11-15 16:57:56 +0000385 for (int i = 0; i < num_output_channels_; i++) {
niklase@google.com470e71d2011-07-07 08:21:25 +0000386 // Split into a low and high band.
andrew@webrtc.org60730cf2014-01-07 17:45:09 +0000387 WebRtcSpl_AnalysisQMF(capture_audio_->data(i),
388 capture_audio_->samples_per_channel(),
389 capture_audio_->low_pass_split_data(i),
390 capture_audio_->high_pass_split_data(i),
391 capture_audio_->analysis_filter_state1(i),
392 capture_audio_->analysis_filter_state2(i));
niklase@google.com470e71d2011-07-07 08:21:25 +0000393 }
394 }
395
396 err = high_pass_filter_->ProcessCaptureAudio(capture_audio_);
397 if (err != kNoError) {
398 return err;
399 }
400
401 err = gain_control_->AnalyzeCaptureAudio(capture_audio_);
402 if (err != kNoError) {
403 return err;
404 }
405
406 err = echo_cancellation_->ProcessCaptureAudio(capture_audio_);
407 if (err != kNoError) {
408 return err;
409 }
410
411 if (echo_control_mobile_->is_enabled() &&
412 noise_suppression_->is_enabled()) {
413 capture_audio_->CopyLowPassToReference();
414 }
415
416 err = noise_suppression_->ProcessCaptureAudio(capture_audio_);
417 if (err != kNoError) {
418 return err;
419 }
420
421 err = echo_control_mobile_->ProcessCaptureAudio(capture_audio_);
422 if (err != kNoError) {
423 return err;
424 }
425
426 err = voice_detection_->ProcessCaptureAudio(capture_audio_);
427 if (err != kNoError) {
428 return err;
429 }
430
431 err = gain_control_->ProcessCaptureAudio(capture_audio_);
432 if (err != kNoError) {
433 return err;
434 }
435
andrew@webrtc.org369166a2012-04-24 18:38:03 +0000436 if (synthesis_needed(data_processed)) {
ajm@google.com808e0e02011-08-03 21:08:51 +0000437 for (int i = 0; i < num_output_channels_; i++) {
niklase@google.com470e71d2011-07-07 08:21:25 +0000438 // Recombine low and high bands.
andrew@webrtc.org60730cf2014-01-07 17:45:09 +0000439 WebRtcSpl_SynthesisQMF(capture_audio_->low_pass_split_data(i),
440 capture_audio_->high_pass_split_data(i),
441 capture_audio_->samples_per_split_channel(),
442 capture_audio_->data(i),
443 capture_audio_->synthesis_filter_state1(i),
444 capture_audio_->synthesis_filter_state2(i));
niklase@google.com470e71d2011-07-07 08:21:25 +0000445 }
446 }
447
andrew@webrtc.org755b04a2011-11-15 16:57:56 +0000448 // The level estimator operates on the recombined data.
449 err = level_estimator_->ProcessStream(capture_audio_);
450 if (err != kNoError) {
451 return err;
452 }
453
andrew@webrtc.org369166a2012-04-24 18:38:03 +0000454 capture_audio_->InterleaveTo(frame, interleave_needed(data_processed));
niklase@google.com470e71d2011-07-07 08:21:25 +0000455
andrew@webrtc.org7bf26462011-12-03 00:03:31 +0000456#ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP
ajm@google.com808e0e02011-08-03 21:08:51 +0000457 if (debug_file_->Open()) {
458 audioproc::Stream* msg = event_msg_->mutable_stream();
andrew@webrtc.org755b04a2011-11-15 16:57:56 +0000459 const size_t data_size = sizeof(int16_t) *
andrew@webrtc.org63a50982012-05-02 23:56:37 +0000460 frame->samples_per_channel_ *
461 frame->num_channels_;
462 msg->set_output_data(frame->data_, data_size);
ajm@google.com808e0e02011-08-03 21:08:51 +0000463 err = WriteMessageToDebugFile();
464 if (err != kNoError) {
465 return err;
466 }
467 }
andrew@webrtc.org7bf26462011-12-03 00:03:31 +0000468#endif
ajm@google.com808e0e02011-08-03 21:08:51 +0000469
andrew@webrtc.org1e916932011-11-29 18:28:57 +0000470 was_stream_delay_set_ = false;
niklase@google.com470e71d2011-07-07 08:21:25 +0000471 return kNoError;
472}
473
andrew@webrtc.org60730cf2014-01-07 17:45:09 +0000474// TODO(ajm): Have AnalyzeReverseStream accept sample rates not matching the
475// primary stream and convert ourselves rather than having the user manage it.
476// We can be smarter and use the splitting filter when appropriate. Similarly,
477// perform downmixing here.
niklase@google.com470e71d2011-07-07 08:21:25 +0000478int AudioProcessingImpl::AnalyzeReverseStream(AudioFrame* frame) {
andrew@webrtc.org40654032012-01-30 20:51:15 +0000479 CriticalSectionScoped crit_scoped(crit_);
niklase@google.com470e71d2011-07-07 08:21:25 +0000480 int err = kNoError;
niklase@google.com470e71d2011-07-07 08:21:25 +0000481 if (frame == NULL) {
482 return kNullPointerError;
483 }
andrew@webrtc.org63a50982012-05-02 23:56:37 +0000484 if (frame->sample_rate_hz_ != sample_rate_hz_) {
niklase@google.com470e71d2011-07-07 08:21:25 +0000485 return kBadSampleRateError;
486 }
andrew@webrtc.org60730cf2014-01-07 17:45:09 +0000487 RETURN_ON_ERR(MaybeInitializeLocked(sample_rate_hz_, num_input_channels_,
488 num_output_channels_, frame->num_channels_));
niklase@google.com470e71d2011-07-07 08:21:25 +0000489
andrew@webrtc.org7bf26462011-12-03 00:03:31 +0000490#ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP
niklase@google.com470e71d2011-07-07 08:21:25 +0000491 if (debug_file_->Open()) {
ajm@google.com808e0e02011-08-03 21:08:51 +0000492 event_msg_->set_type(audioproc::Event::REVERSE_STREAM);
493 audioproc::ReverseStream* msg = event_msg_->mutable_reverse_stream();
andrew@webrtc.org755b04a2011-11-15 16:57:56 +0000494 const size_t data_size = sizeof(int16_t) *
andrew@webrtc.org63a50982012-05-02 23:56:37 +0000495 frame->samples_per_channel_ *
496 frame->num_channels_;
497 msg->set_data(frame->data_, data_size);
ajm@google.com808e0e02011-08-03 21:08:51 +0000498 err = WriteMessageToDebugFile();
499 if (err != kNoError) {
500 return err;
niklase@google.com470e71d2011-07-07 08:21:25 +0000501 }
502 }
andrew@webrtc.org7bf26462011-12-03 00:03:31 +0000503#endif
niklase@google.com470e71d2011-07-07 08:21:25 +0000504
505 render_audio_->DeinterleaveFrom(frame);
506
niklase@google.com470e71d2011-07-07 08:21:25 +0000507 if (sample_rate_hz_ == kSampleRate32kHz) {
ajm@google.com808e0e02011-08-03 21:08:51 +0000508 for (int i = 0; i < num_reverse_channels_; i++) {
niklase@google.com470e71d2011-07-07 08:21:25 +0000509 // Split into low and high band.
andrew@webrtc.org60730cf2014-01-07 17:45:09 +0000510 WebRtcSpl_AnalysisQMF(render_audio_->data(i),
511 render_audio_->samples_per_channel(),
512 render_audio_->low_pass_split_data(i),
513 render_audio_->high_pass_split_data(i),
514 render_audio_->analysis_filter_state1(i),
515 render_audio_->analysis_filter_state2(i));
niklase@google.com470e71d2011-07-07 08:21:25 +0000516 }
517 }
518
519 // TODO(ajm): warnings possible from components?
520 err = echo_cancellation_->ProcessRenderAudio(render_audio_);
521 if (err != kNoError) {
522 return err;
523 }
524
525 err = echo_control_mobile_->ProcessRenderAudio(render_audio_);
526 if (err != kNoError) {
527 return err;
528 }
529
530 err = gain_control_->ProcessRenderAudio(render_audio_);
531 if (err != kNoError) {
532 return err;
533 }
534
niklase@google.com470e71d2011-07-07 08:21:25 +0000535 return err; // TODO(ajm): this is for returning warnings; necessary?
536}
537
538int AudioProcessingImpl::set_stream_delay_ms(int delay) {
andrew@webrtc.org5f23d642012-05-29 21:14:06 +0000539 Error retval = kNoError;
niklase@google.com470e71d2011-07-07 08:21:25 +0000540 was_stream_delay_set_ = true;
andrew@webrtc.org6f9f8172012-03-06 19:03:39 +0000541 delay += delay_offset_ms_;
542
niklase@google.com470e71d2011-07-07 08:21:25 +0000543 if (delay < 0) {
andrew@webrtc.org5f23d642012-05-29 21:14:06 +0000544 delay = 0;
545 retval = kBadStreamParameterWarning;
niklase@google.com470e71d2011-07-07 08:21:25 +0000546 }
547
548 // TODO(ajm): the max is rather arbitrarily chosen; investigate.
549 if (delay > 500) {
andrew@webrtc.org5f23d642012-05-29 21:14:06 +0000550 delay = 500;
551 retval = kBadStreamParameterWarning;
niklase@google.com470e71d2011-07-07 08:21:25 +0000552 }
553
554 stream_delay_ms_ = delay;
andrew@webrtc.org5f23d642012-05-29 21:14:06 +0000555 return retval;
niklase@google.com470e71d2011-07-07 08:21:25 +0000556}
557
558int AudioProcessingImpl::stream_delay_ms() const {
559 return stream_delay_ms_;
560}
561
562bool AudioProcessingImpl::was_stream_delay_set() const {
563 return was_stream_delay_set_;
564}
565
andrew@webrtc.org6f9f8172012-03-06 19:03:39 +0000566void AudioProcessingImpl::set_delay_offset_ms(int offset) {
567 CriticalSectionScoped crit_scoped(crit_);
568 delay_offset_ms_ = offset;
569}
570
571int AudioProcessingImpl::delay_offset_ms() const {
572 return delay_offset_ms_;
573}
574
andrew@webrtc.org75dd2882014-02-11 20:52:30 +0000575void AudioProcessingImpl::set_stream_key_pressed(bool key_pressed) {
576 key_pressed_ = key_pressed;
577}
578
579bool AudioProcessingImpl::stream_key_pressed() const {
580 return key_pressed_;
581}
582
niklase@google.com470e71d2011-07-07 08:21:25 +0000583int AudioProcessingImpl::StartDebugRecording(
584 const char filename[AudioProcessing::kMaxFilenameSize]) {
andrew@webrtc.org40654032012-01-30 20:51:15 +0000585 CriticalSectionScoped crit_scoped(crit_);
niklase@google.com470e71d2011-07-07 08:21:25 +0000586 assert(kMaxFilenameSize == FileWrapper::kMaxFileNameSize);
587
588 if (filename == NULL) {
589 return kNullPointerError;
590 }
591
andrew@webrtc.org7bf26462011-12-03 00:03:31 +0000592#ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP
niklase@google.com470e71d2011-07-07 08:21:25 +0000593 // Stop any ongoing recording.
594 if (debug_file_->Open()) {
595 if (debug_file_->CloseFile() == -1) {
596 return kFileError;
597 }
598 }
599
600 if (debug_file_->OpenFile(filename, false) == -1) {
601 debug_file_->CloseFile();
602 return kFileError;
603 }
604
ajm@google.com808e0e02011-08-03 21:08:51 +0000605 int err = WriteInitMessage();
606 if (err != kNoError) {
607 return err;
niklase@google.com470e71d2011-07-07 08:21:25 +0000608 }
niklase@google.com470e71d2011-07-07 08:21:25 +0000609 return kNoError;
andrew@webrtc.org7bf26462011-12-03 00:03:31 +0000610#else
611 return kUnsupportedFunctionError;
612#endif // WEBRTC_AUDIOPROC_DEBUG_DUMP
niklase@google.com470e71d2011-07-07 08:21:25 +0000613}
614
henrikg@webrtc.org863b5362013-12-06 16:05:17 +0000615int AudioProcessingImpl::StartDebugRecording(FILE* handle) {
616 CriticalSectionScoped crit_scoped(crit_);
617
618 if (handle == NULL) {
619 return kNullPointerError;
620 }
621
622#ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP
623 // Stop any ongoing recording.
624 if (debug_file_->Open()) {
625 if (debug_file_->CloseFile() == -1) {
626 return kFileError;
627 }
628 }
629
630 if (debug_file_->OpenFromFileHandle(handle, true, false) == -1) {
631 return kFileError;
632 }
633
634 int err = WriteInitMessage();
635 if (err != kNoError) {
636 return err;
637 }
638 return kNoError;
639#else
640 return kUnsupportedFunctionError;
641#endif // WEBRTC_AUDIOPROC_DEBUG_DUMP
642}
643
niklase@google.com470e71d2011-07-07 08:21:25 +0000644int AudioProcessingImpl::StopDebugRecording() {
andrew@webrtc.org40654032012-01-30 20:51:15 +0000645 CriticalSectionScoped crit_scoped(crit_);
andrew@webrtc.org7bf26462011-12-03 00:03:31 +0000646
647#ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP
niklase@google.com470e71d2011-07-07 08:21:25 +0000648 // We just return if recording hasn't started.
649 if (debug_file_->Open()) {
650 if (debug_file_->CloseFile() == -1) {
651 return kFileError;
652 }
653 }
niklase@google.com470e71d2011-07-07 08:21:25 +0000654 return kNoError;
andrew@webrtc.org7bf26462011-12-03 00:03:31 +0000655#else
656 return kUnsupportedFunctionError;
657#endif // WEBRTC_AUDIOPROC_DEBUG_DUMP
niklase@google.com470e71d2011-07-07 08:21:25 +0000658}
659
660EchoCancellation* AudioProcessingImpl::echo_cancellation() const {
661 return echo_cancellation_;
662}
663
664EchoControlMobile* AudioProcessingImpl::echo_control_mobile() const {
665 return echo_control_mobile_;
666}
667
668GainControl* AudioProcessingImpl::gain_control() const {
669 return gain_control_;
670}
671
672HighPassFilter* AudioProcessingImpl::high_pass_filter() const {
673 return high_pass_filter_;
674}
675
676LevelEstimator* AudioProcessingImpl::level_estimator() const {
677 return level_estimator_;
678}
679
680NoiseSuppression* AudioProcessingImpl::noise_suppression() const {
681 return noise_suppression_;
682}
683
684VoiceDetection* AudioProcessingImpl::voice_detection() const {
685 return voice_detection_;
686}
687
andrew@webrtc.org369166a2012-04-24 18:38:03 +0000688bool AudioProcessingImpl::is_data_processed() const {
andrew@webrtc.org7bf26462011-12-03 00:03:31 +0000689 int enabled_count = 0;
690 std::list<ProcessingComponent*>::const_iterator it;
691 for (it = component_list_.begin(); it != component_list_.end(); it++) {
692 if ((*it)->is_component_enabled()) {
693 enabled_count++;
694 }
695 }
696
697 // Data is unchanged if no components are enabled, or if only level_estimator_
698 // or voice_detection_ is enabled.
699 if (enabled_count == 0) {
700 return false;
701 } else if (enabled_count == 1) {
702 if (level_estimator_->is_enabled() || voice_detection_->is_enabled()) {
703 return false;
704 }
705 } else if (enabled_count == 2) {
706 if (level_estimator_->is_enabled() && voice_detection_->is_enabled()) {
707 return false;
708 }
709 }
710 return true;
711}
712
andrew@webrtc.org369166a2012-04-24 18:38:03 +0000713bool AudioProcessingImpl::interleave_needed(bool is_data_processed) const {
714 // Check if we've upmixed or downmixed the audio.
715 return (num_output_channels_ != num_input_channels_ || is_data_processed);
andrew@webrtc.org7bf26462011-12-03 00:03:31 +0000716}
717
andrew@webrtc.org369166a2012-04-24 18:38:03 +0000718bool AudioProcessingImpl::synthesis_needed(bool is_data_processed) const {
719 return (is_data_processed && sample_rate_hz_ == kSampleRate32kHz);
720}
721
722bool AudioProcessingImpl::analysis_needed(bool is_data_processed) const {
723 if (!is_data_processed && !voice_detection_->is_enabled()) {
andrew@webrtc.org7bf26462011-12-03 00:03:31 +0000724 // Only level_estimator_ is enabled.
725 return false;
726 } else if (sample_rate_hz_ == kSampleRate32kHz) {
727 // Something besides level_estimator_ is enabled, and we have super-wb.
728 return true;
729 }
730 return false;
731}
732
733#ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP
ajm@google.com808e0e02011-08-03 21:08:51 +0000734int AudioProcessingImpl::WriteMessageToDebugFile() {
735 int32_t size = event_msg_->ByteSize();
736 if (size <= 0) {
737 return kUnspecifiedError;
738 }
andrew@webrtc.org621df672013-10-22 10:27:23 +0000739#if defined(WEBRTC_ARCH_BIG_ENDIAN)
ajm@google.com808e0e02011-08-03 21:08:51 +0000740 // TODO(ajm): Use little-endian "on the wire". For the moment, we can be
741 // pretty safe in assuming little-endian.
742#endif
743
744 if (!event_msg_->SerializeToString(&event_str_)) {
745 return kUnspecifiedError;
746 }
747
748 // Write message preceded by its size.
749 if (!debug_file_->Write(&size, sizeof(int32_t))) {
750 return kFileError;
751 }
752 if (!debug_file_->Write(event_str_.data(), event_str_.length())) {
753 return kFileError;
754 }
755
756 event_msg_->Clear();
757
758 return 0;
759}
760
761int AudioProcessingImpl::WriteInitMessage() {
762 event_msg_->set_type(audioproc::Event::INIT);
763 audioproc::Init* msg = event_msg_->mutable_init();
764 msg->set_sample_rate(sample_rate_hz_);
765 msg->set_device_sample_rate(echo_cancellation_->device_sample_rate_hz());
766 msg->set_num_input_channels(num_input_channels_);
767 msg->set_num_output_channels(num_output_channels_);
768 msg->set_num_reverse_channels(num_reverse_channels_);
769
770 int err = WriteMessageToDebugFile();
771 if (err != kNoError) {
772 return err;
773 }
774
775 return kNoError;
776}
andrew@webrtc.org7bf26462011-12-03 00:03:31 +0000777#endif // WEBRTC_AUDIOPROC_DEBUG_DUMP
niklase@google.com470e71d2011-07-07 08:21:25 +0000778} // namespace webrtc