blob: 4254551d6926289209963692e406cb6d98b91a76 [file] [log] [blame]
niklase@google.com470e71d2011-07-07 08:21:25 +00001/*
andrew@webrtc.org40654032012-01-30 20:51:15 +00002 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
niklase@google.com470e71d2011-07-07 08:21:25 +00003 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
andrew@webrtc.org78693fe2013-03-01 16:36:19 +000011#include "webrtc/modules/audio_processing/audio_processing_impl.h"
niklase@google.com470e71d2011-07-07 08:21:25 +000012
ajm@google.com808e0e02011-08-03 21:08:51 +000013#include <assert.h>
niklase@google.com470e71d2011-07-07 08:21:25 +000014
andrew@webrtc.org60730cf2014-01-07 17:45:09 +000015#include "webrtc/common_audio/signal_processing/include/signal_processing_library.h"
andrew@webrtc.org78693fe2013-03-01 16:36:19 +000016#include "webrtc/modules/audio_processing/audio_buffer.h"
andrew@webrtc.org61e596f2013-07-25 18:28:29 +000017#include "webrtc/modules/audio_processing/echo_cancellation_impl_wrapper.h"
andrew@webrtc.org78693fe2013-03-01 16:36:19 +000018#include "webrtc/modules/audio_processing/echo_control_mobile_impl.h"
19#include "webrtc/modules/audio_processing/gain_control_impl.h"
20#include "webrtc/modules/audio_processing/high_pass_filter_impl.h"
21#include "webrtc/modules/audio_processing/level_estimator_impl.h"
22#include "webrtc/modules/audio_processing/noise_suppression_impl.h"
23#include "webrtc/modules/audio_processing/processing_component.h"
andrew@webrtc.org78693fe2013-03-01 16:36:19 +000024#include "webrtc/modules/audio_processing/voice_detection_impl.h"
25#include "webrtc/modules/interface/module_common_types.h"
andrew@webrtc.org60730cf2014-01-07 17:45:09 +000026#include "webrtc/system_wrappers/interface/compile_assert.h"
andrew@webrtc.org78693fe2013-03-01 16:36:19 +000027#include "webrtc/system_wrappers/interface/critical_section_wrapper.h"
28#include "webrtc/system_wrappers/interface/file_wrapper.h"
29#include "webrtc/system_wrappers/interface/logging.h"
andrew@webrtc.org7bf26462011-12-03 00:03:31 +000030
31#ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP
32// Files generated at build-time by the protobuf compiler.
leozwang@webrtc.orga3736342012-03-16 21:36:00 +000033#ifdef WEBRTC_ANDROID_PLATFORM_BUILD
leozwang@webrtc.org534e4952012-10-22 21:21:52 +000034#include "external/webrtc/webrtc/modules/audio_processing/debug.pb.h"
leozwang@google.comce9bfbb2011-08-03 23:34:31 +000035#else
ajm@google.com808e0e02011-08-03 21:08:51 +000036#include "webrtc/audio_processing/debug.pb.h"
leozwang@google.comce9bfbb2011-08-03 23:34:31 +000037#endif
andrew@webrtc.org7bf26462011-12-03 00:03:31 +000038#endif // WEBRTC_AUDIOPROC_DEBUG_DUMP
niklase@google.com470e71d2011-07-07 08:21:25 +000039
andrew@webrtc.org60730cf2014-01-07 17:45:09 +000040static const int kChunkSizeMs = 10;
41
42#define RETURN_ON_ERR(expr) \
43 do { \
44 int err = expr; \
45 if (err != kNoError) { \
46 return err; \
47 } \
48 } while (0)
49
niklase@google.com470e71d2011-07-07 08:21:25 +000050namespace webrtc {
andrew@webrtc.org60730cf2014-01-07 17:45:09 +000051
52// Throughout webrtc, it's assumed that success is represented by zero.
53COMPILE_ASSERT(AudioProcessing::kNoError == 0, no_error_must_be_zero);
54
niklase@google.com470e71d2011-07-07 08:21:25 +000055AudioProcessing* AudioProcessing::Create(int id) {
andrew@webrtc.orge84978f2014-01-25 02:09:06 +000056 return Create();
57}
58
59AudioProcessing* AudioProcessing::Create() {
60 Config config;
61 return Create(config);
62}
63
64AudioProcessing* AudioProcessing::Create(const Config& config) {
65 AudioProcessingImpl* apm = new AudioProcessingImpl(config);
niklase@google.com470e71d2011-07-07 08:21:25 +000066 if (apm->Initialize() != kNoError) {
67 delete apm;
68 apm = NULL;
69 }
70
71 return apm;
72}
73
pbos@webrtc.org91620802013-08-02 11:44:11 +000074int32_t AudioProcessing::TimeUntilNextProcess() { return -1; }
75int32_t AudioProcessing::Process() { return -1; }
76
andrew@webrtc.orge84978f2014-01-25 02:09:06 +000077AudioProcessingImpl::AudioProcessingImpl(const Config& config)
andrew@webrtc.org60730cf2014-01-07 17:45:09 +000078 : echo_cancellation_(NULL),
niklase@google.com470e71d2011-07-07 08:21:25 +000079 echo_control_mobile_(NULL),
80 gain_control_(NULL),
81 high_pass_filter_(NULL),
82 level_estimator_(NULL),
83 noise_suppression_(NULL),
84 voice_detection_(NULL),
niklase@google.com470e71d2011-07-07 08:21:25 +000085 crit_(CriticalSectionWrapper::CreateCriticalSection()),
86 render_audio_(NULL),
87 capture_audio_(NULL),
andrew@webrtc.org7bf26462011-12-03 00:03:31 +000088#ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP
89 debug_file_(FileWrapper::Create()),
90 event_msg_(new audioproc::Event()),
91#endif
niklase@google.com470e71d2011-07-07 08:21:25 +000092 sample_rate_hz_(kSampleRate16kHz),
93 split_sample_rate_hz_(kSampleRate16kHz),
andrew@webrtc.org60730cf2014-01-07 17:45:09 +000094 samples_per_channel_(kChunkSizeMs * sample_rate_hz_ / 1000),
niklase@google.com470e71d2011-07-07 08:21:25 +000095 stream_delay_ms_(0),
andrew@webrtc.org6f9f8172012-03-06 19:03:39 +000096 delay_offset_ms_(0),
niklase@google.com470e71d2011-07-07 08:21:25 +000097 was_stream_delay_set_(false),
ajm@google.com808e0e02011-08-03 21:08:51 +000098 num_reverse_channels_(1),
99 num_input_channels_(1),
andrew@webrtc.org07b59502014-02-12 16:41:13 +0000100 num_output_channels_(1),
andrew@webrtc.org38bf2492014-02-13 17:43:44 +0000101 output_will_be_muted_(false),
andrew@webrtc.org07b59502014-02-12 16:41:13 +0000102 key_pressed_(false) {
andrew@webrtc.org61e596f2013-07-25 18:28:29 +0000103 echo_cancellation_ = EchoCancellationImplWrapper::Create(this);
niklase@google.com470e71d2011-07-07 08:21:25 +0000104 component_list_.push_back(echo_cancellation_);
105
106 echo_control_mobile_ = new EchoControlMobileImpl(this);
107 component_list_.push_back(echo_control_mobile_);
108
109 gain_control_ = new GainControlImpl(this);
110 component_list_.push_back(gain_control_);
111
112 high_pass_filter_ = new HighPassFilterImpl(this);
113 component_list_.push_back(high_pass_filter_);
114
115 level_estimator_ = new LevelEstimatorImpl(this);
116 component_list_.push_back(level_estimator_);
117
118 noise_suppression_ = new NoiseSuppressionImpl(this);
119 component_list_.push_back(noise_suppression_);
120
121 voice_detection_ = new VoiceDetectionImpl(this);
122 component_list_.push_back(voice_detection_);
andrew@webrtc.orge84978f2014-01-25 02:09:06 +0000123
124 SetExtraOptions(config);
niklase@google.com470e71d2011-07-07 08:21:25 +0000125}
126
127AudioProcessingImpl::~AudioProcessingImpl() {
andrew@webrtc.org81865342012-10-27 00:28:27 +0000128 {
129 CriticalSectionScoped crit_scoped(crit_);
130 while (!component_list_.empty()) {
131 ProcessingComponent* component = component_list_.front();
132 component->Destroy();
133 delete component;
134 component_list_.pop_front();
135 }
niklase@google.com470e71d2011-07-07 08:21:25 +0000136
andrew@webrtc.org7bf26462011-12-03 00:03:31 +0000137#ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP
andrew@webrtc.org81865342012-10-27 00:28:27 +0000138 if (debug_file_->Open()) {
139 debug_file_->CloseFile();
140 }
andrew@webrtc.org7bf26462011-12-03 00:03:31 +0000141#endif
ajm@google.com808e0e02011-08-03 21:08:51 +0000142
andrew@webrtc.org81865342012-10-27 00:28:27 +0000143 if (render_audio_) {
144 delete render_audio_;
145 render_audio_ = NULL;
146 }
147
148 if (capture_audio_) {
149 delete capture_audio_;
150 capture_audio_ = NULL;
151 }
niklase@google.com470e71d2011-07-07 08:21:25 +0000152 }
153
andrew@webrtc.org16cfbe22012-08-29 16:58:25 +0000154 delete crit_;
155 crit_ = NULL;
niklase@google.com470e71d2011-07-07 08:21:25 +0000156}
157
158CriticalSectionWrapper* AudioProcessingImpl::crit() const {
159 return crit_;
160}
161
162int AudioProcessingImpl::split_sample_rate_hz() const {
163 return split_sample_rate_hz_;
164}
165
166int AudioProcessingImpl::Initialize() {
andrew@webrtc.org40654032012-01-30 20:51:15 +0000167 CriticalSectionScoped crit_scoped(crit_);
niklase@google.com470e71d2011-07-07 08:21:25 +0000168 return InitializeLocked();
169}
170
171int AudioProcessingImpl::InitializeLocked() {
172 if (render_audio_ != NULL) {
173 delete render_audio_;
174 render_audio_ = NULL;
175 }
176
177 if (capture_audio_ != NULL) {
178 delete capture_audio_;
179 capture_audio_ = NULL;
180 }
181
ajm@google.com808e0e02011-08-03 21:08:51 +0000182 render_audio_ = new AudioBuffer(num_reverse_channels_,
niklase@google.com470e71d2011-07-07 08:21:25 +0000183 samples_per_channel_);
ajm@google.com808e0e02011-08-03 21:08:51 +0000184 capture_audio_ = new AudioBuffer(num_input_channels_,
niklase@google.com470e71d2011-07-07 08:21:25 +0000185 samples_per_channel_);
186
niklase@google.com470e71d2011-07-07 08:21:25 +0000187 // Initialize all components.
188 std::list<ProcessingComponent*>::iterator it;
andrew@webrtc.org81865342012-10-27 00:28:27 +0000189 for (it = component_list_.begin(); it != component_list_.end(); ++it) {
niklase@google.com470e71d2011-07-07 08:21:25 +0000190 int err = (*it)->Initialize();
191 if (err != kNoError) {
192 return err;
193 }
194 }
195
andrew@webrtc.org7bf26462011-12-03 00:03:31 +0000196#ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP
ajm@google.com808e0e02011-08-03 21:08:51 +0000197 if (debug_file_->Open()) {
198 int err = WriteInitMessage();
199 if (err != kNoError) {
200 return err;
201 }
202 }
andrew@webrtc.org7bf26462011-12-03 00:03:31 +0000203#endif
ajm@google.com808e0e02011-08-03 21:08:51 +0000204
niklase@google.com470e71d2011-07-07 08:21:25 +0000205 return kNoError;
206}
207
andrew@webrtc.org61e596f2013-07-25 18:28:29 +0000208void AudioProcessingImpl::SetExtraOptions(const Config& config) {
andrew@webrtc.orge84978f2014-01-25 02:09:06 +0000209 CriticalSectionScoped crit_scoped(crit_);
andrew@webrtc.org61e596f2013-07-25 18:28:29 +0000210 std::list<ProcessingComponent*>::iterator it;
211 for (it = component_list_.begin(); it != component_list_.end(); ++it)
212 (*it)->SetExtraOptions(config);
213}
214
aluebs@webrtc.org0b72f582013-11-19 15:17:51 +0000215int AudioProcessingImpl::EnableExperimentalNs(bool enable) {
216 return kNoError;
217}
218
niklase@google.com470e71d2011-07-07 08:21:25 +0000219int AudioProcessingImpl::set_sample_rate_hz(int rate) {
andrew@webrtc.org40654032012-01-30 20:51:15 +0000220 CriticalSectionScoped crit_scoped(crit_);
andrew@webrtc.org81865342012-10-27 00:28:27 +0000221 if (rate == sample_rate_hz_) {
222 return kNoError;
223 }
niklase@google.com470e71d2011-07-07 08:21:25 +0000224 if (rate != kSampleRate8kHz &&
225 rate != kSampleRate16kHz &&
226 rate != kSampleRate32kHz) {
227 return kBadParameterError;
228 }
andrew@webrtc.org78693fe2013-03-01 16:36:19 +0000229 if (echo_control_mobile_->is_enabled() && rate > kSampleRate16kHz) {
230 LOG(LS_ERROR) << "AECM only supports 16 kHz or lower sample rates";
231 return kUnsupportedComponentError;
232 }
niklase@google.com470e71d2011-07-07 08:21:25 +0000233
234 sample_rate_hz_ = rate;
235 samples_per_channel_ = rate / 100;
236
237 if (sample_rate_hz_ == kSampleRate32kHz) {
238 split_sample_rate_hz_ = kSampleRate16kHz;
239 } else {
240 split_sample_rate_hz_ = sample_rate_hz_;
241 }
242
243 return InitializeLocked();
244}
245
246int AudioProcessingImpl::sample_rate_hz() const {
henrika@webrtc.org19da7192013-04-05 14:34:57 +0000247 CriticalSectionScoped crit_scoped(crit_);
niklase@google.com470e71d2011-07-07 08:21:25 +0000248 return sample_rate_hz_;
249}
250
251int AudioProcessingImpl::set_num_reverse_channels(int channels) {
andrew@webrtc.org40654032012-01-30 20:51:15 +0000252 CriticalSectionScoped crit_scoped(crit_);
andrew@webrtc.org81865342012-10-27 00:28:27 +0000253 if (channels == num_reverse_channels_) {
254 return kNoError;
255 }
niklase@google.com470e71d2011-07-07 08:21:25 +0000256 // Only stereo supported currently.
257 if (channels > 2 || channels < 1) {
258 return kBadParameterError;
259 }
260
ajm@google.com808e0e02011-08-03 21:08:51 +0000261 num_reverse_channels_ = channels;
niklase@google.com470e71d2011-07-07 08:21:25 +0000262
263 return InitializeLocked();
264}
265
266int AudioProcessingImpl::num_reverse_channels() const {
ajm@google.com808e0e02011-08-03 21:08:51 +0000267 return num_reverse_channels_;
niklase@google.com470e71d2011-07-07 08:21:25 +0000268}
269
270int AudioProcessingImpl::set_num_channels(
271 int input_channels,
272 int output_channels) {
andrew@webrtc.org40654032012-01-30 20:51:15 +0000273 CriticalSectionScoped crit_scoped(crit_);
andrew@webrtc.org81865342012-10-27 00:28:27 +0000274 if (input_channels == num_input_channels_ &&
275 output_channels == num_output_channels_) {
276 return kNoError;
277 }
niklase@google.com470e71d2011-07-07 08:21:25 +0000278 if (output_channels > input_channels) {
279 return kBadParameterError;
280 }
niklase@google.com470e71d2011-07-07 08:21:25 +0000281 // Only stereo supported currently.
andrew@webrtc.org81865342012-10-27 00:28:27 +0000282 if (input_channels > 2 || input_channels < 1 ||
283 output_channels > 2 || output_channels < 1) {
niklase@google.com470e71d2011-07-07 08:21:25 +0000284 return kBadParameterError;
285 }
286
ajm@google.com808e0e02011-08-03 21:08:51 +0000287 num_input_channels_ = input_channels;
288 num_output_channels_ = output_channels;
niklase@google.com470e71d2011-07-07 08:21:25 +0000289
290 return InitializeLocked();
291}
292
293int AudioProcessingImpl::num_input_channels() const {
ajm@google.com808e0e02011-08-03 21:08:51 +0000294 return num_input_channels_;
niklase@google.com470e71d2011-07-07 08:21:25 +0000295}
296
297int AudioProcessingImpl::num_output_channels() const {
ajm@google.com808e0e02011-08-03 21:08:51 +0000298 return num_output_channels_;
niklase@google.com470e71d2011-07-07 08:21:25 +0000299}
300
andrew@webrtc.org17342e52014-02-12 22:28:31 +0000301void AudioProcessingImpl::set_output_will_be_muted(bool muted) {
302 output_will_be_muted_ = muted;
303}
304
305bool AudioProcessingImpl::output_will_be_muted() const {
306 return output_will_be_muted_;
307}
308
andrew@webrtc.org60730cf2014-01-07 17:45:09 +0000309int AudioProcessingImpl::MaybeInitializeLocked(int sample_rate_hz,
310 int num_input_channels, int num_output_channels, int num_reverse_channels) {
311 if (sample_rate_hz == sample_rate_hz_ &&
312 num_input_channels == num_input_channels_ &&
313 num_output_channels == num_output_channels_ &&
314 num_reverse_channels == num_reverse_channels_) {
315 return kNoError;
316 }
317
318 if (sample_rate_hz != kSampleRate8kHz &&
319 sample_rate_hz != kSampleRate16kHz &&
320 sample_rate_hz != kSampleRate32kHz) {
321 return kBadSampleRateError;
322 }
323 if (num_output_channels > num_input_channels) {
324 return kBadNumberChannelsError;
325 }
326 // Only mono and stereo supported currently.
327 if (num_input_channels > 2 || num_input_channels < 1 ||
328 num_output_channels > 2 || num_output_channels < 1 ||
329 num_reverse_channels > 2 || num_reverse_channels < 1) {
330 return kBadNumberChannelsError;
331 }
332 if (echo_control_mobile_->is_enabled() && sample_rate_hz > kSampleRate16kHz) {
333 LOG(LS_ERROR) << "AECM only supports 16 or 8 kHz sample rates";
334 return kUnsupportedComponentError;
335 }
336
337 sample_rate_hz_ = sample_rate_hz;
338 samples_per_channel_ = kChunkSizeMs * sample_rate_hz / 1000;
339 num_input_channels_ = num_input_channels;
340 num_output_channels_ = num_output_channels;
341 num_reverse_channels_ = num_reverse_channels;
342
343 if (sample_rate_hz_ == kSampleRate32kHz) {
344 split_sample_rate_hz_ = kSampleRate16kHz;
345 } else {
346 split_sample_rate_hz_ = sample_rate_hz_;
347 }
348
349 return InitializeLocked();
350}
351
niklase@google.com470e71d2011-07-07 08:21:25 +0000352int AudioProcessingImpl::ProcessStream(AudioFrame* frame) {
andrew@webrtc.org40654032012-01-30 20:51:15 +0000353 CriticalSectionScoped crit_scoped(crit_);
niklase@google.com470e71d2011-07-07 08:21:25 +0000354 int err = kNoError;
355
356 if (frame == NULL) {
357 return kNullPointerError;
358 }
andrew@webrtc.org60730cf2014-01-07 17:45:09 +0000359 // TODO(ajm): We now always set the output channels equal to the input
360 // channels here. Remove the ability to downmix entirely.
361 RETURN_ON_ERR(MaybeInitializeLocked(frame->sample_rate_hz_,
362 frame->num_channels_, frame->num_channels_, num_reverse_channels_));
andrew@webrtc.org63a50982012-05-02 23:56:37 +0000363 if (frame->samples_per_channel_ != samples_per_channel_) {
niklase@google.com470e71d2011-07-07 08:21:25 +0000364 return kBadDataLengthError;
365 }
366
andrew@webrtc.org7bf26462011-12-03 00:03:31 +0000367#ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP
niklase@google.com470e71d2011-07-07 08:21:25 +0000368 if (debug_file_->Open()) {
ajm@google.com808e0e02011-08-03 21:08:51 +0000369 event_msg_->set_type(audioproc::Event::STREAM);
370 audioproc::Stream* msg = event_msg_->mutable_stream();
andrew@webrtc.org755b04a2011-11-15 16:57:56 +0000371 const size_t data_size = sizeof(int16_t) *
andrew@webrtc.org63a50982012-05-02 23:56:37 +0000372 frame->samples_per_channel_ *
373 frame->num_channels_;
374 msg->set_input_data(frame->data_, data_size);
ajm@google.com808e0e02011-08-03 21:08:51 +0000375 msg->set_delay(stream_delay_ms_);
376 msg->set_drift(echo_cancellation_->stream_drift_samples());
377 msg->set_level(gain_control_->stream_analog_level());
andrew@webrtc.orgce8e0772014-02-12 15:28:30 +0000378 msg->set_keypress(key_pressed_);
niklase@google.com470e71d2011-07-07 08:21:25 +0000379 }
andrew@webrtc.org7bf26462011-12-03 00:03:31 +0000380#endif
niklase@google.com470e71d2011-07-07 08:21:25 +0000381
382 capture_audio_->DeinterleaveFrom(frame);
383
384 // TODO(ajm): experiment with mixing and AEC placement.
ajm@google.com808e0e02011-08-03 21:08:51 +0000385 if (num_output_channels_ < num_input_channels_) {
386 capture_audio_->Mix(num_output_channels_);
andrew@webrtc.org63a50982012-05-02 23:56:37 +0000387 frame->num_channels_ = num_output_channels_;
niklase@google.com470e71d2011-07-07 08:21:25 +0000388 }
389
andrew@webrtc.org369166a2012-04-24 18:38:03 +0000390 bool data_processed = is_data_processed();
391 if (analysis_needed(data_processed)) {
andrew@webrtc.org755b04a2011-11-15 16:57:56 +0000392 for (int i = 0; i < num_output_channels_; i++) {
niklase@google.com470e71d2011-07-07 08:21:25 +0000393 // Split into a low and high band.
andrew@webrtc.org60730cf2014-01-07 17:45:09 +0000394 WebRtcSpl_AnalysisQMF(capture_audio_->data(i),
395 capture_audio_->samples_per_channel(),
396 capture_audio_->low_pass_split_data(i),
397 capture_audio_->high_pass_split_data(i),
398 capture_audio_->analysis_filter_state1(i),
399 capture_audio_->analysis_filter_state2(i));
niklase@google.com470e71d2011-07-07 08:21:25 +0000400 }
401 }
402
403 err = high_pass_filter_->ProcessCaptureAudio(capture_audio_);
404 if (err != kNoError) {
405 return err;
406 }
407
408 err = gain_control_->AnalyzeCaptureAudio(capture_audio_);
409 if (err != kNoError) {
410 return err;
411 }
412
413 err = echo_cancellation_->ProcessCaptureAudio(capture_audio_);
414 if (err != kNoError) {
415 return err;
416 }
417
418 if (echo_control_mobile_->is_enabled() &&
419 noise_suppression_->is_enabled()) {
420 capture_audio_->CopyLowPassToReference();
421 }
422
423 err = noise_suppression_->ProcessCaptureAudio(capture_audio_);
424 if (err != kNoError) {
425 return err;
426 }
427
428 err = echo_control_mobile_->ProcessCaptureAudio(capture_audio_);
429 if (err != kNoError) {
430 return err;
431 }
432
433 err = voice_detection_->ProcessCaptureAudio(capture_audio_);
434 if (err != kNoError) {
435 return err;
436 }
437
438 err = gain_control_->ProcessCaptureAudio(capture_audio_);
439 if (err != kNoError) {
440 return err;
441 }
442
andrew@webrtc.org369166a2012-04-24 18:38:03 +0000443 if (synthesis_needed(data_processed)) {
ajm@google.com808e0e02011-08-03 21:08:51 +0000444 for (int i = 0; i < num_output_channels_; i++) {
niklase@google.com470e71d2011-07-07 08:21:25 +0000445 // Recombine low and high bands.
andrew@webrtc.org60730cf2014-01-07 17:45:09 +0000446 WebRtcSpl_SynthesisQMF(capture_audio_->low_pass_split_data(i),
447 capture_audio_->high_pass_split_data(i),
448 capture_audio_->samples_per_split_channel(),
449 capture_audio_->data(i),
450 capture_audio_->synthesis_filter_state1(i),
451 capture_audio_->synthesis_filter_state2(i));
niklase@google.com470e71d2011-07-07 08:21:25 +0000452 }
453 }
454
andrew@webrtc.org755b04a2011-11-15 16:57:56 +0000455 // The level estimator operates on the recombined data.
456 err = level_estimator_->ProcessStream(capture_audio_);
457 if (err != kNoError) {
458 return err;
459 }
460
andrew@webrtc.org369166a2012-04-24 18:38:03 +0000461 capture_audio_->InterleaveTo(frame, interleave_needed(data_processed));
niklase@google.com470e71d2011-07-07 08:21:25 +0000462
andrew@webrtc.org7bf26462011-12-03 00:03:31 +0000463#ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP
ajm@google.com808e0e02011-08-03 21:08:51 +0000464 if (debug_file_->Open()) {
465 audioproc::Stream* msg = event_msg_->mutable_stream();
andrew@webrtc.org755b04a2011-11-15 16:57:56 +0000466 const size_t data_size = sizeof(int16_t) *
andrew@webrtc.org63a50982012-05-02 23:56:37 +0000467 frame->samples_per_channel_ *
468 frame->num_channels_;
469 msg->set_output_data(frame->data_, data_size);
ajm@google.com808e0e02011-08-03 21:08:51 +0000470 err = WriteMessageToDebugFile();
471 if (err != kNoError) {
472 return err;
473 }
474 }
andrew@webrtc.org7bf26462011-12-03 00:03:31 +0000475#endif
ajm@google.com808e0e02011-08-03 21:08:51 +0000476
andrew@webrtc.org1e916932011-11-29 18:28:57 +0000477 was_stream_delay_set_ = false;
niklase@google.com470e71d2011-07-07 08:21:25 +0000478 return kNoError;
479}
480
andrew@webrtc.org60730cf2014-01-07 17:45:09 +0000481// TODO(ajm): Have AnalyzeReverseStream accept sample rates not matching the
482// primary stream and convert ourselves rather than having the user manage it.
483// We can be smarter and use the splitting filter when appropriate. Similarly,
484// perform downmixing here.
niklase@google.com470e71d2011-07-07 08:21:25 +0000485int AudioProcessingImpl::AnalyzeReverseStream(AudioFrame* frame) {
andrew@webrtc.org40654032012-01-30 20:51:15 +0000486 CriticalSectionScoped crit_scoped(crit_);
niklase@google.com470e71d2011-07-07 08:21:25 +0000487 int err = kNoError;
niklase@google.com470e71d2011-07-07 08:21:25 +0000488 if (frame == NULL) {
489 return kNullPointerError;
490 }
andrew@webrtc.org63a50982012-05-02 23:56:37 +0000491 if (frame->sample_rate_hz_ != sample_rate_hz_) {
niklase@google.com470e71d2011-07-07 08:21:25 +0000492 return kBadSampleRateError;
493 }
andrew@webrtc.org60730cf2014-01-07 17:45:09 +0000494 RETURN_ON_ERR(MaybeInitializeLocked(sample_rate_hz_, num_input_channels_,
495 num_output_channels_, frame->num_channels_));
niklase@google.com470e71d2011-07-07 08:21:25 +0000496
andrew@webrtc.org7bf26462011-12-03 00:03:31 +0000497#ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP
niklase@google.com470e71d2011-07-07 08:21:25 +0000498 if (debug_file_->Open()) {
ajm@google.com808e0e02011-08-03 21:08:51 +0000499 event_msg_->set_type(audioproc::Event::REVERSE_STREAM);
500 audioproc::ReverseStream* msg = event_msg_->mutable_reverse_stream();
andrew@webrtc.org755b04a2011-11-15 16:57:56 +0000501 const size_t data_size = sizeof(int16_t) *
andrew@webrtc.org63a50982012-05-02 23:56:37 +0000502 frame->samples_per_channel_ *
503 frame->num_channels_;
504 msg->set_data(frame->data_, data_size);
ajm@google.com808e0e02011-08-03 21:08:51 +0000505 err = WriteMessageToDebugFile();
506 if (err != kNoError) {
507 return err;
niklase@google.com470e71d2011-07-07 08:21:25 +0000508 }
509 }
andrew@webrtc.org7bf26462011-12-03 00:03:31 +0000510#endif
niklase@google.com470e71d2011-07-07 08:21:25 +0000511
512 render_audio_->DeinterleaveFrom(frame);
513
niklase@google.com470e71d2011-07-07 08:21:25 +0000514 if (sample_rate_hz_ == kSampleRate32kHz) {
ajm@google.com808e0e02011-08-03 21:08:51 +0000515 for (int i = 0; i < num_reverse_channels_; i++) {
niklase@google.com470e71d2011-07-07 08:21:25 +0000516 // Split into low and high band.
andrew@webrtc.org60730cf2014-01-07 17:45:09 +0000517 WebRtcSpl_AnalysisQMF(render_audio_->data(i),
518 render_audio_->samples_per_channel(),
519 render_audio_->low_pass_split_data(i),
520 render_audio_->high_pass_split_data(i),
521 render_audio_->analysis_filter_state1(i),
522 render_audio_->analysis_filter_state2(i));
niklase@google.com470e71d2011-07-07 08:21:25 +0000523 }
524 }
525
526 // TODO(ajm): warnings possible from components?
527 err = echo_cancellation_->ProcessRenderAudio(render_audio_);
528 if (err != kNoError) {
529 return err;
530 }
531
532 err = echo_control_mobile_->ProcessRenderAudio(render_audio_);
533 if (err != kNoError) {
534 return err;
535 }
536
537 err = gain_control_->ProcessRenderAudio(render_audio_);
538 if (err != kNoError) {
539 return err;
540 }
541
niklase@google.com470e71d2011-07-07 08:21:25 +0000542 return err; // TODO(ajm): this is for returning warnings; necessary?
543}
544
545int AudioProcessingImpl::set_stream_delay_ms(int delay) {
andrew@webrtc.org5f23d642012-05-29 21:14:06 +0000546 Error retval = kNoError;
niklase@google.com470e71d2011-07-07 08:21:25 +0000547 was_stream_delay_set_ = true;
andrew@webrtc.org6f9f8172012-03-06 19:03:39 +0000548 delay += delay_offset_ms_;
549
niklase@google.com470e71d2011-07-07 08:21:25 +0000550 if (delay < 0) {
andrew@webrtc.org5f23d642012-05-29 21:14:06 +0000551 delay = 0;
552 retval = kBadStreamParameterWarning;
niklase@google.com470e71d2011-07-07 08:21:25 +0000553 }
554
555 // TODO(ajm): the max is rather arbitrarily chosen; investigate.
556 if (delay > 500) {
andrew@webrtc.org5f23d642012-05-29 21:14:06 +0000557 delay = 500;
558 retval = kBadStreamParameterWarning;
niklase@google.com470e71d2011-07-07 08:21:25 +0000559 }
560
561 stream_delay_ms_ = delay;
andrew@webrtc.org5f23d642012-05-29 21:14:06 +0000562 return retval;
niklase@google.com470e71d2011-07-07 08:21:25 +0000563}
564
565int AudioProcessingImpl::stream_delay_ms() const {
566 return stream_delay_ms_;
567}
568
569bool AudioProcessingImpl::was_stream_delay_set() const {
570 return was_stream_delay_set_;
571}
572
andrew@webrtc.org6f9f8172012-03-06 19:03:39 +0000573void AudioProcessingImpl::set_delay_offset_ms(int offset) {
574 CriticalSectionScoped crit_scoped(crit_);
575 delay_offset_ms_ = offset;
576}
577
578int AudioProcessingImpl::delay_offset_ms() const {
579 return delay_offset_ms_;
580}
581
andrew@webrtc.org75dd2882014-02-11 20:52:30 +0000582void AudioProcessingImpl::set_stream_key_pressed(bool key_pressed) {
583 key_pressed_ = key_pressed;
584}
585
586bool AudioProcessingImpl::stream_key_pressed() const {
587 return key_pressed_;
588}
589
niklase@google.com470e71d2011-07-07 08:21:25 +0000590int AudioProcessingImpl::StartDebugRecording(
591 const char filename[AudioProcessing::kMaxFilenameSize]) {
andrew@webrtc.org40654032012-01-30 20:51:15 +0000592 CriticalSectionScoped crit_scoped(crit_);
niklase@google.com470e71d2011-07-07 08:21:25 +0000593 assert(kMaxFilenameSize == FileWrapper::kMaxFileNameSize);
594
595 if (filename == NULL) {
596 return kNullPointerError;
597 }
598
andrew@webrtc.org7bf26462011-12-03 00:03:31 +0000599#ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP
niklase@google.com470e71d2011-07-07 08:21:25 +0000600 // Stop any ongoing recording.
601 if (debug_file_->Open()) {
602 if (debug_file_->CloseFile() == -1) {
603 return kFileError;
604 }
605 }
606
607 if (debug_file_->OpenFile(filename, false) == -1) {
608 debug_file_->CloseFile();
609 return kFileError;
610 }
611
ajm@google.com808e0e02011-08-03 21:08:51 +0000612 int err = WriteInitMessage();
613 if (err != kNoError) {
614 return err;
niklase@google.com470e71d2011-07-07 08:21:25 +0000615 }
niklase@google.com470e71d2011-07-07 08:21:25 +0000616 return kNoError;
andrew@webrtc.org7bf26462011-12-03 00:03:31 +0000617#else
618 return kUnsupportedFunctionError;
619#endif // WEBRTC_AUDIOPROC_DEBUG_DUMP
niklase@google.com470e71d2011-07-07 08:21:25 +0000620}
621
henrikg@webrtc.org863b5362013-12-06 16:05:17 +0000622int AudioProcessingImpl::StartDebugRecording(FILE* handle) {
623 CriticalSectionScoped crit_scoped(crit_);
624
625 if (handle == NULL) {
626 return kNullPointerError;
627 }
628
629#ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP
630 // Stop any ongoing recording.
631 if (debug_file_->Open()) {
632 if (debug_file_->CloseFile() == -1) {
633 return kFileError;
634 }
635 }
636
637 if (debug_file_->OpenFromFileHandle(handle, true, false) == -1) {
638 return kFileError;
639 }
640
641 int err = WriteInitMessage();
642 if (err != kNoError) {
643 return err;
644 }
645 return kNoError;
646#else
647 return kUnsupportedFunctionError;
648#endif // WEBRTC_AUDIOPROC_DEBUG_DUMP
649}
650
niklase@google.com470e71d2011-07-07 08:21:25 +0000651int AudioProcessingImpl::StopDebugRecording() {
andrew@webrtc.org40654032012-01-30 20:51:15 +0000652 CriticalSectionScoped crit_scoped(crit_);
andrew@webrtc.org7bf26462011-12-03 00:03:31 +0000653
654#ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP
niklase@google.com470e71d2011-07-07 08:21:25 +0000655 // We just return if recording hasn't started.
656 if (debug_file_->Open()) {
657 if (debug_file_->CloseFile() == -1) {
658 return kFileError;
659 }
660 }
niklase@google.com470e71d2011-07-07 08:21:25 +0000661 return kNoError;
andrew@webrtc.org7bf26462011-12-03 00:03:31 +0000662#else
663 return kUnsupportedFunctionError;
664#endif // WEBRTC_AUDIOPROC_DEBUG_DUMP
niklase@google.com470e71d2011-07-07 08:21:25 +0000665}
666
667EchoCancellation* AudioProcessingImpl::echo_cancellation() const {
668 return echo_cancellation_;
669}
670
671EchoControlMobile* AudioProcessingImpl::echo_control_mobile() const {
672 return echo_control_mobile_;
673}
674
675GainControl* AudioProcessingImpl::gain_control() const {
676 return gain_control_;
677}
678
679HighPassFilter* AudioProcessingImpl::high_pass_filter() const {
680 return high_pass_filter_;
681}
682
683LevelEstimator* AudioProcessingImpl::level_estimator() const {
684 return level_estimator_;
685}
686
687NoiseSuppression* AudioProcessingImpl::noise_suppression() const {
688 return noise_suppression_;
689}
690
691VoiceDetection* AudioProcessingImpl::voice_detection() const {
692 return voice_detection_;
693}
694
pbos@webrtc.orgb7192b82013-04-10 07:50:54 +0000695int32_t AudioProcessingImpl::ChangeUniqueId(const int32_t id) {
niklase@google.com470e71d2011-07-07 08:21:25 +0000696 return kNoError;
697}
ajm@google.com808e0e02011-08-03 21:08:51 +0000698
andrew@webrtc.org369166a2012-04-24 18:38:03 +0000699bool AudioProcessingImpl::is_data_processed() const {
andrew@webrtc.org7bf26462011-12-03 00:03:31 +0000700 int enabled_count = 0;
701 std::list<ProcessingComponent*>::const_iterator it;
702 for (it = component_list_.begin(); it != component_list_.end(); it++) {
703 if ((*it)->is_component_enabled()) {
704 enabled_count++;
705 }
706 }
707
708 // Data is unchanged if no components are enabled, or if only level_estimator_
709 // or voice_detection_ is enabled.
710 if (enabled_count == 0) {
711 return false;
712 } else if (enabled_count == 1) {
713 if (level_estimator_->is_enabled() || voice_detection_->is_enabled()) {
714 return false;
715 }
716 } else if (enabled_count == 2) {
717 if (level_estimator_->is_enabled() && voice_detection_->is_enabled()) {
718 return false;
719 }
720 }
721 return true;
722}
723
andrew@webrtc.org369166a2012-04-24 18:38:03 +0000724bool AudioProcessingImpl::interleave_needed(bool is_data_processed) const {
725 // Check if we've upmixed or downmixed the audio.
726 return (num_output_channels_ != num_input_channels_ || is_data_processed);
andrew@webrtc.org7bf26462011-12-03 00:03:31 +0000727}
728
andrew@webrtc.org369166a2012-04-24 18:38:03 +0000729bool AudioProcessingImpl::synthesis_needed(bool is_data_processed) const {
730 return (is_data_processed && sample_rate_hz_ == kSampleRate32kHz);
731}
732
733bool AudioProcessingImpl::analysis_needed(bool is_data_processed) const {
734 if (!is_data_processed && !voice_detection_->is_enabled()) {
andrew@webrtc.org7bf26462011-12-03 00:03:31 +0000735 // Only level_estimator_ is enabled.
736 return false;
737 } else if (sample_rate_hz_ == kSampleRate32kHz) {
738 // Something besides level_estimator_ is enabled, and we have super-wb.
739 return true;
740 }
741 return false;
742}
743
744#ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP
ajm@google.com808e0e02011-08-03 21:08:51 +0000745int AudioProcessingImpl::WriteMessageToDebugFile() {
746 int32_t size = event_msg_->ByteSize();
747 if (size <= 0) {
748 return kUnspecifiedError;
749 }
andrew@webrtc.org621df672013-10-22 10:27:23 +0000750#if defined(WEBRTC_ARCH_BIG_ENDIAN)
ajm@google.com808e0e02011-08-03 21:08:51 +0000751 // TODO(ajm): Use little-endian "on the wire". For the moment, we can be
752 // pretty safe in assuming little-endian.
753#endif
754
755 if (!event_msg_->SerializeToString(&event_str_)) {
756 return kUnspecifiedError;
757 }
758
759 // Write message preceded by its size.
760 if (!debug_file_->Write(&size, sizeof(int32_t))) {
761 return kFileError;
762 }
763 if (!debug_file_->Write(event_str_.data(), event_str_.length())) {
764 return kFileError;
765 }
766
767 event_msg_->Clear();
768
769 return 0;
770}
771
772int AudioProcessingImpl::WriteInitMessage() {
773 event_msg_->set_type(audioproc::Event::INIT);
774 audioproc::Init* msg = event_msg_->mutable_init();
775 msg->set_sample_rate(sample_rate_hz_);
776 msg->set_device_sample_rate(echo_cancellation_->device_sample_rate_hz());
777 msg->set_num_input_channels(num_input_channels_);
778 msg->set_num_output_channels(num_output_channels_);
779 msg->set_num_reverse_channels(num_reverse_channels_);
780
781 int err = WriteMessageToDebugFile();
782 if (err != kNoError) {
783 return err;
784 }
785
786 return kNoError;
787}
andrew@webrtc.org7bf26462011-12-03 00:03:31 +0000788#endif // WEBRTC_AUDIOPROC_DEBUG_DUMP
niklase@google.com470e71d2011-07-07 08:21:25 +0000789} // namespace webrtc