blob: 272c786d1c464bc9ff1bffbac3feb5d1f63e06b8 [file] [log] [blame]
niklase@google.com470e71d2011-07-07 08:21:25 +00001/*
andrew@webrtc.org40654032012-01-30 20:51:15 +00002 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
niklase@google.com470e71d2011-07-07 08:21:25 +00003 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
andrew@webrtc.org78693fe2013-03-01 16:36:19 +000011#include "webrtc/modules/audio_processing/audio_processing_impl.h"
niklase@google.com470e71d2011-07-07 08:21:25 +000012
ajm@google.com808e0e02011-08-03 21:08:51 +000013#include <assert.h>
niklase@google.com470e71d2011-07-07 08:21:25 +000014
andrew@webrtc.org17e40642014-03-04 20:58:13 +000015#include "webrtc/common_audio/include/audio_util.h"
andrew@webrtc.org60730cf2014-01-07 17:45:09 +000016#include "webrtc/common_audio/signal_processing/include/signal_processing_library.h"
andrew@webrtc.org78693fe2013-03-01 16:36:19 +000017#include "webrtc/modules/audio_processing/audio_buffer.h"
andrew@webrtc.org56e4a052014-02-27 22:23:17 +000018#include "webrtc/modules/audio_processing/echo_cancellation_impl.h"
andrew@webrtc.org78693fe2013-03-01 16:36:19 +000019#include "webrtc/modules/audio_processing/echo_control_mobile_impl.h"
20#include "webrtc/modules/audio_processing/gain_control_impl.h"
21#include "webrtc/modules/audio_processing/high_pass_filter_impl.h"
22#include "webrtc/modules/audio_processing/level_estimator_impl.h"
23#include "webrtc/modules/audio_processing/noise_suppression_impl.h"
24#include "webrtc/modules/audio_processing/processing_component.h"
andrew@webrtc.org78693fe2013-03-01 16:36:19 +000025#include "webrtc/modules/audio_processing/voice_detection_impl.h"
26#include "webrtc/modules/interface/module_common_types.h"
andrew@webrtc.org60730cf2014-01-07 17:45:09 +000027#include "webrtc/system_wrappers/interface/compile_assert.h"
andrew@webrtc.org78693fe2013-03-01 16:36:19 +000028#include "webrtc/system_wrappers/interface/critical_section_wrapper.h"
29#include "webrtc/system_wrappers/interface/file_wrapper.h"
30#include "webrtc/system_wrappers/interface/logging.h"
andrew@webrtc.org7bf26462011-12-03 00:03:31 +000031
32#ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP
33// Files generated at build-time by the protobuf compiler.
leozwang@webrtc.orga3736342012-03-16 21:36:00 +000034#ifdef WEBRTC_ANDROID_PLATFORM_BUILD
leozwang@webrtc.org534e4952012-10-22 21:21:52 +000035#include "external/webrtc/webrtc/modules/audio_processing/debug.pb.h"
leozwang@google.comce9bfbb2011-08-03 23:34:31 +000036#else
ajm@google.com808e0e02011-08-03 21:08:51 +000037#include "webrtc/audio_processing/debug.pb.h"
leozwang@google.comce9bfbb2011-08-03 23:34:31 +000038#endif
andrew@webrtc.org7bf26462011-12-03 00:03:31 +000039#endif // WEBRTC_AUDIOPROC_DEBUG_DUMP
niklase@google.com470e71d2011-07-07 08:21:25 +000040
andrew@webrtc.org60730cf2014-01-07 17:45:09 +000041#define RETURN_ON_ERR(expr) \
42 do { \
43 int err = expr; \
44 if (err != kNoError) { \
45 return err; \
46 } \
47 } while (0)
48
niklase@google.com470e71d2011-07-07 08:21:25 +000049namespace webrtc {
andrew@webrtc.org17e40642014-03-04 20:58:13 +000050namespace {
51
52const int kChunkSizeMs = 10;
53
54int ChannelsFromLayout(AudioProcessing::ChannelLayout layout) {
55 switch (layout) {
56 case AudioProcessing::kMono:
57 case AudioProcessing::kMonoAndKeyboard:
58 return 1;
59 case AudioProcessing::kStereo:
60 case AudioProcessing::kStereoAndKeyboard:
61 return 2;
62 }
63 assert(false);
64 return -1;
65}
66
67} // namespace
andrew@webrtc.org60730cf2014-01-07 17:45:09 +000068
69// Throughout webrtc, it's assumed that success is represented by zero.
70COMPILE_ASSERT(AudioProcessing::kNoError == 0, no_error_must_be_zero);
71
niklase@google.com470e71d2011-07-07 08:21:25 +000072AudioProcessing* AudioProcessing::Create(int id) {
andrew@webrtc.orge84978f2014-01-25 02:09:06 +000073 return Create();
74}
75
76AudioProcessing* AudioProcessing::Create() {
77 Config config;
78 return Create(config);
79}
80
81AudioProcessing* AudioProcessing::Create(const Config& config) {
82 AudioProcessingImpl* apm = new AudioProcessingImpl(config);
niklase@google.com470e71d2011-07-07 08:21:25 +000083 if (apm->Initialize() != kNoError) {
84 delete apm;
85 apm = NULL;
86 }
87
88 return apm;
89}
90
andrew@webrtc.orge84978f2014-01-25 02:09:06 +000091AudioProcessingImpl::AudioProcessingImpl(const Config& config)
andrew@webrtc.org60730cf2014-01-07 17:45:09 +000092 : echo_cancellation_(NULL),
niklase@google.com470e71d2011-07-07 08:21:25 +000093 echo_control_mobile_(NULL),
94 gain_control_(NULL),
95 high_pass_filter_(NULL),
96 level_estimator_(NULL),
97 noise_suppression_(NULL),
98 voice_detection_(NULL),
niklase@google.com470e71d2011-07-07 08:21:25 +000099 crit_(CriticalSectionWrapper::CreateCriticalSection()),
100 render_audio_(NULL),
101 capture_audio_(NULL),
andrew@webrtc.org7bf26462011-12-03 00:03:31 +0000102#ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP
103 debug_file_(FileWrapper::Create()),
104 event_msg_(new audioproc::Event()),
105#endif
niklase@google.com470e71d2011-07-07 08:21:25 +0000106 sample_rate_hz_(kSampleRate16kHz),
107 split_sample_rate_hz_(kSampleRate16kHz),
andrew@webrtc.org60730cf2014-01-07 17:45:09 +0000108 samples_per_channel_(kChunkSizeMs * sample_rate_hz_ / 1000),
niklase@google.com470e71d2011-07-07 08:21:25 +0000109 stream_delay_ms_(0),
andrew@webrtc.org6f9f8172012-03-06 19:03:39 +0000110 delay_offset_ms_(0),
niklase@google.com470e71d2011-07-07 08:21:25 +0000111 was_stream_delay_set_(false),
ajm@google.com808e0e02011-08-03 21:08:51 +0000112 num_reverse_channels_(1),
113 num_input_channels_(1),
andrew@webrtc.org07b59502014-02-12 16:41:13 +0000114 num_output_channels_(1),
andrew@webrtc.org38bf2492014-02-13 17:43:44 +0000115 output_will_be_muted_(false),
andrew@webrtc.org07b59502014-02-12 16:41:13 +0000116 key_pressed_(false) {
andrew@webrtc.org56e4a052014-02-27 22:23:17 +0000117 echo_cancellation_ = new EchoCancellationImpl(this, crit_);
niklase@google.com470e71d2011-07-07 08:21:25 +0000118 component_list_.push_back(echo_cancellation_);
119
andrew@webrtc.org56e4a052014-02-27 22:23:17 +0000120 echo_control_mobile_ = new EchoControlMobileImpl(this, crit_);
niklase@google.com470e71d2011-07-07 08:21:25 +0000121 component_list_.push_back(echo_control_mobile_);
122
andrew@webrtc.org56e4a052014-02-27 22:23:17 +0000123 gain_control_ = new GainControlImpl(this, crit_);
niklase@google.com470e71d2011-07-07 08:21:25 +0000124 component_list_.push_back(gain_control_);
125
andrew@webrtc.org56e4a052014-02-27 22:23:17 +0000126 high_pass_filter_ = new HighPassFilterImpl(this, crit_);
niklase@google.com470e71d2011-07-07 08:21:25 +0000127 component_list_.push_back(high_pass_filter_);
128
andrew@webrtc.org56e4a052014-02-27 22:23:17 +0000129 level_estimator_ = new LevelEstimatorImpl(this, crit_);
niklase@google.com470e71d2011-07-07 08:21:25 +0000130 component_list_.push_back(level_estimator_);
131
andrew@webrtc.org56e4a052014-02-27 22:23:17 +0000132 noise_suppression_ = new NoiseSuppressionImpl(this, crit_);
niklase@google.com470e71d2011-07-07 08:21:25 +0000133 component_list_.push_back(noise_suppression_);
134
andrew@webrtc.org56e4a052014-02-27 22:23:17 +0000135 voice_detection_ = new VoiceDetectionImpl(this, crit_);
niklase@google.com470e71d2011-07-07 08:21:25 +0000136 component_list_.push_back(voice_detection_);
andrew@webrtc.orge84978f2014-01-25 02:09:06 +0000137
138 SetExtraOptions(config);
niklase@google.com470e71d2011-07-07 08:21:25 +0000139}
140
141AudioProcessingImpl::~AudioProcessingImpl() {
andrew@webrtc.org81865342012-10-27 00:28:27 +0000142 {
143 CriticalSectionScoped crit_scoped(crit_);
144 while (!component_list_.empty()) {
145 ProcessingComponent* component = component_list_.front();
146 component->Destroy();
147 delete component;
148 component_list_.pop_front();
149 }
niklase@google.com470e71d2011-07-07 08:21:25 +0000150
andrew@webrtc.org7bf26462011-12-03 00:03:31 +0000151#ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP
andrew@webrtc.org81865342012-10-27 00:28:27 +0000152 if (debug_file_->Open()) {
153 debug_file_->CloseFile();
154 }
andrew@webrtc.org7bf26462011-12-03 00:03:31 +0000155#endif
ajm@google.com808e0e02011-08-03 21:08:51 +0000156
andrew@webrtc.org81865342012-10-27 00:28:27 +0000157 if (render_audio_) {
158 delete render_audio_;
159 render_audio_ = NULL;
160 }
161
162 if (capture_audio_) {
163 delete capture_audio_;
164 capture_audio_ = NULL;
165 }
niklase@google.com470e71d2011-07-07 08:21:25 +0000166 }
167
andrew@webrtc.org16cfbe22012-08-29 16:58:25 +0000168 delete crit_;
169 crit_ = NULL;
niklase@google.com470e71d2011-07-07 08:21:25 +0000170}
171
niklase@google.com470e71d2011-07-07 08:21:25 +0000172int AudioProcessingImpl::split_sample_rate_hz() const {
173 return split_sample_rate_hz_;
174}
175
176int AudioProcessingImpl::Initialize() {
andrew@webrtc.org40654032012-01-30 20:51:15 +0000177 CriticalSectionScoped crit_scoped(crit_);
niklase@google.com470e71d2011-07-07 08:21:25 +0000178 return InitializeLocked();
179}
180
181int AudioProcessingImpl::InitializeLocked() {
182 if (render_audio_ != NULL) {
183 delete render_audio_;
184 render_audio_ = NULL;
185 }
186
187 if (capture_audio_ != NULL) {
188 delete capture_audio_;
189 capture_audio_ = NULL;
190 }
191
ajm@google.com808e0e02011-08-03 21:08:51 +0000192 render_audio_ = new AudioBuffer(num_reverse_channels_,
niklase@google.com470e71d2011-07-07 08:21:25 +0000193 samples_per_channel_);
ajm@google.com808e0e02011-08-03 21:08:51 +0000194 capture_audio_ = new AudioBuffer(num_input_channels_,
niklase@google.com470e71d2011-07-07 08:21:25 +0000195 samples_per_channel_);
196
niklase@google.com470e71d2011-07-07 08:21:25 +0000197 // Initialize all components.
198 std::list<ProcessingComponent*>::iterator it;
andrew@webrtc.org81865342012-10-27 00:28:27 +0000199 for (it = component_list_.begin(); it != component_list_.end(); ++it) {
niklase@google.com470e71d2011-07-07 08:21:25 +0000200 int err = (*it)->Initialize();
201 if (err != kNoError) {
202 return err;
203 }
204 }
205
andrew@webrtc.org7bf26462011-12-03 00:03:31 +0000206#ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP
ajm@google.com808e0e02011-08-03 21:08:51 +0000207 if (debug_file_->Open()) {
208 int err = WriteInitMessage();
209 if (err != kNoError) {
210 return err;
211 }
212 }
andrew@webrtc.org7bf26462011-12-03 00:03:31 +0000213#endif
ajm@google.com808e0e02011-08-03 21:08:51 +0000214
niklase@google.com470e71d2011-07-07 08:21:25 +0000215 return kNoError;
216}
217
andrew@webrtc.org61e596f2013-07-25 18:28:29 +0000218void AudioProcessingImpl::SetExtraOptions(const Config& config) {
andrew@webrtc.orge84978f2014-01-25 02:09:06 +0000219 CriticalSectionScoped crit_scoped(crit_);
andrew@webrtc.org61e596f2013-07-25 18:28:29 +0000220 std::list<ProcessingComponent*>::iterator it;
221 for (it = component_list_.begin(); it != component_list_.end(); ++it)
222 (*it)->SetExtraOptions(config);
223}
224
aluebs@webrtc.org0b72f582013-11-19 15:17:51 +0000225int AudioProcessingImpl::EnableExperimentalNs(bool enable) {
226 return kNoError;
227}
228
niklase@google.com470e71d2011-07-07 08:21:25 +0000229int AudioProcessingImpl::set_sample_rate_hz(int rate) {
andrew@webrtc.org40654032012-01-30 20:51:15 +0000230 CriticalSectionScoped crit_scoped(crit_);
andrew@webrtc.org81865342012-10-27 00:28:27 +0000231 if (rate == sample_rate_hz_) {
232 return kNoError;
233 }
niklase@google.com470e71d2011-07-07 08:21:25 +0000234 if (rate != kSampleRate8kHz &&
235 rate != kSampleRate16kHz &&
236 rate != kSampleRate32kHz) {
237 return kBadParameterError;
238 }
andrew@webrtc.org78693fe2013-03-01 16:36:19 +0000239 if (echo_control_mobile_->is_enabled() && rate > kSampleRate16kHz) {
240 LOG(LS_ERROR) << "AECM only supports 16 kHz or lower sample rates";
241 return kUnsupportedComponentError;
242 }
niklase@google.com470e71d2011-07-07 08:21:25 +0000243
244 sample_rate_hz_ = rate;
245 samples_per_channel_ = rate / 100;
246
247 if (sample_rate_hz_ == kSampleRate32kHz) {
248 split_sample_rate_hz_ = kSampleRate16kHz;
249 } else {
250 split_sample_rate_hz_ = sample_rate_hz_;
251 }
252
253 return InitializeLocked();
254}
255
256int AudioProcessingImpl::sample_rate_hz() const {
henrika@webrtc.org19da7192013-04-05 14:34:57 +0000257 CriticalSectionScoped crit_scoped(crit_);
niklase@google.com470e71d2011-07-07 08:21:25 +0000258 return sample_rate_hz_;
259}
260
261int AudioProcessingImpl::set_num_reverse_channels(int channels) {
andrew@webrtc.org40654032012-01-30 20:51:15 +0000262 CriticalSectionScoped crit_scoped(crit_);
andrew@webrtc.org81865342012-10-27 00:28:27 +0000263 if (channels == num_reverse_channels_) {
264 return kNoError;
265 }
niklase@google.com470e71d2011-07-07 08:21:25 +0000266 // Only stereo supported currently.
267 if (channels > 2 || channels < 1) {
268 return kBadParameterError;
269 }
270
ajm@google.com808e0e02011-08-03 21:08:51 +0000271 num_reverse_channels_ = channels;
niklase@google.com470e71d2011-07-07 08:21:25 +0000272
273 return InitializeLocked();
274}
275
276int AudioProcessingImpl::num_reverse_channels() const {
ajm@google.com808e0e02011-08-03 21:08:51 +0000277 return num_reverse_channels_;
niklase@google.com470e71d2011-07-07 08:21:25 +0000278}
279
280int AudioProcessingImpl::set_num_channels(
281 int input_channels,
282 int output_channels) {
andrew@webrtc.org40654032012-01-30 20:51:15 +0000283 CriticalSectionScoped crit_scoped(crit_);
andrew@webrtc.org81865342012-10-27 00:28:27 +0000284 if (input_channels == num_input_channels_ &&
285 output_channels == num_output_channels_) {
286 return kNoError;
287 }
niklase@google.com470e71d2011-07-07 08:21:25 +0000288 if (output_channels > input_channels) {
289 return kBadParameterError;
290 }
niklase@google.com470e71d2011-07-07 08:21:25 +0000291 // Only stereo supported currently.
andrew@webrtc.org81865342012-10-27 00:28:27 +0000292 if (input_channels > 2 || input_channels < 1 ||
293 output_channels > 2 || output_channels < 1) {
niklase@google.com470e71d2011-07-07 08:21:25 +0000294 return kBadParameterError;
295 }
296
ajm@google.com808e0e02011-08-03 21:08:51 +0000297 num_input_channels_ = input_channels;
298 num_output_channels_ = output_channels;
niklase@google.com470e71d2011-07-07 08:21:25 +0000299
300 return InitializeLocked();
301}
302
303int AudioProcessingImpl::num_input_channels() const {
ajm@google.com808e0e02011-08-03 21:08:51 +0000304 return num_input_channels_;
niklase@google.com470e71d2011-07-07 08:21:25 +0000305}
306
307int AudioProcessingImpl::num_output_channels() const {
ajm@google.com808e0e02011-08-03 21:08:51 +0000308 return num_output_channels_;
niklase@google.com470e71d2011-07-07 08:21:25 +0000309}
310
andrew@webrtc.org17342e52014-02-12 22:28:31 +0000311void AudioProcessingImpl::set_output_will_be_muted(bool muted) {
312 output_will_be_muted_ = muted;
313}
314
315bool AudioProcessingImpl::output_will_be_muted() const {
316 return output_will_be_muted_;
317}
318
andrew@webrtc.org17e40642014-03-04 20:58:13 +0000319// Calls InitializeLocked() if any of the audio parameters have changed from
320// their current values.
andrew@webrtc.org60730cf2014-01-07 17:45:09 +0000321int AudioProcessingImpl::MaybeInitializeLocked(int sample_rate_hz,
322 int num_input_channels, int num_output_channels, int num_reverse_channels) {
323 if (sample_rate_hz == sample_rate_hz_ &&
324 num_input_channels == num_input_channels_ &&
325 num_output_channels == num_output_channels_ &&
326 num_reverse_channels == num_reverse_channels_) {
327 return kNoError;
328 }
329
330 if (sample_rate_hz != kSampleRate8kHz &&
331 sample_rate_hz != kSampleRate16kHz &&
332 sample_rate_hz != kSampleRate32kHz) {
333 return kBadSampleRateError;
334 }
335 if (num_output_channels > num_input_channels) {
336 return kBadNumberChannelsError;
337 }
338 // Only mono and stereo supported currently.
339 if (num_input_channels > 2 || num_input_channels < 1 ||
340 num_output_channels > 2 || num_output_channels < 1 ||
341 num_reverse_channels > 2 || num_reverse_channels < 1) {
342 return kBadNumberChannelsError;
343 }
344 if (echo_control_mobile_->is_enabled() && sample_rate_hz > kSampleRate16kHz) {
345 LOG(LS_ERROR) << "AECM only supports 16 or 8 kHz sample rates";
346 return kUnsupportedComponentError;
347 }
348
349 sample_rate_hz_ = sample_rate_hz;
350 samples_per_channel_ = kChunkSizeMs * sample_rate_hz / 1000;
351 num_input_channels_ = num_input_channels;
352 num_output_channels_ = num_output_channels;
353 num_reverse_channels_ = num_reverse_channels;
354
355 if (sample_rate_hz_ == kSampleRate32kHz) {
356 split_sample_rate_hz_ = kSampleRate16kHz;
357 } else {
358 split_sample_rate_hz_ = sample_rate_hz_;
359 }
360
361 return InitializeLocked();
362}
363
andrew@webrtc.org17e40642014-03-04 20:58:13 +0000364int AudioProcessingImpl::ProcessStream(float* const* data,
365 int samples_per_channel,
366 int sample_rate_hz,
367 ChannelLayout input_layout,
368 ChannelLayout output_layout) {
andrew@webrtc.org40654032012-01-30 20:51:15 +0000369 CriticalSectionScoped crit_scoped(crit_);
andrew@webrtc.org17e40642014-03-04 20:58:13 +0000370 if (!data) {
niklase@google.com470e71d2011-07-07 08:21:25 +0000371 return kNullPointerError;
372 }
andrew@webrtc.org17e40642014-03-04 20:58:13 +0000373
374 const int num_input_channels = ChannelsFromLayout(input_layout);
andrew@webrtc.org60730cf2014-01-07 17:45:09 +0000375 // TODO(ajm): We now always set the output channels equal to the input
andrew@webrtc.org17e40642014-03-04 20:58:13 +0000376 // channels here. Restore the ability to downmix.
377 RETURN_ON_ERR(MaybeInitializeLocked(sample_rate_hz,
378 num_input_channels, num_input_channels, num_reverse_channels_));
379 if (samples_per_channel != samples_per_channel_) {
380 return kBadDataLengthError;
381 }
382
383#ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP
384 if (debug_file_->Open()) {
385 event_msg_->set_type(audioproc::Event::STREAM);
386 audioproc::Stream* msg = event_msg_->mutable_stream();
387 const size_t channel_size = sizeof(float) * samples_per_channel;
388 for (int i = 0; i < num_input_channels; ++i)
389 msg->set_input_channel(i, data[i], channel_size);
390 }
391#endif
392
393 capture_audio_->CopyFrom(data, samples_per_channel, num_output_channels_);
394 RETURN_ON_ERR(ProcessStreamLocked());
395 if (output_copy_needed(is_data_processed())) {
396 capture_audio_->CopyTo(samples_per_channel, num_output_channels_, data);
397 }
398
399#ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP
400 if (debug_file_->Open()) {
401 audioproc::Stream* msg = event_msg_->mutable_stream();
402 const size_t channel_size = sizeof(float) * samples_per_channel;
403 for (int i = 0; i < num_output_channels_; ++i)
404 msg->set_output_channel(i, data[i], channel_size);
405 RETURN_ON_ERR(WriteMessageToDebugFile());
406 }
407#endif
408
409 return kNoError;
410}
411
412int AudioProcessingImpl::ProcessStream(AudioFrame* frame) {
413 CriticalSectionScoped crit_scoped(crit_);
414 if (!frame) {
415 return kNullPointerError;
416 }
417
418 // TODO(ajm): We now always set the output channels equal to the input
419 // channels here. Restore the ability to downmix.
andrew@webrtc.org60730cf2014-01-07 17:45:09 +0000420 RETURN_ON_ERR(MaybeInitializeLocked(frame->sample_rate_hz_,
421 frame->num_channels_, frame->num_channels_, num_reverse_channels_));
andrew@webrtc.org63a50982012-05-02 23:56:37 +0000422 if (frame->samples_per_channel_ != samples_per_channel_) {
niklase@google.com470e71d2011-07-07 08:21:25 +0000423 return kBadDataLengthError;
424 }
425
andrew@webrtc.org7bf26462011-12-03 00:03:31 +0000426#ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP
niklase@google.com470e71d2011-07-07 08:21:25 +0000427 if (debug_file_->Open()) {
ajm@google.com808e0e02011-08-03 21:08:51 +0000428 event_msg_->set_type(audioproc::Event::STREAM);
429 audioproc::Stream* msg = event_msg_->mutable_stream();
andrew@webrtc.org755b04a2011-11-15 16:57:56 +0000430 const size_t data_size = sizeof(int16_t) *
andrew@webrtc.org63a50982012-05-02 23:56:37 +0000431 frame->samples_per_channel_ *
432 frame->num_channels_;
433 msg->set_input_data(frame->data_, data_size);
andrew@webrtc.org17e40642014-03-04 20:58:13 +0000434 }
435#endif
436
437 capture_audio_->DeinterleaveFrom(frame);
438 if (num_output_channels_ < num_input_channels_) {
439 capture_audio_->Mix(num_output_channels_);
440 frame->num_channels_ = num_output_channels_;
441 }
442 RETURN_ON_ERR(ProcessStreamLocked());
443 capture_audio_->InterleaveTo(frame, output_copy_needed(is_data_processed()));
444
445#ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP
446 if (debug_file_->Open()) {
447 audioproc::Stream* msg = event_msg_->mutable_stream();
448 const size_t data_size = sizeof(int16_t) *
449 frame->samples_per_channel_ *
450 frame->num_channels_;
451 msg->set_output_data(frame->data_, data_size);
452 RETURN_ON_ERR(WriteMessageToDebugFile());
453 }
454#endif
455
456 return kNoError;
457}
458
459
460int AudioProcessingImpl::ProcessStreamLocked() {
461#ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP
462 if (debug_file_->Open()) {
463 audioproc::Stream* msg = event_msg_->mutable_stream();
ajm@google.com808e0e02011-08-03 21:08:51 +0000464 msg->set_delay(stream_delay_ms_);
465 msg->set_drift(echo_cancellation_->stream_drift_samples());
466 msg->set_level(gain_control_->stream_analog_level());
andrew@webrtc.orgce8e0772014-02-12 15:28:30 +0000467 msg->set_keypress(key_pressed_);
niklase@google.com470e71d2011-07-07 08:21:25 +0000468 }
andrew@webrtc.org7bf26462011-12-03 00:03:31 +0000469#endif
niklase@google.com470e71d2011-07-07 08:21:25 +0000470
andrew@webrtc.org369166a2012-04-24 18:38:03 +0000471 bool data_processed = is_data_processed();
472 if (analysis_needed(data_processed)) {
andrew@webrtc.org755b04a2011-11-15 16:57:56 +0000473 for (int i = 0; i < num_output_channels_; i++) {
niklase@google.com470e71d2011-07-07 08:21:25 +0000474 // Split into a low and high band.
andrew@webrtc.org60730cf2014-01-07 17:45:09 +0000475 WebRtcSpl_AnalysisQMF(capture_audio_->data(i),
476 capture_audio_->samples_per_channel(),
477 capture_audio_->low_pass_split_data(i),
478 capture_audio_->high_pass_split_data(i),
479 capture_audio_->analysis_filter_state1(i),
480 capture_audio_->analysis_filter_state2(i));
niklase@google.com470e71d2011-07-07 08:21:25 +0000481 }
482 }
483
andrew@webrtc.org17e40642014-03-04 20:58:13 +0000484 RETURN_ON_ERR(high_pass_filter_->ProcessCaptureAudio(capture_audio_));
485 RETURN_ON_ERR(gain_control_->AnalyzeCaptureAudio(capture_audio_));
486 RETURN_ON_ERR(echo_cancellation_->ProcessCaptureAudio(capture_audio_));
niklase@google.com470e71d2011-07-07 08:21:25 +0000487
488 if (echo_control_mobile_->is_enabled() &&
489 noise_suppression_->is_enabled()) {
490 capture_audio_->CopyLowPassToReference();
491 }
andrew@webrtc.org17e40642014-03-04 20:58:13 +0000492 RETURN_ON_ERR(noise_suppression_->ProcessCaptureAudio(capture_audio_));
493 RETURN_ON_ERR(echo_control_mobile_->ProcessCaptureAudio(capture_audio_));
494 RETURN_ON_ERR(voice_detection_->ProcessCaptureAudio(capture_audio_));
495 RETURN_ON_ERR(gain_control_->ProcessCaptureAudio(capture_audio_));
niklase@google.com470e71d2011-07-07 08:21:25 +0000496
andrew@webrtc.org369166a2012-04-24 18:38:03 +0000497 if (synthesis_needed(data_processed)) {
ajm@google.com808e0e02011-08-03 21:08:51 +0000498 for (int i = 0; i < num_output_channels_; i++) {
niklase@google.com470e71d2011-07-07 08:21:25 +0000499 // Recombine low and high bands.
andrew@webrtc.org60730cf2014-01-07 17:45:09 +0000500 WebRtcSpl_SynthesisQMF(capture_audio_->low_pass_split_data(i),
501 capture_audio_->high_pass_split_data(i),
502 capture_audio_->samples_per_split_channel(),
503 capture_audio_->data(i),
504 capture_audio_->synthesis_filter_state1(i),
505 capture_audio_->synthesis_filter_state2(i));
niklase@google.com470e71d2011-07-07 08:21:25 +0000506 }
507 }
508
andrew@webrtc.org755b04a2011-11-15 16:57:56 +0000509 // The level estimator operates on the recombined data.
andrew@webrtc.org17e40642014-03-04 20:58:13 +0000510 RETURN_ON_ERR(level_estimator_->ProcessStream(capture_audio_));
ajm@google.com808e0e02011-08-03 21:08:51 +0000511
andrew@webrtc.org1e916932011-11-29 18:28:57 +0000512 was_stream_delay_set_ = false;
niklase@google.com470e71d2011-07-07 08:21:25 +0000513 return kNoError;
514}
515
andrew@webrtc.org17e40642014-03-04 20:58:13 +0000516int AudioProcessingImpl::AnalyzeReverseStream(const float* const* data,
517 int samples_per_channel,
518 int sample_rate_hz,
519 ChannelLayout layout) {
520 CriticalSectionScoped crit_scoped(crit_);
521 if (data == NULL) {
522 return kNullPointerError;
523 }
524 if (sample_rate_hz != sample_rate_hz_) {
525 return kBadSampleRateError;
526 }
527
528 const int num_channels = ChannelsFromLayout(layout);
529 RETURN_ON_ERR(MaybeInitializeLocked(sample_rate_hz_, num_input_channels_,
530 num_output_channels_, num_channels));
531 if (samples_per_channel != samples_per_channel_) {
532 return kBadDataLengthError;
533 }
534
535#ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP
536 if (debug_file_->Open()) {
537 event_msg_->set_type(audioproc::Event::REVERSE_STREAM);
538 audioproc::ReverseStream* msg = event_msg_->mutable_reverse_stream();
539 const size_t channel_size = sizeof(float) * samples_per_channel;
540 for (int i = 0; i < num_channels; ++i)
541 msg->set_channel(i, data[i], channel_size);
542 RETURN_ON_ERR(WriteMessageToDebugFile());
543 }
544#endif
545
546 render_audio_->CopyFrom(data, samples_per_channel, num_channels);
547 return AnalyzeReverseStreamLocked();
548}
549
niklase@google.com470e71d2011-07-07 08:21:25 +0000550int AudioProcessingImpl::AnalyzeReverseStream(AudioFrame* frame) {
andrew@webrtc.org40654032012-01-30 20:51:15 +0000551 CriticalSectionScoped crit_scoped(crit_);
niklase@google.com470e71d2011-07-07 08:21:25 +0000552 if (frame == NULL) {
553 return kNullPointerError;
554 }
andrew@webrtc.org63a50982012-05-02 23:56:37 +0000555 if (frame->sample_rate_hz_ != sample_rate_hz_) {
niklase@google.com470e71d2011-07-07 08:21:25 +0000556 return kBadSampleRateError;
557 }
andrew@webrtc.org60730cf2014-01-07 17:45:09 +0000558 RETURN_ON_ERR(MaybeInitializeLocked(sample_rate_hz_, num_input_channels_,
559 num_output_channels_, frame->num_channels_));
andrew@webrtc.org17e40642014-03-04 20:58:13 +0000560 if (frame->samples_per_channel_ != samples_per_channel_) {
561 return kBadDataLengthError;
562 }
niklase@google.com470e71d2011-07-07 08:21:25 +0000563
andrew@webrtc.org7bf26462011-12-03 00:03:31 +0000564#ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP
niklase@google.com470e71d2011-07-07 08:21:25 +0000565 if (debug_file_->Open()) {
ajm@google.com808e0e02011-08-03 21:08:51 +0000566 event_msg_->set_type(audioproc::Event::REVERSE_STREAM);
567 audioproc::ReverseStream* msg = event_msg_->mutable_reverse_stream();
andrew@webrtc.org755b04a2011-11-15 16:57:56 +0000568 const size_t data_size = sizeof(int16_t) *
andrew@webrtc.org63a50982012-05-02 23:56:37 +0000569 frame->samples_per_channel_ *
570 frame->num_channels_;
571 msg->set_data(frame->data_, data_size);
andrew@webrtc.org17e40642014-03-04 20:58:13 +0000572 RETURN_ON_ERR(WriteMessageToDebugFile());
niklase@google.com470e71d2011-07-07 08:21:25 +0000573 }
andrew@webrtc.org7bf26462011-12-03 00:03:31 +0000574#endif
niklase@google.com470e71d2011-07-07 08:21:25 +0000575
576 render_audio_->DeinterleaveFrom(frame);
andrew@webrtc.org17e40642014-03-04 20:58:13 +0000577 return AnalyzeReverseStreamLocked();
578}
niklase@google.com470e71d2011-07-07 08:21:25 +0000579
andrew@webrtc.org17e40642014-03-04 20:58:13 +0000580// TODO(ajm): Have AnalyzeReverseStream accept sample rates not matching the
581// primary stream and convert ourselves rather than having the user manage it.
582// We can be smarter and use the splitting filter when appropriate. Similarly,
583// perform downmixing here.
584int AudioProcessingImpl::AnalyzeReverseStreamLocked() {
niklase@google.com470e71d2011-07-07 08:21:25 +0000585 if (sample_rate_hz_ == kSampleRate32kHz) {
ajm@google.com808e0e02011-08-03 21:08:51 +0000586 for (int i = 0; i < num_reverse_channels_; i++) {
niklase@google.com470e71d2011-07-07 08:21:25 +0000587 // Split into low and high band.
andrew@webrtc.org60730cf2014-01-07 17:45:09 +0000588 WebRtcSpl_AnalysisQMF(render_audio_->data(i),
589 render_audio_->samples_per_channel(),
590 render_audio_->low_pass_split_data(i),
591 render_audio_->high_pass_split_data(i),
592 render_audio_->analysis_filter_state1(i),
593 render_audio_->analysis_filter_state2(i));
niklase@google.com470e71d2011-07-07 08:21:25 +0000594 }
595 }
596
andrew@webrtc.org17e40642014-03-04 20:58:13 +0000597 RETURN_ON_ERR(echo_cancellation_->ProcessRenderAudio(render_audio_));
598 RETURN_ON_ERR(echo_control_mobile_->ProcessRenderAudio(render_audio_));
599 RETURN_ON_ERR(gain_control_->ProcessRenderAudio(render_audio_));
niklase@google.com470e71d2011-07-07 08:21:25 +0000600
andrew@webrtc.org17e40642014-03-04 20:58:13 +0000601 return kNoError;
niklase@google.com470e71d2011-07-07 08:21:25 +0000602}
603
604int AudioProcessingImpl::set_stream_delay_ms(int delay) {
andrew@webrtc.org5f23d642012-05-29 21:14:06 +0000605 Error retval = kNoError;
niklase@google.com470e71d2011-07-07 08:21:25 +0000606 was_stream_delay_set_ = true;
andrew@webrtc.org6f9f8172012-03-06 19:03:39 +0000607 delay += delay_offset_ms_;
608
niklase@google.com470e71d2011-07-07 08:21:25 +0000609 if (delay < 0) {
andrew@webrtc.org5f23d642012-05-29 21:14:06 +0000610 delay = 0;
611 retval = kBadStreamParameterWarning;
niklase@google.com470e71d2011-07-07 08:21:25 +0000612 }
613
614 // TODO(ajm): the max is rather arbitrarily chosen; investigate.
615 if (delay > 500) {
andrew@webrtc.org5f23d642012-05-29 21:14:06 +0000616 delay = 500;
617 retval = kBadStreamParameterWarning;
niklase@google.com470e71d2011-07-07 08:21:25 +0000618 }
619
620 stream_delay_ms_ = delay;
andrew@webrtc.org5f23d642012-05-29 21:14:06 +0000621 return retval;
niklase@google.com470e71d2011-07-07 08:21:25 +0000622}
623
624int AudioProcessingImpl::stream_delay_ms() const {
625 return stream_delay_ms_;
626}
627
628bool AudioProcessingImpl::was_stream_delay_set() const {
629 return was_stream_delay_set_;
630}
631
andrew@webrtc.org17e40642014-03-04 20:58:13 +0000632void AudioProcessingImpl::set_stream_key_pressed(bool key_pressed) {
633 key_pressed_ = key_pressed;
634}
635
636bool AudioProcessingImpl::stream_key_pressed() const {
637 return key_pressed_;
638}
639
andrew@webrtc.org6f9f8172012-03-06 19:03:39 +0000640void AudioProcessingImpl::set_delay_offset_ms(int offset) {
641 CriticalSectionScoped crit_scoped(crit_);
642 delay_offset_ms_ = offset;
643}
644
645int AudioProcessingImpl::delay_offset_ms() const {
646 return delay_offset_ms_;
647}
648
niklase@google.com470e71d2011-07-07 08:21:25 +0000649int AudioProcessingImpl::StartDebugRecording(
650 const char filename[AudioProcessing::kMaxFilenameSize]) {
andrew@webrtc.org40654032012-01-30 20:51:15 +0000651 CriticalSectionScoped crit_scoped(crit_);
niklase@google.com470e71d2011-07-07 08:21:25 +0000652 assert(kMaxFilenameSize == FileWrapper::kMaxFileNameSize);
653
654 if (filename == NULL) {
655 return kNullPointerError;
656 }
657
andrew@webrtc.org7bf26462011-12-03 00:03:31 +0000658#ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP
niklase@google.com470e71d2011-07-07 08:21:25 +0000659 // Stop any ongoing recording.
660 if (debug_file_->Open()) {
661 if (debug_file_->CloseFile() == -1) {
662 return kFileError;
663 }
664 }
665
666 if (debug_file_->OpenFile(filename, false) == -1) {
667 debug_file_->CloseFile();
668 return kFileError;
669 }
670
ajm@google.com808e0e02011-08-03 21:08:51 +0000671 int err = WriteInitMessage();
672 if (err != kNoError) {
673 return err;
niklase@google.com470e71d2011-07-07 08:21:25 +0000674 }
niklase@google.com470e71d2011-07-07 08:21:25 +0000675 return kNoError;
andrew@webrtc.org7bf26462011-12-03 00:03:31 +0000676#else
677 return kUnsupportedFunctionError;
678#endif // WEBRTC_AUDIOPROC_DEBUG_DUMP
niklase@google.com470e71d2011-07-07 08:21:25 +0000679}
680
henrikg@webrtc.org863b5362013-12-06 16:05:17 +0000681int AudioProcessingImpl::StartDebugRecording(FILE* handle) {
682 CriticalSectionScoped crit_scoped(crit_);
683
684 if (handle == NULL) {
685 return kNullPointerError;
686 }
687
688#ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP
689 // Stop any ongoing recording.
690 if (debug_file_->Open()) {
691 if (debug_file_->CloseFile() == -1) {
692 return kFileError;
693 }
694 }
695
696 if (debug_file_->OpenFromFileHandle(handle, true, false) == -1) {
697 return kFileError;
698 }
699
700 int err = WriteInitMessage();
701 if (err != kNoError) {
702 return err;
703 }
704 return kNoError;
705#else
706 return kUnsupportedFunctionError;
707#endif // WEBRTC_AUDIOPROC_DEBUG_DUMP
708}
709
niklase@google.com470e71d2011-07-07 08:21:25 +0000710int AudioProcessingImpl::StopDebugRecording() {
andrew@webrtc.org40654032012-01-30 20:51:15 +0000711 CriticalSectionScoped crit_scoped(crit_);
andrew@webrtc.org7bf26462011-12-03 00:03:31 +0000712
713#ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP
niklase@google.com470e71d2011-07-07 08:21:25 +0000714 // We just return if recording hasn't started.
715 if (debug_file_->Open()) {
716 if (debug_file_->CloseFile() == -1) {
717 return kFileError;
718 }
719 }
niklase@google.com470e71d2011-07-07 08:21:25 +0000720 return kNoError;
andrew@webrtc.org7bf26462011-12-03 00:03:31 +0000721#else
722 return kUnsupportedFunctionError;
723#endif // WEBRTC_AUDIOPROC_DEBUG_DUMP
niklase@google.com470e71d2011-07-07 08:21:25 +0000724}
725
726EchoCancellation* AudioProcessingImpl::echo_cancellation() const {
727 return echo_cancellation_;
728}
729
730EchoControlMobile* AudioProcessingImpl::echo_control_mobile() const {
731 return echo_control_mobile_;
732}
733
734GainControl* AudioProcessingImpl::gain_control() const {
735 return gain_control_;
736}
737
738HighPassFilter* AudioProcessingImpl::high_pass_filter() const {
739 return high_pass_filter_;
740}
741
742LevelEstimator* AudioProcessingImpl::level_estimator() const {
743 return level_estimator_;
744}
745
746NoiseSuppression* AudioProcessingImpl::noise_suppression() const {
747 return noise_suppression_;
748}
749
750VoiceDetection* AudioProcessingImpl::voice_detection() const {
751 return voice_detection_;
752}
753
andrew@webrtc.org369166a2012-04-24 18:38:03 +0000754bool AudioProcessingImpl::is_data_processed() const {
andrew@webrtc.org7bf26462011-12-03 00:03:31 +0000755 int enabled_count = 0;
756 std::list<ProcessingComponent*>::const_iterator it;
757 for (it = component_list_.begin(); it != component_list_.end(); it++) {
758 if ((*it)->is_component_enabled()) {
759 enabled_count++;
760 }
761 }
762
763 // Data is unchanged if no components are enabled, or if only level_estimator_
764 // or voice_detection_ is enabled.
765 if (enabled_count == 0) {
766 return false;
767 } else if (enabled_count == 1) {
768 if (level_estimator_->is_enabled() || voice_detection_->is_enabled()) {
769 return false;
770 }
771 } else if (enabled_count == 2) {
772 if (level_estimator_->is_enabled() && voice_detection_->is_enabled()) {
773 return false;
774 }
775 }
776 return true;
777}
778
andrew@webrtc.org17e40642014-03-04 20:58:13 +0000779bool AudioProcessingImpl::output_copy_needed(bool is_data_processed) const {
andrew@webrtc.org369166a2012-04-24 18:38:03 +0000780 // Check if we've upmixed or downmixed the audio.
781 return (num_output_channels_ != num_input_channels_ || is_data_processed);
andrew@webrtc.org7bf26462011-12-03 00:03:31 +0000782}
783
andrew@webrtc.org369166a2012-04-24 18:38:03 +0000784bool AudioProcessingImpl::synthesis_needed(bool is_data_processed) const {
785 return (is_data_processed && sample_rate_hz_ == kSampleRate32kHz);
786}
787
788bool AudioProcessingImpl::analysis_needed(bool is_data_processed) const {
789 if (!is_data_processed && !voice_detection_->is_enabled()) {
andrew@webrtc.org7bf26462011-12-03 00:03:31 +0000790 // Only level_estimator_ is enabled.
791 return false;
792 } else if (sample_rate_hz_ == kSampleRate32kHz) {
793 // Something besides level_estimator_ is enabled, and we have super-wb.
794 return true;
795 }
796 return false;
797}
798
799#ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP
ajm@google.com808e0e02011-08-03 21:08:51 +0000800int AudioProcessingImpl::WriteMessageToDebugFile() {
801 int32_t size = event_msg_->ByteSize();
802 if (size <= 0) {
803 return kUnspecifiedError;
804 }
andrew@webrtc.org621df672013-10-22 10:27:23 +0000805#if defined(WEBRTC_ARCH_BIG_ENDIAN)
ajm@google.com808e0e02011-08-03 21:08:51 +0000806 // TODO(ajm): Use little-endian "on the wire". For the moment, we can be
807 // pretty safe in assuming little-endian.
808#endif
809
810 if (!event_msg_->SerializeToString(&event_str_)) {
811 return kUnspecifiedError;
812 }
813
814 // Write message preceded by its size.
815 if (!debug_file_->Write(&size, sizeof(int32_t))) {
816 return kFileError;
817 }
818 if (!debug_file_->Write(event_str_.data(), event_str_.length())) {
819 return kFileError;
820 }
821
822 event_msg_->Clear();
823
andrew@webrtc.org17e40642014-03-04 20:58:13 +0000824 return kNoError;
ajm@google.com808e0e02011-08-03 21:08:51 +0000825}
826
827int AudioProcessingImpl::WriteInitMessage() {
828 event_msg_->set_type(audioproc::Event::INIT);
829 audioproc::Init* msg = event_msg_->mutable_init();
830 msg->set_sample_rate(sample_rate_hz_);
831 msg->set_device_sample_rate(echo_cancellation_->device_sample_rate_hz());
832 msg->set_num_input_channels(num_input_channels_);
833 msg->set_num_output_channels(num_output_channels_);
834 msg->set_num_reverse_channels(num_reverse_channels_);
835
836 int err = WriteMessageToDebugFile();
837 if (err != kNoError) {
838 return err;
839 }
840
841 return kNoError;
842}
andrew@webrtc.org7bf26462011-12-03 00:03:31 +0000843#endif // WEBRTC_AUDIOPROC_DEBUG_DUMP
niklase@google.com470e71d2011-07-07 08:21:25 +0000844} // namespace webrtc