blob: 011325f8ca1cc00a525d01e11ef296db4ecddf52 [file] [log] [blame]
niklase@google.com470e71d2011-07-07 08:21:25 +00001/*
andrew@webrtc.org40654032012-01-30 20:51:15 +00002 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
niklase@google.com470e71d2011-07-07 08:21:25 +00003 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
andrew@webrtc.org78693fe2013-03-01 16:36:19 +000011#include "webrtc/modules/audio_processing/audio_processing_impl.h"
niklase@google.com470e71d2011-07-07 08:21:25 +000012
ajm@google.com808e0e02011-08-03 21:08:51 +000013#include <assert.h>
Michael Graczyk86c6d332015-07-23 11:41:39 -070014#include <algorithm>
niklase@google.com470e71d2011-07-07 08:21:25 +000015
Bjorn Volcker1ca324f2015-06-29 14:57:29 +020016#include "webrtc/base/checks.h"
xians@webrtc.orge46bc772014-10-10 08:36:56 +000017#include "webrtc/base/platform_file.h"
peah369f8282015-12-17 06:42:29 -080018#include "webrtc/base/trace_event.h"
ekmeyerson60d9b332015-08-14 10:35:55 -070019#include "webrtc/common_audio/audio_converter.h"
Michael Graczykdfa36052015-03-25 16:37:27 -070020#include "webrtc/common_audio/channel_buffer.h"
ekmeyerson60d9b332015-08-14 10:35:55 -070021#include "webrtc/common_audio/include/audio_util.h"
andrew@webrtc.org60730cf2014-01-07 17:45:09 +000022#include "webrtc/common_audio/signal_processing/include/signal_processing_library.h"
Bjorn Volcker1ca324f2015-06-29 14:57:29 +020023#include "webrtc/modules/audio_processing/aec/aec_core.h"
pbos@webrtc.org788acd12014-12-15 09:41:24 +000024#include "webrtc/modules/audio_processing/agc/agc_manager_direct.h"
andrew@webrtc.org78693fe2013-03-01 16:36:19 +000025#include "webrtc/modules/audio_processing/audio_buffer.h"
mgraczyk@chromium.org0f663de2015-03-13 00:13:32 +000026#include "webrtc/modules/audio_processing/beamformer/nonlinear_beamformer.h"
andrew@webrtc.orgddbb8a22014-04-22 21:00:04 +000027#include "webrtc/modules/audio_processing/common.h"
andrew@webrtc.org56e4a052014-02-27 22:23:17 +000028#include "webrtc/modules/audio_processing/echo_cancellation_impl.h"
andrew@webrtc.org78693fe2013-03-01 16:36:19 +000029#include "webrtc/modules/audio_processing/echo_control_mobile_impl.h"
peahbe615622016-02-13 16:40:47 -080030#include "webrtc/modules/audio_processing/gain_control_for_experimental_agc.h"
andrew@webrtc.org78693fe2013-03-01 16:36:19 +000031#include "webrtc/modules/audio_processing/gain_control_impl.h"
32#include "webrtc/modules/audio_processing/high_pass_filter_impl.h"
peah1bcfce52016-08-26 07:16:04 -070033#if WEBRTC_INTELLIGIBILITY_ENHANCER
ekmeyerson60d9b332015-08-14 10:35:55 -070034#include "webrtc/modules/audio_processing/intelligibility/intelligibility_enhancer.h"
peah1bcfce52016-08-26 07:16:04 -070035#endif
peahca4cac72016-06-29 15:26:12 -070036#include "webrtc/modules/audio_processing/level_controller/level_controller.h"
andrew@webrtc.org78693fe2013-03-01 16:36:19 +000037#include "webrtc/modules/audio_processing/level_estimator_impl.h"
38#include "webrtc/modules/audio_processing/noise_suppression_impl.h"
aluebs@webrtc.orgae643ce2014-12-19 19:57:34 +000039#include "webrtc/modules/audio_processing/transient/transient_suppressor.h"
andrew@webrtc.org78693fe2013-03-01 16:36:19 +000040#include "webrtc/modules/audio_processing/voice_detection_impl.h"
Henrik Kjellanderff761fb2015-11-04 08:31:52 +010041#include "webrtc/modules/include/module_common_types.h"
Henrik Kjellander98f53512015-10-28 18:17:40 +010042#include "webrtc/system_wrappers/include/file_wrapper.h"
43#include "webrtc/system_wrappers/include/logging.h"
44#include "webrtc/system_wrappers/include/metrics.h"
andrew@webrtc.org7bf26462011-12-03 00:03:31 +000045
46#ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP
47// Files generated at build-time by the protobuf compiler.
leozwang@webrtc.orga3736342012-03-16 21:36:00 +000048#ifdef WEBRTC_ANDROID_PLATFORM_BUILD
leozwang@webrtc.org534e4952012-10-22 21:21:52 +000049#include "external/webrtc/webrtc/modules/audio_processing/debug.pb.h"
leozwang@google.comce9bfbb2011-08-03 23:34:31 +000050#else
kjellander78ddd732016-02-09 08:13:06 -080051#include "webrtc/modules/audio_processing/debug.pb.h"
leozwang@google.comce9bfbb2011-08-03 23:34:31 +000052#endif
andrew@webrtc.org7bf26462011-12-03 00:03:31 +000053#endif // WEBRTC_AUDIOPROC_DEBUG_DUMP
niklase@google.com470e71d2011-07-07 08:21:25 +000054
peah1bcfce52016-08-26 07:16:04 -070055// Check to verify that the define for the intelligibility enhancer is properly
56// set.
57#if !defined(WEBRTC_INTELLIGIBILITY_ENHANCER) || \
58 (WEBRTC_INTELLIGIBILITY_ENHANCER != 0 && \
59 WEBRTC_INTELLIGIBILITY_ENHANCER != 1)
60#error "Set WEBRTC_INTELLIGIBILITY_ENHANCER to either 0 or 1"
61#endif
62
Michael Graczyk86c6d332015-07-23 11:41:39 -070063#define RETURN_ON_ERR(expr) \
64 do { \
65 int err = (expr); \
66 if (err != kNoError) { \
67 return err; \
68 } \
andrew@webrtc.org60730cf2014-01-07 17:45:09 +000069 } while (0)
70
niklase@google.com470e71d2011-07-07 08:21:25 +000071namespace webrtc {
aluebsdf6416a2016-03-16 18:26:35 -070072
73const int AudioProcessing::kNativeSampleRatesHz[] = {
74 AudioProcessing::kSampleRate8kHz,
75 AudioProcessing::kSampleRate16kHz,
76#ifdef WEBRTC_ARCH_ARM_FAMILY
77 AudioProcessing::kSampleRate32kHz};
78#else
79 AudioProcessing::kSampleRate32kHz,
80 AudioProcessing::kSampleRate48kHz};
81#endif // WEBRTC_ARCH_ARM_FAMILY
82const size_t AudioProcessing::kNumNativeSampleRates =
83 arraysize(AudioProcessing::kNativeSampleRatesHz);
84const int AudioProcessing::kMaxNativeSampleRateHz = AudioProcessing::
85 kNativeSampleRatesHz[AudioProcessing::kNumNativeSampleRates - 1];
86
Michael Graczyk86c6d332015-07-23 11:41:39 -070087namespace {
88
89static bool LayoutHasKeyboard(AudioProcessing::ChannelLayout layout) {
90 switch (layout) {
91 case AudioProcessing::kMono:
92 case AudioProcessing::kStereo:
93 return false;
94 case AudioProcessing::kMonoAndKeyboard:
95 case AudioProcessing::kStereoAndKeyboard:
96 return true;
97 }
98
99 assert(false);
100 return false;
101}
aluebsdf6416a2016-03-16 18:26:35 -0700102
103bool is_multi_band(int sample_rate_hz) {
104 return sample_rate_hz == AudioProcessing::kSampleRate32kHz ||
105 sample_rate_hz == AudioProcessing::kSampleRate48kHz;
106}
107
peah423d2362016-04-09 16:06:52 -0700108int ClosestHigherNativeRate(int min_proc_rate) {
aluebsdf6416a2016-03-16 18:26:35 -0700109 for (int rate : AudioProcessing::kNativeSampleRatesHz) {
110 if (rate >= min_proc_rate) {
111 return rate;
112 }
113 }
114 return AudioProcessing::kMaxNativeSampleRateHz;
115}
116
Michael Graczyk86c6d332015-07-23 11:41:39 -0700117} // namespace
andrew@webrtc.org60730cf2014-01-07 17:45:09 +0000118
119// Throughout webrtc, it's assumed that success is represented by zero.
kwiberg@webrtc.org2ebfac52015-01-14 10:51:54 +0000120static_assert(AudioProcessing::kNoError == 0, "kNoError must be zero");
andrew@webrtc.org60730cf2014-01-07 17:45:09 +0000121
solenberg5e465c32015-12-08 13:22:33 -0800122struct AudioProcessingImpl::ApmPublicSubmodules {
peahbfa97112016-03-10 21:09:04 -0800123 ApmPublicSubmodules() {}
solenberg5e465c32015-12-08 13:22:33 -0800124 // Accessed externally of APM without any lock acquired.
peahb624d8c2016-03-05 03:01:14 -0800125 std::unique_ptr<EchoCancellationImpl> echo_cancellation;
peahbb9edbd2016-03-10 12:54:25 -0800126 std::unique_ptr<EchoControlMobileImpl> echo_control_mobile;
peahbfa97112016-03-10 21:09:04 -0800127 std::unique_ptr<GainControlImpl> gain_control;
kwiberg88788ad2016-02-19 07:04:49 -0800128 std::unique_ptr<HighPassFilterImpl> high_pass_filter;
129 std::unique_ptr<LevelEstimatorImpl> level_estimator;
130 std::unique_ptr<NoiseSuppressionImpl> noise_suppression;
131 std::unique_ptr<VoiceDetectionImpl> voice_detection;
132 std::unique_ptr<GainControlForExperimentalAgc>
peahbe615622016-02-13 16:40:47 -0800133 gain_control_for_experimental_agc;
solenberg5e465c32015-12-08 13:22:33 -0800134
135 // Accessed internally from both render and capture.
kwiberg88788ad2016-02-19 07:04:49 -0800136 std::unique_ptr<TransientSuppressor> transient_suppressor;
peah1bcfce52016-08-26 07:16:04 -0700137#if WEBRTC_INTELLIGIBILITY_ENHANCER
kwiberg88788ad2016-02-19 07:04:49 -0800138 std::unique_ptr<IntelligibilityEnhancer> intelligibility_enhancer;
peah1bcfce52016-08-26 07:16:04 -0700139#endif
solenberg5e465c32015-12-08 13:22:33 -0800140};
141
142struct AudioProcessingImpl::ApmPrivateSubmodules {
Alejandro Luebsf4022ff2016-07-01 17:19:09 -0700143 explicit ApmPrivateSubmodules(NonlinearBeamformer* beamformer)
solenberg5e465c32015-12-08 13:22:33 -0800144 : beamformer(beamformer) {}
145 // Accessed internally from capture or during initialization
Alejandro Luebsf4022ff2016-07-01 17:19:09 -0700146 std::unique_ptr<NonlinearBeamformer> beamformer;
kwiberg88788ad2016-02-19 07:04:49 -0800147 std::unique_ptr<AgcManagerDirect> agc_manager;
peahca4cac72016-06-29 15:26:12 -0700148 std::unique_ptr<LevelController> level_controller;
solenberg5e465c32015-12-08 13:22:33 -0800149};
150
andrew@webrtc.orge84978f2014-01-25 02:09:06 +0000151AudioProcessing* AudioProcessing::Create() {
152 Config config;
aluebs@webrtc.orgd82f55d2015-01-15 18:07:21 +0000153 return Create(config, nullptr);
andrew@webrtc.orge84978f2014-01-25 02:09:06 +0000154}
155
156AudioProcessing* AudioProcessing::Create(const Config& config) {
aluebs@webrtc.orgd82f55d2015-01-15 18:07:21 +0000157 return Create(config, nullptr);
158}
159
160AudioProcessing* AudioProcessing::Create(const Config& config,
Alejandro Luebsf4022ff2016-07-01 17:19:09 -0700161 NonlinearBeamformer* beamformer) {
aluebs@webrtc.orgd82f55d2015-01-15 18:07:21 +0000162 AudioProcessingImpl* apm = new AudioProcessingImpl(config, beamformer);
niklase@google.com470e71d2011-07-07 08:21:25 +0000163 if (apm->Initialize() != kNoError) {
164 delete apm;
peahdf3efa82015-11-28 12:35:15 -0800165 apm = nullptr;
niklase@google.com470e71d2011-07-07 08:21:25 +0000166 }
167
168 return apm;
169}
170
andrew@webrtc.orge84978f2014-01-25 02:09:06 +0000171AudioProcessingImpl::AudioProcessingImpl(const Config& config)
aluebs@webrtc.orgd82f55d2015-01-15 18:07:21 +0000172 : AudioProcessingImpl(config, nullptr) {}
173
174AudioProcessingImpl::AudioProcessingImpl(const Config& config,
Alejandro Luebsf4022ff2016-07-01 17:19:09 -0700175 NonlinearBeamformer* beamformer)
peahdf3efa82015-11-28 12:35:15 -0800176 : public_submodules_(new ApmPublicSubmodules()),
177 private_submodules_(new ApmPrivateSubmodules(beamformer)),
178 constants_(config.Get<ExperimentalAgc>().startup_min_volume,
pbos@webrtc.org788acd12014-12-15 09:41:24 +0000179#if defined(WEBRTC_ANDROID) || defined(WEBRTC_IOS)
Alejandro Luebsc9b0c262016-05-16 15:32:38 -0700180 false),
pbos@webrtc.org788acd12014-12-15 09:41:24 +0000181#else
Alejandro Luebsc9b0c262016-05-16 15:32:38 -0700182 config.Get<ExperimentalAgc>().enabled),
pbos@webrtc.org788acd12014-12-15 09:41:24 +0000183#endif
andrew1c7075f2015-06-24 18:14:14 -0700184#if defined(WEBRTC_ANDROID) || defined(WEBRTC_IOS)
aluebs2a346882016-01-11 18:04:30 -0800185 capture_(false,
andrew1c7075f2015-06-24 18:14:14 -0700186#else
aluebs2a346882016-01-11 18:04:30 -0800187 capture_(config.Get<ExperimentalNs>().enabled,
andrew1c7075f2015-06-24 18:14:14 -0700188#endif
aluebs2a346882016-01-11 18:04:30 -0800189 config.Get<Beamforming>().array_geometry,
aluebsb2328d12016-01-11 20:32:29 -0800190 config.Get<Beamforming>().target_direction),
Alejandro Luebsc9b0c262016-05-16 15:32:38 -0700191 capture_nonlocked_(config.Get<Beamforming>().enabled,
peahca4cac72016-06-29 15:26:12 -0700192 config.Get<Intelligibility>().enabled,
193 config.Get<LevelControl>().enabled) {
peahdf3efa82015-11-28 12:35:15 -0800194 {
195 rtc::CritScope cs_render(&crit_render_);
196 rtc::CritScope cs_capture(&crit_capture_);
niklase@google.com470e71d2011-07-07 08:21:25 +0000197
peahb624d8c2016-03-05 03:01:14 -0800198 public_submodules_->echo_cancellation.reset(
peahb58a1582016-03-15 09:34:24 -0700199 new EchoCancellationImpl(&crit_render_, &crit_capture_));
peahbb9edbd2016-03-10 12:54:25 -0800200 public_submodules_->echo_control_mobile.reset(
peah253534d2016-03-15 04:32:28 -0700201 new EchoControlMobileImpl(&crit_render_, &crit_capture_));
peahbfa97112016-03-10 21:09:04 -0800202 public_submodules_->gain_control.reset(
peahb8fbb542016-03-15 02:28:08 -0700203 new GainControlImpl(&crit_capture_, &crit_capture_));
solenberg70f99032015-12-08 11:07:32 -0800204 public_submodules_->high_pass_filter.reset(
205 new HighPassFilterImpl(&crit_capture_));
solenberg949028f2015-12-15 11:39:38 -0800206 public_submodules_->level_estimator.reset(
207 new LevelEstimatorImpl(&crit_capture_));
solenberg5e465c32015-12-08 13:22:33 -0800208 public_submodules_->noise_suppression.reset(
209 new NoiseSuppressionImpl(&crit_capture_));
solenberga29386c2015-12-16 03:31:12 -0800210 public_submodules_->voice_detection.reset(
211 new VoiceDetectionImpl(&crit_capture_));
peahbe615622016-02-13 16:40:47 -0800212 public_submodules_->gain_control_for_experimental_agc.reset(
peahbfa97112016-03-10 21:09:04 -0800213 new GainControlForExperimentalAgc(
214 public_submodules_->gain_control.get(), &crit_capture_));
peahca4cac72016-06-29 15:26:12 -0700215
216 private_submodules_->level_controller.reset(new LevelController());
peahdf3efa82015-11-28 12:35:15 -0800217 }
pbos@webrtc.org788acd12014-12-15 09:41:24 +0000218
andrew@webrtc.orge84978f2014-01-25 02:09:06 +0000219 SetExtraOptions(config);
niklase@google.com470e71d2011-07-07 08:21:25 +0000220}
221
222AudioProcessingImpl::~AudioProcessingImpl() {
peahdf3efa82015-11-28 12:35:15 -0800223 // Depends on gain_control_ and
peahbe615622016-02-13 16:40:47 -0800224 // public_submodules_->gain_control_for_experimental_agc.
peahdf3efa82015-11-28 12:35:15 -0800225 private_submodules_->agc_manager.reset();
226 // Depends on gain_control_.
peahbe615622016-02-13 16:40:47 -0800227 public_submodules_->gain_control_for_experimental_agc.reset();
niklase@google.com470e71d2011-07-07 08:21:25 +0000228
andrew@webrtc.org7bf26462011-12-03 00:03:31 +0000229#ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP
tommia6219cc2016-06-15 10:30:14 -0700230 debug_dump_.debug_file->CloseFile();
peahdf3efa82015-11-28 12:35:15 -0800231#endif
niklase@google.com470e71d2011-07-07 08:21:25 +0000232}
233
niklase@google.com470e71d2011-07-07 08:21:25 +0000234int AudioProcessingImpl::Initialize() {
peahdf3efa82015-11-28 12:35:15 -0800235 // Run in a single-threaded manner during initialization.
236 rtc::CritScope cs_render(&crit_render_);
237 rtc::CritScope cs_capture(&crit_capture_);
niklase@google.com470e71d2011-07-07 08:21:25 +0000238 return InitializeLocked();
239}
240
andrew@webrtc.orgddbb8a22014-04-22 21:00:04 +0000241int AudioProcessingImpl::Initialize(int input_sample_rate_hz,
242 int output_sample_rate_hz,
243 int reverse_sample_rate_hz,
244 ChannelLayout input_layout,
245 ChannelLayout output_layout,
246 ChannelLayout reverse_layout) {
Michael Graczyk86c6d332015-07-23 11:41:39 -0700247 const ProcessingConfig processing_config = {
ekmeyerson60d9b332015-08-14 10:35:55 -0700248 {{input_sample_rate_hz,
249 ChannelsFromLayout(input_layout),
Michael Graczyk86c6d332015-07-23 11:41:39 -0700250 LayoutHasKeyboard(input_layout)},
ekmeyerson60d9b332015-08-14 10:35:55 -0700251 {output_sample_rate_hz,
252 ChannelsFromLayout(output_layout),
Michael Graczyk86c6d332015-07-23 11:41:39 -0700253 LayoutHasKeyboard(output_layout)},
ekmeyerson60d9b332015-08-14 10:35:55 -0700254 {reverse_sample_rate_hz,
255 ChannelsFromLayout(reverse_layout),
256 LayoutHasKeyboard(reverse_layout)},
257 {reverse_sample_rate_hz,
258 ChannelsFromLayout(reverse_layout),
Michael Graczyk86c6d332015-07-23 11:41:39 -0700259 LayoutHasKeyboard(reverse_layout)}}};
260
261 return Initialize(processing_config);
262}
263
264int AudioProcessingImpl::Initialize(const ProcessingConfig& processing_config) {
peahdf3efa82015-11-28 12:35:15 -0800265 // Run in a single-threaded manner during initialization.
266 rtc::CritScope cs_render(&crit_render_);
267 rtc::CritScope cs_capture(&crit_capture_);
Michael Graczyk86c6d332015-07-23 11:41:39 -0700268 return InitializeLocked(processing_config);
andrew@webrtc.orga8b97372014-03-10 22:26:12 +0000269}
270
peahdf3efa82015-11-28 12:35:15 -0800271int AudioProcessingImpl::MaybeInitializeRender(
peah81b9bfe2015-11-27 02:47:28 -0800272 const ProcessingConfig& processing_config) {
peahdf3efa82015-11-28 12:35:15 -0800273 return MaybeInitialize(processing_config);
peah81b9bfe2015-11-27 02:47:28 -0800274}
275
peahdf3efa82015-11-28 12:35:15 -0800276int AudioProcessingImpl::MaybeInitializeCapture(
peah81b9bfe2015-11-27 02:47:28 -0800277 const ProcessingConfig& processing_config) {
peahdf3efa82015-11-28 12:35:15 -0800278 return MaybeInitialize(processing_config);
peah81b9bfe2015-11-27 02:47:28 -0800279}
280
peah192164e2015-11-17 02:16:45 -0800281// Calls InitializeLocked() if any of the audio parameters have changed from
peahdf3efa82015-11-28 12:35:15 -0800282// their current values (needs to be called while holding the crit_render_lock).
283int AudioProcessingImpl::MaybeInitialize(
peah192164e2015-11-17 02:16:45 -0800284 const ProcessingConfig& processing_config) {
peahdf3efa82015-11-28 12:35:15 -0800285 // Called from both threads. Thread check is therefore not possible.
286 if (processing_config == formats_.api_format) {
peah192164e2015-11-17 02:16:45 -0800287 return kNoError;
288 }
peahdf3efa82015-11-28 12:35:15 -0800289
290 rtc::CritScope cs_capture(&crit_capture_);
peah192164e2015-11-17 02:16:45 -0800291 return InitializeLocked(processing_config);
292}
293
niklase@google.com470e71d2011-07-07 08:21:25 +0000294int AudioProcessingImpl::InitializeLocked() {
Michael Graczyk86c6d332015-07-23 11:41:39 -0700295 const int fwd_audio_buffer_channels =
aluebsb2328d12016-01-11 20:32:29 -0800296 capture_nonlocked_.beamformer_enabled
peahdf3efa82015-11-28 12:35:15 -0800297 ? formats_.api_format.input_stream().num_channels()
298 : formats_.api_format.output_stream().num_channels();
ekmeyerson60d9b332015-08-14 10:35:55 -0700299 const int rev_audio_buffer_out_num_frames =
peahdf3efa82015-11-28 12:35:15 -0800300 formats_.api_format.reverse_output_stream().num_frames() == 0
301 ? formats_.rev_proc_format.num_frames()
302 : formats_.api_format.reverse_output_stream().num_frames();
303 if (formats_.api_format.reverse_input_stream().num_channels() > 0) {
304 render_.render_audio.reset(new AudioBuffer(
305 formats_.api_format.reverse_input_stream().num_frames(),
306 formats_.api_format.reverse_input_stream().num_channels(),
307 formats_.rev_proc_format.num_frames(),
308 formats_.rev_proc_format.num_channels(),
ekmeyerson60d9b332015-08-14 10:35:55 -0700309 rev_audio_buffer_out_num_frames));
310 if (rev_conversion_needed()) {
kwibergc2b785d2016-02-24 05:22:32 -0800311 render_.render_converter = AudioConverter::Create(
peahdf3efa82015-11-28 12:35:15 -0800312 formats_.api_format.reverse_input_stream().num_channels(),
313 formats_.api_format.reverse_input_stream().num_frames(),
314 formats_.api_format.reverse_output_stream().num_channels(),
kwibergc2b785d2016-02-24 05:22:32 -0800315 formats_.api_format.reverse_output_stream().num_frames());
ekmeyerson60d9b332015-08-14 10:35:55 -0700316 } else {
peahdf3efa82015-11-28 12:35:15 -0800317 render_.render_converter.reset(nullptr);
ekmeyerson60d9b332015-08-14 10:35:55 -0700318 }
Michael Graczyk86c6d332015-07-23 11:41:39 -0700319 } else {
peahdf3efa82015-11-28 12:35:15 -0800320 render_.render_audio.reset(nullptr);
321 render_.render_converter.reset(nullptr);
Michael Graczyk86c6d332015-07-23 11:41:39 -0700322 }
peahdf3efa82015-11-28 12:35:15 -0800323 capture_.capture_audio.reset(
324 new AudioBuffer(formats_.api_format.input_stream().num_frames(),
325 formats_.api_format.input_stream().num_channels(),
326 capture_nonlocked_.fwd_proc_format.num_frames(),
327 fwd_audio_buffer_channels,
328 formats_.api_format.output_stream().num_frames()));
niklase@google.com470e71d2011-07-07 08:21:25 +0000329
peahbfa97112016-03-10 21:09:04 -0800330 InitializeGainController();
peahb624d8c2016-03-05 03:01:14 -0800331 InitializeEchoCanceller();
peahbb9edbd2016-03-10 12:54:25 -0800332 InitializeEchoControlMobile();
Bjorn Volckeradc46c42015-04-15 11:42:40 +0200333 InitializeExperimentalAgc();
Bjorn Volckeradc46c42015-04-15 11:42:40 +0200334 InitializeTransient();
aluebs@webrtc.orgae643ce2014-12-19 19:57:34 +0000335 InitializeBeamformer();
peah1bcfce52016-08-26 07:16:04 -0700336#if WEBRTC_INTELLIGIBILITY_ENHANCER
ekmeyerson60d9b332015-08-14 10:35:55 -0700337 InitializeIntelligibility();
peah1bcfce52016-08-26 07:16:04 -0700338#endif
solenberg70f99032015-12-08 11:07:32 -0800339 InitializeHighPassFilter();
solenberg5e465c32015-12-08 13:22:33 -0800340 InitializeNoiseSuppression();
solenberg949028f2015-12-15 11:39:38 -0800341 InitializeLevelEstimator();
solenberga29386c2015-12-16 03:31:12 -0800342 InitializeVoiceDetection();
peahca4cac72016-06-29 15:26:12 -0700343 InitializeLevelController();
solenberg70f99032015-12-08 11:07:32 -0800344
andrew@webrtc.org7bf26462011-12-03 00:03:31 +0000345#ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP
tommia6219cc2016-06-15 10:30:14 -0700346 if (debug_dump_.debug_file->is_open()) {
ajm@google.com808e0e02011-08-03 21:08:51 +0000347 int err = WriteInitMessage();
348 if (err != kNoError) {
349 return err;
350 }
351 }
andrew@webrtc.org7bf26462011-12-03 00:03:31 +0000352#endif
ajm@google.com808e0e02011-08-03 21:08:51 +0000353
niklase@google.com470e71d2011-07-07 08:21:25 +0000354 return kNoError;
355}
356
Michael Graczyk86c6d332015-07-23 11:41:39 -0700357int AudioProcessingImpl::InitializeLocked(const ProcessingConfig& config) {
358 for (const auto& stream : config.streams) {
Michael Graczyk86c6d332015-07-23 11:41:39 -0700359 if (stream.num_channels() > 0 && stream.sample_rate_hz() <= 0) {
360 return kBadSampleRateError;
361 }
andrew@webrtc.orga8b97372014-03-10 22:26:12 +0000362 }
Michael Graczyk86c6d332015-07-23 11:41:39 -0700363
Peter Kasting69558702016-01-12 16:26:35 -0800364 const size_t num_in_channels = config.input_stream().num_channels();
365 const size_t num_out_channels = config.output_stream().num_channels();
Michael Graczyk86c6d332015-07-23 11:41:39 -0700366
367 // Need at least one input channel.
368 // Need either one output channel or as many outputs as there are inputs.
369 if (num_in_channels == 0 ||
370 !(num_out_channels == 1 || num_out_channels == num_in_channels)) {
Michael Graczykc2047542015-07-22 21:06:11 -0700371 return kBadNumberChannelsError;
372 }
373
aluebsb2328d12016-01-11 20:32:29 -0800374 if (capture_nonlocked_.beamformer_enabled &&
Peter Kasting69558702016-01-12 16:26:35 -0800375 num_in_channels != capture_.array_geometry.size()) {
Michael Graczyk86c6d332015-07-23 11:41:39 -0700376 return kBadNumberChannelsError;
377 }
378
peahdf3efa82015-11-28 12:35:15 -0800379 formats_.api_format = config;
andrew@webrtc.orgddbb8a22014-04-22 21:00:04 +0000380
peah423d2362016-04-09 16:06:52 -0700381 capture_nonlocked_.fwd_proc_format = StreamConfig(ClosestHigherNativeRate(
382 std::min(formats_.api_format.input_stream().sample_rate_hz(),
383 formats_.api_format.output_stream().sample_rate_hz())));
andrew@webrtc.orga8b97372014-03-10 22:26:12 +0000384
aluebseb3603b2016-04-20 15:27:58 -0700385 int rev_proc_rate = ClosestHigherNativeRate(std::min(
386 formats_.api_format.reverse_input_stream().sample_rate_hz(),
387 formats_.api_format.reverse_output_stream().sample_rate_hz()));
388 // TODO(aluebs): Remove this restriction once we figure out why the 3-band
389 // splitting filter degrades the AEC performance.
390 if (rev_proc_rate > kSampleRate32kHz) {
391 rev_proc_rate = is_rev_processed() ? kSampleRate32kHz : kSampleRate16kHz;
392 }
393 // If the forward sample rate is 8 kHz, the reverse stream is also processed
394 // at this rate.
peahdf3efa82015-11-28 12:35:15 -0800395 if (capture_nonlocked_.fwd_proc_format.sample_rate_hz() == kSampleRate8kHz) {
andrew@webrtc.orgddbb8a22014-04-22 21:00:04 +0000396 rev_proc_rate = kSampleRate8kHz;
andrew@webrtc.orga8b97372014-03-10 22:26:12 +0000397 } else {
aluebseb3603b2016-04-20 15:27:58 -0700398 rev_proc_rate = std::max(rev_proc_rate, static_cast<int>(kSampleRate16kHz));
andrew@webrtc.orgddbb8a22014-04-22 21:00:04 +0000399 }
400
andrew@webrtc.org30be8272014-09-24 20:06:23 +0000401 // Always downmix the reverse stream to mono for analysis. This has been
402 // demonstrated to work well for AEC in most practical scenarios.
peahdf3efa82015-11-28 12:35:15 -0800403 formats_.rev_proc_format = StreamConfig(rev_proc_rate, 1);
andrew@webrtc.orgddbb8a22014-04-22 21:00:04 +0000404
peahdf3efa82015-11-28 12:35:15 -0800405 if (capture_nonlocked_.fwd_proc_format.sample_rate_hz() == kSampleRate32kHz ||
406 capture_nonlocked_.fwd_proc_format.sample_rate_hz() == kSampleRate48kHz) {
407 capture_nonlocked_.split_rate = kSampleRate16kHz;
andrew@webrtc.orgddbb8a22014-04-22 21:00:04 +0000408 } else {
peahdf3efa82015-11-28 12:35:15 -0800409 capture_nonlocked_.split_rate =
410 capture_nonlocked_.fwd_proc_format.sample_rate_hz();
andrew@webrtc.orga8b97372014-03-10 22:26:12 +0000411 }
412
413 return InitializeLocked();
414}
415
andrew@webrtc.org61e596f2013-07-25 18:28:29 +0000416void AudioProcessingImpl::SetExtraOptions(const Config& config) {
peahdf3efa82015-11-28 12:35:15 -0800417 // Run in a single-threaded manner when setting the extra options.
418 rtc::CritScope cs_render(&crit_render_);
419 rtc::CritScope cs_capture(&crit_capture_);
pbos@webrtc.org788acd12014-12-15 09:41:24 +0000420
peahb624d8c2016-03-05 03:01:14 -0800421 public_submodules_->echo_cancellation->SetExtraOptions(config);
422
peahdf3efa82015-11-28 12:35:15 -0800423 if (capture_.transient_suppressor_enabled !=
424 config.Get<ExperimentalNs>().enabled) {
425 capture_.transient_suppressor_enabled =
426 config.Get<ExperimentalNs>().enabled;
pbos@webrtc.org788acd12014-12-15 09:41:24 +0000427 InitializeTransient();
428 }
aluebs2a346882016-01-11 18:04:30 -0800429
peahca4cac72016-06-29 15:26:12 -0700430 if (capture_nonlocked_.level_controller_enabled !=
431 config.Get<LevelControl>().enabled) {
432 capture_nonlocked_.level_controller_enabled =
433 config.Get<LevelControl>().enabled;
434 LOG(LS_INFO) << "Level controller activated: "
435 << config.Get<LevelControl>().enabled;
436
peahca4cac72016-06-29 15:26:12 -0700437 InitializeLevelController();
438 }
439
peah1bcfce52016-08-26 07:16:04 -0700440#if WEBRTC_INTELLIGIBILITY_ENHANCER
Alejandro Luebsc9b0c262016-05-16 15:32:38 -0700441 if(capture_nonlocked_.intelligibility_enabled !=
442 config.Get<Intelligibility>().enabled) {
443 capture_nonlocked_.intelligibility_enabled =
444 config.Get<Intelligibility>().enabled;
445 InitializeIntelligibility();
446 }
peah1bcfce52016-08-26 07:16:04 -0700447#endif
Alejandro Luebsc9b0c262016-05-16 15:32:38 -0700448
aluebs2a346882016-01-11 18:04:30 -0800449#ifdef WEBRTC_ANDROID_PLATFORM_BUILD
aluebsb2328d12016-01-11 20:32:29 -0800450 if (capture_nonlocked_.beamformer_enabled !=
451 config.Get<Beamforming>().enabled) {
452 capture_nonlocked_.beamformer_enabled = config.Get<Beamforming>().enabled;
aluebs2a346882016-01-11 18:04:30 -0800453 if (config.Get<Beamforming>().array_geometry.size() > 1) {
454 capture_.array_geometry = config.Get<Beamforming>().array_geometry;
455 }
456 capture_.target_direction = config.Get<Beamforming>().target_direction;
457 InitializeBeamformer();
458 }
459#endif // WEBRTC_ANDROID_PLATFORM_BUILD
andrew@webrtc.org61e596f2013-07-25 18:28:29 +0000460}
461
andrew@webrtc.orgddbb8a22014-04-22 21:00:04 +0000462int AudioProcessingImpl::proc_sample_rate_hz() const {
peahdf3efa82015-11-28 12:35:15 -0800463 // Used as callback from submodules, hence locking is not allowed.
464 return capture_nonlocked_.fwd_proc_format.sample_rate_hz();
niklase@google.com470e71d2011-07-07 08:21:25 +0000465}
466
andrew@webrtc.orgddbb8a22014-04-22 21:00:04 +0000467int AudioProcessingImpl::proc_split_sample_rate_hz() const {
peahdf3efa82015-11-28 12:35:15 -0800468 // Used as callback from submodules, hence locking is not allowed.
469 return capture_nonlocked_.split_rate;
niklase@google.com470e71d2011-07-07 08:21:25 +0000470}
471
Peter Kasting69558702016-01-12 16:26:35 -0800472size_t AudioProcessingImpl::num_reverse_channels() const {
peahdf3efa82015-11-28 12:35:15 -0800473 // Used as callback from submodules, hence locking is not allowed.
474 return formats_.rev_proc_format.num_channels();
niklase@google.com470e71d2011-07-07 08:21:25 +0000475}
476
Peter Kasting69558702016-01-12 16:26:35 -0800477size_t AudioProcessingImpl::num_input_channels() const {
peahdf3efa82015-11-28 12:35:15 -0800478 // Used as callback from submodules, hence locking is not allowed.
479 return formats_.api_format.input_stream().num_channels();
niklase@google.com470e71d2011-07-07 08:21:25 +0000480}
481
Peter Kasting69558702016-01-12 16:26:35 -0800482size_t AudioProcessingImpl::num_proc_channels() const {
aluebsb2328d12016-01-11 20:32:29 -0800483 // Used as callback from submodules, hence locking is not allowed.
484 return capture_nonlocked_.beamformer_enabled ? 1 : num_output_channels();
485}
486
Peter Kasting69558702016-01-12 16:26:35 -0800487size_t AudioProcessingImpl::num_output_channels() const {
peahdf3efa82015-11-28 12:35:15 -0800488 // Used as callback from submodules, hence locking is not allowed.
489 return formats_.api_format.output_stream().num_channels();
niklase@google.com470e71d2011-07-07 08:21:25 +0000490}
491
andrew@webrtc.org17342e52014-02-12 22:28:31 +0000492void AudioProcessingImpl::set_output_will_be_muted(bool muted) {
peahdf3efa82015-11-28 12:35:15 -0800493 rtc::CritScope cs(&crit_capture_);
494 capture_.output_will_be_muted = muted;
495 if (private_submodules_->agc_manager.get()) {
496 private_submodules_->agc_manager->SetCaptureMuted(
497 capture_.output_will_be_muted);
pbos@webrtc.org788acd12014-12-15 09:41:24 +0000498 }
andrew@webrtc.org17342e52014-02-12 22:28:31 +0000499}
500
andrew@webrtc.org17342e52014-02-12 22:28:31 +0000501
andrew@webrtc.orgddbb8a22014-04-22 21:00:04 +0000502int AudioProcessingImpl::ProcessStream(const float* const* src,
Peter Kastingdce40cf2015-08-24 14:52:23 -0700503 size_t samples_per_channel,
andrew@webrtc.orgddbb8a22014-04-22 21:00:04 +0000504 int input_sample_rate_hz,
andrew@webrtc.org17e40642014-03-04 20:58:13 +0000505 ChannelLayout input_layout,
andrew@webrtc.orgddbb8a22014-04-22 21:00:04 +0000506 int output_sample_rate_hz,
507 ChannelLayout output_layout,
508 float* const* dest) {
peah369f8282015-12-17 06:42:29 -0800509 TRACE_EVENT0("webrtc", "AudioProcessing::ProcessStream_ChannelLayout");
peahdf3efa82015-11-28 12:35:15 -0800510 StreamConfig input_stream;
511 StreamConfig output_stream;
512 {
513 // Access the formats_.api_format.input_stream beneath the capture lock.
514 // The lock must be released as it is later required in the call
515 // to ProcessStream(,,,);
516 rtc::CritScope cs(&crit_capture_);
517 input_stream = formats_.api_format.input_stream();
518 output_stream = formats_.api_format.output_stream();
519 }
520
Michael Graczyk86c6d332015-07-23 11:41:39 -0700521 input_stream.set_sample_rate_hz(input_sample_rate_hz);
522 input_stream.set_num_channels(ChannelsFromLayout(input_layout));
523 input_stream.set_has_keyboard(LayoutHasKeyboard(input_layout));
Michael Graczyk86c6d332015-07-23 11:41:39 -0700524 output_stream.set_sample_rate_hz(output_sample_rate_hz);
525 output_stream.set_num_channels(ChannelsFromLayout(output_layout));
526 output_stream.set_has_keyboard(LayoutHasKeyboard(output_layout));
527
528 if (samples_per_channel != input_stream.num_frames()) {
529 return kBadDataLengthError;
530 }
531 return ProcessStream(src, input_stream, output_stream, dest);
532}
533
534int AudioProcessingImpl::ProcessStream(const float* const* src,
535 const StreamConfig& input_config,
536 const StreamConfig& output_config,
537 float* const* dest) {
peah369f8282015-12-17 06:42:29 -0800538 TRACE_EVENT0("webrtc", "AudioProcessing::ProcessStream_StreamConfig");
peahdf3efa82015-11-28 12:35:15 -0800539 ProcessingConfig processing_config;
540 {
541 // Acquire the capture lock in order to safely call the function
542 // that retrieves the render side data. This function accesses apm
543 // getters that need the capture lock held when being called.
544 rtc::CritScope cs_capture(&crit_capture_);
545 public_submodules_->echo_cancellation->ReadQueuedRenderData();
546 public_submodules_->echo_control_mobile->ReadQueuedRenderData();
547 public_submodules_->gain_control->ReadQueuedRenderData();
548
549 if (!src || !dest) {
550 return kNullPointerError;
551 }
552
553 processing_config = formats_.api_format;
niklase@google.com470e71d2011-07-07 08:21:25 +0000554 }
andrew@webrtc.org17e40642014-03-04 20:58:13 +0000555
Michael Graczyk86c6d332015-07-23 11:41:39 -0700556 processing_config.input_stream() = input_config;
557 processing_config.output_stream() = output_config;
558
peahdf3efa82015-11-28 12:35:15 -0800559 {
560 // Do conditional reinitialization.
561 rtc::CritScope cs_render(&crit_render_);
562 RETURN_ON_ERR(MaybeInitializeCapture(processing_config));
563 }
564 rtc::CritScope cs_capture(&crit_capture_);
Michael Graczyk86c6d332015-07-23 11:41:39 -0700565 assert(processing_config.input_stream().num_frames() ==
peahdf3efa82015-11-28 12:35:15 -0800566 formats_.api_format.input_stream().num_frames());
andrew@webrtc.org17e40642014-03-04 20:58:13 +0000567
568#ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP
tommia6219cc2016-06-15 10:30:14 -0700569 if (debug_dump_.debug_file->is_open()) {
Minyue13b96ba2015-10-03 00:39:14 +0200570 RETURN_ON_ERR(WriteConfigMessage(false));
571
peahdf3efa82015-11-28 12:35:15 -0800572 debug_dump_.capture.event_msg->set_type(audioproc::Event::STREAM);
573 audioproc::Stream* msg = debug_dump_.capture.event_msg->mutable_stream();
aluebs@webrtc.org59a1b1b2014-08-28 10:43:09 +0000574 const size_t channel_size =
peahdf3efa82015-11-28 12:35:15 -0800575 sizeof(float) * formats_.api_format.input_stream().num_frames();
Peter Kasting69558702016-01-12 16:26:35 -0800576 for (size_t i = 0; i < formats_.api_format.input_stream().num_channels();
577 ++i)
andrew@webrtc.orgddbb8a22014-04-22 21:00:04 +0000578 msg->add_input_channel(src[i], channel_size);
andrew@webrtc.org17e40642014-03-04 20:58:13 +0000579 }
580#endif
581
peahdf3efa82015-11-28 12:35:15 -0800582 capture_.capture_audio->CopyFrom(src, formats_.api_format.input_stream());
andrew@webrtc.org17e40642014-03-04 20:58:13 +0000583 RETURN_ON_ERR(ProcessStreamLocked());
peahdf3efa82015-11-28 12:35:15 -0800584 capture_.capture_audio->CopyTo(formats_.api_format.output_stream(), dest);
andrew@webrtc.org17e40642014-03-04 20:58:13 +0000585
586#ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP
tommia6219cc2016-06-15 10:30:14 -0700587 if (debug_dump_.debug_file->is_open()) {
peahdf3efa82015-11-28 12:35:15 -0800588 audioproc::Stream* msg = debug_dump_.capture.event_msg->mutable_stream();
aluebs@webrtc.org59a1b1b2014-08-28 10:43:09 +0000589 const size_t channel_size =
peahdf3efa82015-11-28 12:35:15 -0800590 sizeof(float) * formats_.api_format.output_stream().num_frames();
Peter Kasting69558702016-01-12 16:26:35 -0800591 for (size_t i = 0; i < formats_.api_format.output_stream().num_channels();
592 ++i)
andrew@webrtc.orgddbb8a22014-04-22 21:00:04 +0000593 msg->add_output_channel(dest[i], channel_size);
peahdf3efa82015-11-28 12:35:15 -0800594 RETURN_ON_ERR(WriteMessageToDebugFile(debug_dump_.debug_file.get(),
ivocd66b44d2016-01-15 03:06:36 -0800595 &debug_dump_.num_bytes_left_for_log_,
peahdf3efa82015-11-28 12:35:15 -0800596 &crit_debug_, &debug_dump_.capture));
andrew@webrtc.org17e40642014-03-04 20:58:13 +0000597 }
598#endif
599
600 return kNoError;
601}
602
603int AudioProcessingImpl::ProcessStream(AudioFrame* frame) {
peah369f8282015-12-17 06:42:29 -0800604 TRACE_EVENT0("webrtc", "AudioProcessing::ProcessStream_AudioFrame");
peahdf3efa82015-11-28 12:35:15 -0800605 {
606 // Acquire the capture lock in order to safely call the function
607 // that retrieves the render side data. This function accesses apm
608 // getters that need the capture lock held when being called.
609 // The lock needs to be released as
610 // public_submodules_->echo_control_mobile->is_enabled() aquires this lock
611 // as well.
612 rtc::CritScope cs_capture(&crit_capture_);
613 public_submodules_->echo_cancellation->ReadQueuedRenderData();
614 public_submodules_->echo_control_mobile->ReadQueuedRenderData();
615 public_submodules_->gain_control->ReadQueuedRenderData();
616 }
peahfa6228e2015-11-16 16:27:42 -0800617
andrew@webrtc.org17e40642014-03-04 20:58:13 +0000618 if (!frame) {
619 return kNullPointerError;
620 }
andrew@webrtc.orgddbb8a22014-04-22 21:00:04 +0000621 // Must be a native rate.
622 if (frame->sample_rate_hz_ != kSampleRate8kHz &&
623 frame->sample_rate_hz_ != kSampleRate16kHz &&
aluebs@webrtc.org087da132014-11-17 23:01:23 +0000624 frame->sample_rate_hz_ != kSampleRate32kHz &&
625 frame->sample_rate_hz_ != kSampleRate48kHz) {
andrew@webrtc.orgddbb8a22014-04-22 21:00:04 +0000626 return kBadSampleRateError;
627 }
peah192164e2015-11-17 02:16:45 -0800628
peahdf3efa82015-11-28 12:35:15 -0800629 ProcessingConfig processing_config;
630 {
631 // Aquire lock for the access of api_format.
632 // The lock is released immediately due to the conditional
633 // reinitialization.
634 rtc::CritScope cs_capture(&crit_capture_);
635 // TODO(ajm): The input and output rates and channels are currently
636 // constrained to be identical in the int16 interface.
637 processing_config = formats_.api_format;
638 }
Michael Graczyk86c6d332015-07-23 11:41:39 -0700639 processing_config.input_stream().set_sample_rate_hz(frame->sample_rate_hz_);
640 processing_config.input_stream().set_num_channels(frame->num_channels_);
641 processing_config.output_stream().set_sample_rate_hz(frame->sample_rate_hz_);
642 processing_config.output_stream().set_num_channels(frame->num_channels_);
643
peahdf3efa82015-11-28 12:35:15 -0800644 {
645 // Do conditional reinitialization.
646 rtc::CritScope cs_render(&crit_render_);
647 RETURN_ON_ERR(MaybeInitializeCapture(processing_config));
648 }
649 rtc::CritScope cs_capture(&crit_capture_);
peah192164e2015-11-17 02:16:45 -0800650 if (frame->samples_per_channel_ !=
peahdf3efa82015-11-28 12:35:15 -0800651 formats_.api_format.input_stream().num_frames()) {
niklase@google.com470e71d2011-07-07 08:21:25 +0000652 return kBadDataLengthError;
653 }
654
andrew@webrtc.org7bf26462011-12-03 00:03:31 +0000655#ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP
tommia6219cc2016-06-15 10:30:14 -0700656 if (debug_dump_.debug_file->is_open()) {
peah644fa962016-08-18 06:48:33 -0700657 RETURN_ON_ERR(WriteConfigMessage(false));
658
peahdf3efa82015-11-28 12:35:15 -0800659 debug_dump_.capture.event_msg->set_type(audioproc::Event::STREAM);
660 audioproc::Stream* msg = debug_dump_.capture.event_msg->mutable_stream();
Michael Graczyk86c6d332015-07-23 11:41:39 -0700661 const size_t data_size =
662 sizeof(int16_t) * frame->samples_per_channel_ * frame->num_channels_;
andrew@webrtc.org63a50982012-05-02 23:56:37 +0000663 msg->set_input_data(frame->data_, data_size);
andrew@webrtc.org17e40642014-03-04 20:58:13 +0000664 }
665#endif
666
peahdf3efa82015-11-28 12:35:15 -0800667 capture_.capture_audio->DeinterleaveFrom(frame);
andrew@webrtc.org17e40642014-03-04 20:58:13 +0000668 RETURN_ON_ERR(ProcessStreamLocked());
aluebsdf6416a2016-03-16 18:26:35 -0700669 capture_.capture_audio->InterleaveTo(frame, output_copy_needed());
andrew@webrtc.org17e40642014-03-04 20:58:13 +0000670
671#ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP
tommia6219cc2016-06-15 10:30:14 -0700672 if (debug_dump_.debug_file->is_open()) {
peahdf3efa82015-11-28 12:35:15 -0800673 audioproc::Stream* msg = debug_dump_.capture.event_msg->mutable_stream();
Michael Graczyk86c6d332015-07-23 11:41:39 -0700674 const size_t data_size =
675 sizeof(int16_t) * frame->samples_per_channel_ * frame->num_channels_;
andrew@webrtc.org17e40642014-03-04 20:58:13 +0000676 msg->set_output_data(frame->data_, data_size);
peahdf3efa82015-11-28 12:35:15 -0800677 RETURN_ON_ERR(WriteMessageToDebugFile(debug_dump_.debug_file.get(),
ivocd66b44d2016-01-15 03:06:36 -0800678 &debug_dump_.num_bytes_left_for_log_,
peahdf3efa82015-11-28 12:35:15 -0800679 &crit_debug_, &debug_dump_.capture));
andrew@webrtc.org17e40642014-03-04 20:58:13 +0000680 }
681#endif
682
683 return kNoError;
684}
685
andrew@webrtc.org17e40642014-03-04 20:58:13 +0000686int AudioProcessingImpl::ProcessStreamLocked() {
peahb58a1582016-03-15 09:34:24 -0700687 // Ensure that not both the AEC and AECM are active at the same time.
688 // TODO(peah): Simplify once the public API Enable functions for these
689 // are moved to APM.
690 RTC_DCHECK(!(public_submodules_->echo_cancellation->is_enabled() &&
691 public_submodules_->echo_control_mobile->is_enabled()));
692
andrew@webrtc.org17e40642014-03-04 20:58:13 +0000693#ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP
tommia6219cc2016-06-15 10:30:14 -0700694 if (debug_dump_.debug_file->is_open()) {
peahdf3efa82015-11-28 12:35:15 -0800695 audioproc::Stream* msg = debug_dump_.capture.event_msg->mutable_stream();
696 msg->set_delay(capture_nonlocked_.stream_delay_ms);
697 msg->set_drift(
698 public_submodules_->echo_cancellation->stream_drift_samples());
bjornv@webrtc.org63da1dd2015-02-06 19:44:21 +0000699 msg->set_level(gain_control()->stream_analog_level());
peahdf3efa82015-11-28 12:35:15 -0800700 msg->set_keypress(capture_.key_pressed);
niklase@google.com470e71d2011-07-07 08:21:25 +0000701 }
andrew@webrtc.org7bf26462011-12-03 00:03:31 +0000702#endif
niklase@google.com470e71d2011-07-07 08:21:25 +0000703
Bjorn Volcker1ca324f2015-06-29 14:57:29 +0200704 MaybeUpdateHistograms();
705
peahdf3efa82015-11-28 12:35:15 -0800706 AudioBuffer* ca = capture_.capture_audio.get(); // For brevity.
ekmeyerson60d9b332015-08-14 10:35:55 -0700707
peahbe615622016-02-13 16:40:47 -0800708 if (constants_.use_experimental_agc &&
peahdf3efa82015-11-28 12:35:15 -0800709 public_submodules_->gain_control->is_enabled()) {
710 private_submodules_->agc_manager->AnalyzePreProcess(
711 ca->channels()[0], ca->num_channels(),
712 capture_nonlocked_.fwd_proc_format.num_frames());
pbos@webrtc.org788acd12014-12-15 09:41:24 +0000713 }
714
aluebsdf6416a2016-03-16 18:26:35 -0700715 if (fwd_analysis_needed()) {
aluebs@webrtc.orgbe05c742014-11-14 22:18:10 +0000716 ca->SplitIntoFrequencyBands();
niklase@google.com470e71d2011-07-07 08:21:25 +0000717 }
718
aluebsb2328d12016-01-11 20:32:29 -0800719 if (capture_nonlocked_.beamformer_enabled) {
Alejandro Luebsf4022ff2016-07-01 17:19:09 -0700720 private_submodules_->beamformer->AnalyzeChunk(*ca->split_data_f());
721 // Discards all channels by the leftmost one.
aluebs@webrtc.orgae643ce2014-12-19 19:57:34 +0000722 ca->set_num_channels(1);
723 }
aluebs@webrtc.orgae643ce2014-12-19 19:57:34 +0000724
solenberg70f99032015-12-08 11:07:32 -0800725 public_submodules_->high_pass_filter->ProcessCaptureAudio(ca);
peahdf3efa82015-11-28 12:35:15 -0800726 RETURN_ON_ERR(public_submodules_->gain_control->AnalyzeCaptureAudio(ca));
solenberg5e465c32015-12-08 13:22:33 -0800727 public_submodules_->noise_suppression->AnalyzeCaptureAudio(ca);
peahb58a1582016-03-15 09:34:24 -0700728
729 // Ensure that the stream delay was set before the call to the
730 // AEC ProcessCaptureAudio function.
731 if (public_submodules_->echo_cancellation->is_enabled() &&
732 !was_stream_delay_set()) {
733 return AudioProcessing::kStreamParameterNotSetError;
734 }
735
736 RETURN_ON_ERR(public_submodules_->echo_cancellation->ProcessCaptureAudio(
737 ca, stream_delay_ms()));
niklase@google.com470e71d2011-07-07 08:21:25 +0000738
peahdf3efa82015-11-28 12:35:15 -0800739 if (public_submodules_->echo_control_mobile->is_enabled() &&
740 public_submodules_->noise_suppression->is_enabled()) {
andrew@webrtc.org103657b2014-04-24 18:28:56 +0000741 ca->CopyLowPassToReference();
niklase@google.com470e71d2011-07-07 08:21:25 +0000742 }
solenberg5e465c32015-12-08 13:22:33 -0800743 public_submodules_->noise_suppression->ProcessCaptureAudio(ca);
peah1bcfce52016-08-26 07:16:04 -0700744#if WEBRTC_INTELLIGIBILITY_ENHANCER
Alejandro Luebsc9b0c262016-05-16 15:32:38 -0700745 if (capture_nonlocked_.intelligibility_enabled) {
aluebsc466bad2016-02-10 12:03:00 -0800746 RTC_DCHECK(public_submodules_->noise_suppression->is_enabled());
Alejandro Luebsc9b0c262016-05-16 15:32:38 -0700747 int gain_db = public_submodules_->gain_control->is_enabled() ?
748 public_submodules_->gain_control->compression_gain_db() :
749 0;
Alejandro Luebs50411102016-06-30 15:35:41 -0700750 float gain = std::pow(10.f, gain_db / 20.f);
751 gain *= capture_nonlocked_.level_controller_enabled ?
752 private_submodules_->level_controller->GetLastGain() :
753 1.f;
aluebsc466bad2016-02-10 12:03:00 -0800754 public_submodules_->intelligibility_enhancer->SetCaptureNoiseEstimate(
Alejandro Luebs50411102016-06-30 15:35:41 -0700755 public_submodules_->noise_suppression->NoiseEstimate(), gain);
aluebsc466bad2016-02-10 12:03:00 -0800756 }
peah1bcfce52016-08-26 07:16:04 -0700757#endif
peah253534d2016-03-15 04:32:28 -0700758
759 // Ensure that the stream delay was set before the call to the
760 // AECM ProcessCaptureAudio function.
761 if (public_submodules_->echo_control_mobile->is_enabled() &&
762 !was_stream_delay_set()) {
763 return AudioProcessing::kStreamParameterNotSetError;
764 }
765
766 RETURN_ON_ERR(public_submodules_->echo_control_mobile->ProcessCaptureAudio(
767 ca, stream_delay_ms()));
768
Alejandro Luebsf4022ff2016-07-01 17:19:09 -0700769 if (capture_nonlocked_.beamformer_enabled) {
770 private_submodules_->beamformer->PostFilter(ca->split_data_f());
771 }
772
solenberga29386c2015-12-16 03:31:12 -0800773 public_submodules_->voice_detection->ProcessCaptureAudio(ca);
pbos@webrtc.org788acd12014-12-15 09:41:24 +0000774
peahbe615622016-02-13 16:40:47 -0800775 if (constants_.use_experimental_agc &&
peahdf3efa82015-11-28 12:35:15 -0800776 public_submodules_->gain_control->is_enabled() &&
aluebsb2328d12016-01-11 20:32:29 -0800777 (!capture_nonlocked_.beamformer_enabled ||
peahdf3efa82015-11-28 12:35:15 -0800778 private_submodules_->beamformer->is_target_present())) {
779 private_submodules_->agc_manager->Process(
780 ca->split_bands_const(0)[kBand0To8kHz], ca->num_frames_per_band(),
781 capture_nonlocked_.split_rate);
pbos@webrtc.org788acd12014-12-15 09:41:24 +0000782 }
peahb8fbb542016-03-15 02:28:08 -0700783 RETURN_ON_ERR(public_submodules_->gain_control->ProcessCaptureAudio(
784 ca, echo_cancellation()->stream_has_echo()));
niklase@google.com470e71d2011-07-07 08:21:25 +0000785
aluebsdf6416a2016-03-16 18:26:35 -0700786 if (fwd_synthesis_needed()) {
aluebs@webrtc.orgbe05c742014-11-14 22:18:10 +0000787 ca->MergeFrequencyBands();
niklase@google.com470e71d2011-07-07 08:21:25 +0000788 }
789
pbos@webrtc.org788acd12014-12-15 09:41:24 +0000790 // TODO(aluebs): Investigate if the transient suppression placement should be
791 // before or after the AGC.
peahdf3efa82015-11-28 12:35:15 -0800792 if (capture_.transient_suppressor_enabled) {
pbos@webrtc.org788acd12014-12-15 09:41:24 +0000793 float voice_probability =
peahdf3efa82015-11-28 12:35:15 -0800794 private_submodules_->agc_manager.get()
795 ? private_submodules_->agc_manager->voice_probability()
796 : 1.f;
pbos@webrtc.org788acd12014-12-15 09:41:24 +0000797
peahdf3efa82015-11-28 12:35:15 -0800798 public_submodules_->transient_suppressor->Suppress(
Michael Graczyk86c6d332015-07-23 11:41:39 -0700799 ca->channels_f()[0], ca->num_frames(), ca->num_channels(),
800 ca->split_bands_const_f(0)[kBand0To8kHz], ca->num_frames_per_band(),
801 ca->keyboard_data(), ca->num_keyboard_frames(), voice_probability,
peahdf3efa82015-11-28 12:35:15 -0800802 capture_.key_pressed);
pbos@webrtc.org788acd12014-12-15 09:41:24 +0000803 }
804
peahca4cac72016-06-29 15:26:12 -0700805 if (capture_nonlocked_.level_controller_enabled) {
806 private_submodules_->level_controller->Process(ca);
807 }
808
andrew@webrtc.org755b04a2011-11-15 16:57:56 +0000809 // The level estimator operates on the recombined data.
solenberg949028f2015-12-15 11:39:38 -0800810 public_submodules_->level_estimator->ProcessStream(ca);
ajm@google.com808e0e02011-08-03 21:08:51 +0000811
peahdf3efa82015-11-28 12:35:15 -0800812 capture_.was_stream_delay_set = false;
niklase@google.com470e71d2011-07-07 08:21:25 +0000813 return kNoError;
814}
815
andrew@webrtc.org17e40642014-03-04 20:58:13 +0000816int AudioProcessingImpl::AnalyzeReverseStream(const float* const* data,
Peter Kastingdce40cf2015-08-24 14:52:23 -0700817 size_t samples_per_channel,
ekmeyerson60d9b332015-08-14 10:35:55 -0700818 int rev_sample_rate_hz,
andrew@webrtc.org17e40642014-03-04 20:58:13 +0000819 ChannelLayout layout) {
peah369f8282015-12-17 06:42:29 -0800820 TRACE_EVENT0("webrtc", "AudioProcessing::AnalyzeReverseStream_ChannelLayout");
peahdf3efa82015-11-28 12:35:15 -0800821 rtc::CritScope cs(&crit_render_);
Michael Graczyk86c6d332015-07-23 11:41:39 -0700822 const StreamConfig reverse_config = {
ekmeyerson60d9b332015-08-14 10:35:55 -0700823 rev_sample_rate_hz, ChannelsFromLayout(layout), LayoutHasKeyboard(layout),
Michael Graczyk86c6d332015-07-23 11:41:39 -0700824 };
825 if (samples_per_channel != reverse_config.num_frames()) {
826 return kBadDataLengthError;
827 }
peahdf3efa82015-11-28 12:35:15 -0800828 return AnalyzeReverseStreamLocked(data, reverse_config, reverse_config);
ekmeyerson60d9b332015-08-14 10:35:55 -0700829}
830
831int AudioProcessingImpl::ProcessReverseStream(
832 const float* const* src,
833 const StreamConfig& reverse_input_config,
834 const StreamConfig& reverse_output_config,
835 float* const* dest) {
peah369f8282015-12-17 06:42:29 -0800836 TRACE_EVENT0("webrtc", "AudioProcessing::ProcessReverseStream_StreamConfig");
peahdf3efa82015-11-28 12:35:15 -0800837 rtc::CritScope cs(&crit_render_);
838 RETURN_ON_ERR(AnalyzeReverseStreamLocked(src, reverse_input_config,
839 reverse_output_config));
ekmeyerson60d9b332015-08-14 10:35:55 -0700840 if (is_rev_processed()) {
peahdf3efa82015-11-28 12:35:15 -0800841 render_.render_audio->CopyTo(formats_.api_format.reverse_output_stream(),
842 dest);
peah81b9bfe2015-11-27 02:47:28 -0800843 } else if (render_check_rev_conversion_needed()) {
peahdf3efa82015-11-28 12:35:15 -0800844 render_.render_converter->Convert(src, reverse_input_config.num_samples(),
845 dest,
846 reverse_output_config.num_samples());
ekmeyerson60d9b332015-08-14 10:35:55 -0700847 } else {
848 CopyAudioIfNeeded(src, reverse_input_config.num_frames(),
849 reverse_input_config.num_channels(), dest);
850 }
851
852 return kNoError;
Michael Graczyk86c6d332015-07-23 11:41:39 -0700853}
854
peahdf3efa82015-11-28 12:35:15 -0800855int AudioProcessingImpl::AnalyzeReverseStreamLocked(
ekmeyerson60d9b332015-08-14 10:35:55 -0700856 const float* const* src,
857 const StreamConfig& reverse_input_config,
858 const StreamConfig& reverse_output_config) {
peahdf3efa82015-11-28 12:35:15 -0800859 if (src == nullptr) {
andrew@webrtc.org17e40642014-03-04 20:58:13 +0000860 return kNullPointerError;
861 }
andrew@webrtc.org17e40642014-03-04 20:58:13 +0000862
Peter Kasting69558702016-01-12 16:26:35 -0800863 if (reverse_input_config.num_channels() == 0) {
Michael Graczyk86c6d332015-07-23 11:41:39 -0700864 return kBadNumberChannelsError;
andrew@webrtc.org17e40642014-03-04 20:58:13 +0000865 }
866
peahdf3efa82015-11-28 12:35:15 -0800867 ProcessingConfig processing_config = formats_.api_format;
ekmeyerson60d9b332015-08-14 10:35:55 -0700868 processing_config.reverse_input_stream() = reverse_input_config;
869 processing_config.reverse_output_stream() = reverse_output_config;
Michael Graczyk86c6d332015-07-23 11:41:39 -0700870
peahdf3efa82015-11-28 12:35:15 -0800871 RETURN_ON_ERR(MaybeInitializeRender(processing_config));
ekmeyerson60d9b332015-08-14 10:35:55 -0700872 assert(reverse_input_config.num_frames() ==
peahdf3efa82015-11-28 12:35:15 -0800873 formats_.api_format.reverse_input_stream().num_frames());
Michael Graczyk86c6d332015-07-23 11:41:39 -0700874
andrew@webrtc.org17e40642014-03-04 20:58:13 +0000875#ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP
tommia6219cc2016-06-15 10:30:14 -0700876 if (debug_dump_.debug_file->is_open()) {
peahdf3efa82015-11-28 12:35:15 -0800877 debug_dump_.render.event_msg->set_type(audioproc::Event::REVERSE_STREAM);
878 audioproc::ReverseStream* msg =
879 debug_dump_.render.event_msg->mutable_reverse_stream();
aluebs@webrtc.org59a1b1b2014-08-28 10:43:09 +0000880 const size_t channel_size =
peahdf3efa82015-11-28 12:35:15 -0800881 sizeof(float) * formats_.api_format.reverse_input_stream().num_frames();
Peter Kasting69558702016-01-12 16:26:35 -0800882 for (size_t i = 0;
peahdf3efa82015-11-28 12:35:15 -0800883 i < formats_.api_format.reverse_input_stream().num_channels(); ++i)
ekmeyerson60d9b332015-08-14 10:35:55 -0700884 msg->add_channel(src[i], channel_size);
peahdf3efa82015-11-28 12:35:15 -0800885 RETURN_ON_ERR(WriteMessageToDebugFile(debug_dump_.debug_file.get(),
ivocd66b44d2016-01-15 03:06:36 -0800886 &debug_dump_.num_bytes_left_for_log_,
peahdf3efa82015-11-28 12:35:15 -0800887 &crit_debug_, &debug_dump_.render));
andrew@webrtc.org17e40642014-03-04 20:58:13 +0000888 }
889#endif
890
peahdf3efa82015-11-28 12:35:15 -0800891 render_.render_audio->CopyFrom(src,
892 formats_.api_format.reverse_input_stream());
ekmeyerson60d9b332015-08-14 10:35:55 -0700893 return ProcessReverseStreamLocked();
894}
895
896int AudioProcessingImpl::ProcessReverseStream(AudioFrame* frame) {
peah369f8282015-12-17 06:42:29 -0800897 TRACE_EVENT0("webrtc", "AudioProcessing::ProcessReverseStream_AudioFrame");
peahdf3efa82015-11-28 12:35:15 -0800898 rtc::CritScope cs(&crit_render_);
peahdf3efa82015-11-28 12:35:15 -0800899 if (frame == nullptr) {
niklase@google.com470e71d2011-07-07 08:21:25 +0000900 return kNullPointerError;
901 }
andrew@webrtc.orgddbb8a22014-04-22 21:00:04 +0000902 // Must be a native rate.
903 if (frame->sample_rate_hz_ != kSampleRate8kHz &&
904 frame->sample_rate_hz_ != kSampleRate16kHz &&
aluebs@webrtc.org087da132014-11-17 23:01:23 +0000905 frame->sample_rate_hz_ != kSampleRate32kHz &&
906 frame->sample_rate_hz_ != kSampleRate48kHz) {
andrew@webrtc.orgddbb8a22014-04-22 21:00:04 +0000907 return kBadSampleRateError;
908 }
andrew@webrtc.orga8b97372014-03-10 22:26:12 +0000909
Michael Graczyk86c6d332015-07-23 11:41:39 -0700910 if (frame->num_channels_ <= 0) {
911 return kBadNumberChannelsError;
912 }
913
peahdf3efa82015-11-28 12:35:15 -0800914 ProcessingConfig processing_config = formats_.api_format;
ekmeyerson60d9b332015-08-14 10:35:55 -0700915 processing_config.reverse_input_stream().set_sample_rate_hz(
916 frame->sample_rate_hz_);
917 processing_config.reverse_input_stream().set_num_channels(
918 frame->num_channels_);
919 processing_config.reverse_output_stream().set_sample_rate_hz(
920 frame->sample_rate_hz_);
921 processing_config.reverse_output_stream().set_num_channels(
922 frame->num_channels_);
Michael Graczyk86c6d332015-07-23 11:41:39 -0700923
peahdf3efa82015-11-28 12:35:15 -0800924 RETURN_ON_ERR(MaybeInitializeRender(processing_config));
Michael Graczyk86c6d332015-07-23 11:41:39 -0700925 if (frame->samples_per_channel_ !=
peahdf3efa82015-11-28 12:35:15 -0800926 formats_.api_format.reverse_input_stream().num_frames()) {
andrew@webrtc.org17e40642014-03-04 20:58:13 +0000927 return kBadDataLengthError;
928 }
niklase@google.com470e71d2011-07-07 08:21:25 +0000929
andrew@webrtc.org7bf26462011-12-03 00:03:31 +0000930#ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP
tommia6219cc2016-06-15 10:30:14 -0700931 if (debug_dump_.debug_file->is_open()) {
peahdf3efa82015-11-28 12:35:15 -0800932 debug_dump_.render.event_msg->set_type(audioproc::Event::REVERSE_STREAM);
933 audioproc::ReverseStream* msg =
934 debug_dump_.render.event_msg->mutable_reverse_stream();
Michael Graczyk86c6d332015-07-23 11:41:39 -0700935 const size_t data_size =
936 sizeof(int16_t) * frame->samples_per_channel_ * frame->num_channels_;
andrew@webrtc.org63a50982012-05-02 23:56:37 +0000937 msg->set_data(frame->data_, data_size);
peahdf3efa82015-11-28 12:35:15 -0800938 RETURN_ON_ERR(WriteMessageToDebugFile(debug_dump_.debug_file.get(),
ivocd66b44d2016-01-15 03:06:36 -0800939 &debug_dump_.num_bytes_left_for_log_,
peahdf3efa82015-11-28 12:35:15 -0800940 &crit_debug_, &debug_dump_.render));
niklase@google.com470e71d2011-07-07 08:21:25 +0000941 }
andrew@webrtc.org7bf26462011-12-03 00:03:31 +0000942#endif
peahdf3efa82015-11-28 12:35:15 -0800943 render_.render_audio->DeinterleaveFrom(frame);
aluebsb0319552016-03-17 20:39:53 -0700944 RETURN_ON_ERR(ProcessReverseStreamLocked());
945 if (is_rev_processed()) {
946 render_.render_audio->InterleaveTo(frame, true);
947 }
948 return kNoError;
andrew@webrtc.org17e40642014-03-04 20:58:13 +0000949}
niklase@google.com470e71d2011-07-07 08:21:25 +0000950
ekmeyerson60d9b332015-08-14 10:35:55 -0700951int AudioProcessingImpl::ProcessReverseStreamLocked() {
peahdf3efa82015-11-28 12:35:15 -0800952 AudioBuffer* ra = render_.render_audio.get(); // For brevity.
aluebsdf6416a2016-03-16 18:26:35 -0700953 if (rev_analysis_needed()) {
aluebs@webrtc.orgbe05c742014-11-14 22:18:10 +0000954 ra->SplitIntoFrequencyBands();
niklase@google.com470e71d2011-07-07 08:21:25 +0000955 }
956
peah1bcfce52016-08-26 07:16:04 -0700957#if WEBRTC_INTELLIGIBILITY_ENHANCER
Alejandro Luebsc9b0c262016-05-16 15:32:38 -0700958 if (capture_nonlocked_.intelligibility_enabled) {
peahdf3efa82015-11-28 12:35:15 -0800959 public_submodules_->intelligibility_enhancer->ProcessRenderAudio(
960 ra->split_channels_f(kBand0To8kHz), capture_nonlocked_.split_rate,
961 ra->num_channels());
ekmeyerson60d9b332015-08-14 10:35:55 -0700962 }
peah1bcfce52016-08-26 07:16:04 -0700963#endif
ekmeyerson60d9b332015-08-14 10:35:55 -0700964
peahdf3efa82015-11-28 12:35:15 -0800965 RETURN_ON_ERR(public_submodules_->echo_cancellation->ProcessRenderAudio(ra));
966 RETURN_ON_ERR(
967 public_submodules_->echo_control_mobile->ProcessRenderAudio(ra));
peahbe615622016-02-13 16:40:47 -0800968 if (!constants_.use_experimental_agc) {
peahdf3efa82015-11-28 12:35:15 -0800969 RETURN_ON_ERR(public_submodules_->gain_control->ProcessRenderAudio(ra));
pbos@webrtc.org788acd12014-12-15 09:41:24 +0000970 }
niklase@google.com470e71d2011-07-07 08:21:25 +0000971
aluebsdf6416a2016-03-16 18:26:35 -0700972 if (rev_synthesis_needed()) {
ekmeyerson60d9b332015-08-14 10:35:55 -0700973 ra->MergeFrequencyBands();
974 }
975
andrew@webrtc.org17e40642014-03-04 20:58:13 +0000976 return kNoError;
niklase@google.com470e71d2011-07-07 08:21:25 +0000977}
978
979int AudioProcessingImpl::set_stream_delay_ms(int delay) {
peahdf3efa82015-11-28 12:35:15 -0800980 rtc::CritScope cs(&crit_capture_);
andrew@webrtc.org5f23d642012-05-29 21:14:06 +0000981 Error retval = kNoError;
peahdf3efa82015-11-28 12:35:15 -0800982 capture_.was_stream_delay_set = true;
983 delay += capture_.delay_offset_ms;
andrew@webrtc.org6f9f8172012-03-06 19:03:39 +0000984
niklase@google.com470e71d2011-07-07 08:21:25 +0000985 if (delay < 0) {
andrew@webrtc.org5f23d642012-05-29 21:14:06 +0000986 delay = 0;
987 retval = kBadStreamParameterWarning;
niklase@google.com470e71d2011-07-07 08:21:25 +0000988 }
989
990 // TODO(ajm): the max is rather arbitrarily chosen; investigate.
991 if (delay > 500) {
andrew@webrtc.org5f23d642012-05-29 21:14:06 +0000992 delay = 500;
993 retval = kBadStreamParameterWarning;
niklase@google.com470e71d2011-07-07 08:21:25 +0000994 }
995
peahdf3efa82015-11-28 12:35:15 -0800996 capture_nonlocked_.stream_delay_ms = delay;
andrew@webrtc.org5f23d642012-05-29 21:14:06 +0000997 return retval;
niklase@google.com470e71d2011-07-07 08:21:25 +0000998}
999
1000int AudioProcessingImpl::stream_delay_ms() const {
peahdf3efa82015-11-28 12:35:15 -08001001 // Used as callback from submodules, hence locking is not allowed.
1002 return capture_nonlocked_.stream_delay_ms;
niklase@google.com470e71d2011-07-07 08:21:25 +00001003}
1004
1005bool AudioProcessingImpl::was_stream_delay_set() const {
peahdf3efa82015-11-28 12:35:15 -08001006 // Used as callback from submodules, hence locking is not allowed.
1007 return capture_.was_stream_delay_set;
niklase@google.com470e71d2011-07-07 08:21:25 +00001008}
1009
andrew@webrtc.org17e40642014-03-04 20:58:13 +00001010void AudioProcessingImpl::set_stream_key_pressed(bool key_pressed) {
peahdf3efa82015-11-28 12:35:15 -08001011 rtc::CritScope cs(&crit_capture_);
1012 capture_.key_pressed = key_pressed;
andrew@webrtc.org17e40642014-03-04 20:58:13 +00001013}
1014
andrew@webrtc.org6f9f8172012-03-06 19:03:39 +00001015void AudioProcessingImpl::set_delay_offset_ms(int offset) {
peahdf3efa82015-11-28 12:35:15 -08001016 rtc::CritScope cs(&crit_capture_);
1017 capture_.delay_offset_ms = offset;
andrew@webrtc.org6f9f8172012-03-06 19:03:39 +00001018}
1019
1020int AudioProcessingImpl::delay_offset_ms() const {
peahdf3efa82015-11-28 12:35:15 -08001021 rtc::CritScope cs(&crit_capture_);
1022 return capture_.delay_offset_ms;
andrew@webrtc.org6f9f8172012-03-06 19:03:39 +00001023}
1024
niklase@google.com470e71d2011-07-07 08:21:25 +00001025int AudioProcessingImpl::StartDebugRecording(
ivocd66b44d2016-01-15 03:06:36 -08001026 const char filename[AudioProcessing::kMaxFilenameSize],
1027 int64_t max_log_size_bytes) {
peahdf3efa82015-11-28 12:35:15 -08001028 // Run in a single-threaded manner.
1029 rtc::CritScope cs_render(&crit_render_);
1030 rtc::CritScope cs_capture(&crit_capture_);
André Susano Pinto664cdaf2015-05-20 11:11:07 +02001031 static_assert(kMaxFilenameSize == FileWrapper::kMaxFileNameSize, "");
niklase@google.com470e71d2011-07-07 08:21:25 +00001032
peahdf3efa82015-11-28 12:35:15 -08001033 if (filename == nullptr) {
niklase@google.com470e71d2011-07-07 08:21:25 +00001034 return kNullPointerError;
1035 }
1036
andrew@webrtc.org7bf26462011-12-03 00:03:31 +00001037#ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP
ivocd66b44d2016-01-15 03:06:36 -08001038 debug_dump_.num_bytes_left_for_log_ = max_log_size_bytes;
niklase@google.com470e71d2011-07-07 08:21:25 +00001039 // Stop any ongoing recording.
tommia6219cc2016-06-15 10:30:14 -07001040 debug_dump_.debug_file->CloseFile();
niklase@google.com470e71d2011-07-07 08:21:25 +00001041
tommia6219cc2016-06-15 10:30:14 -07001042 if (!debug_dump_.debug_file->OpenFile(filename, false)) {
niklase@google.com470e71d2011-07-07 08:21:25 +00001043 return kFileError;
1044 }
1045
Minyue13b96ba2015-10-03 00:39:14 +02001046 RETURN_ON_ERR(WriteConfigMessage(true));
1047 RETURN_ON_ERR(WriteInitMessage());
niklase@google.com470e71d2011-07-07 08:21:25 +00001048 return kNoError;
andrew@webrtc.org7bf26462011-12-03 00:03:31 +00001049#else
1050 return kUnsupportedFunctionError;
1051#endif // WEBRTC_AUDIOPROC_DEBUG_DUMP
niklase@google.com470e71d2011-07-07 08:21:25 +00001052}
1053
ivocd66b44d2016-01-15 03:06:36 -08001054int AudioProcessingImpl::StartDebugRecording(FILE* handle,
1055 int64_t max_log_size_bytes) {
peahdf3efa82015-11-28 12:35:15 -08001056 // Run in a single-threaded manner.
1057 rtc::CritScope cs_render(&crit_render_);
1058 rtc::CritScope cs_capture(&crit_capture_);
henrikg@webrtc.org863b5362013-12-06 16:05:17 +00001059
peahdf3efa82015-11-28 12:35:15 -08001060 if (handle == nullptr) {
henrikg@webrtc.org863b5362013-12-06 16:05:17 +00001061 return kNullPointerError;
1062 }
1063
1064#ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP
ivocd66b44d2016-01-15 03:06:36 -08001065 debug_dump_.num_bytes_left_for_log_ = max_log_size_bytes;
1066
henrikg@webrtc.org863b5362013-12-06 16:05:17 +00001067 // Stop any ongoing recording.
tommia6219cc2016-06-15 10:30:14 -07001068 debug_dump_.debug_file->CloseFile();
henrikg@webrtc.org863b5362013-12-06 16:05:17 +00001069
tommia6219cc2016-06-15 10:30:14 -07001070 if (!debug_dump_.debug_file->OpenFromFileHandle(handle)) {
henrikg@webrtc.org863b5362013-12-06 16:05:17 +00001071 return kFileError;
1072 }
1073
Minyue13b96ba2015-10-03 00:39:14 +02001074 RETURN_ON_ERR(WriteConfigMessage(true));
1075 RETURN_ON_ERR(WriteInitMessage());
henrikg@webrtc.org863b5362013-12-06 16:05:17 +00001076 return kNoError;
1077#else
1078 return kUnsupportedFunctionError;
1079#endif // WEBRTC_AUDIOPROC_DEBUG_DUMP
1080}
1081
xians@webrtc.orge46bc772014-10-10 08:36:56 +00001082int AudioProcessingImpl::StartDebugRecordingForPlatformFile(
1083 rtc::PlatformFile handle) {
peahdf3efa82015-11-28 12:35:15 -08001084 // Run in a single-threaded manner.
1085 rtc::CritScope cs_render(&crit_render_);
1086 rtc::CritScope cs_capture(&crit_capture_);
xians@webrtc.orge46bc772014-10-10 08:36:56 +00001087 FILE* stream = rtc::FdopenPlatformFileForWriting(handle);
ivocd66b44d2016-01-15 03:06:36 -08001088 return StartDebugRecording(stream, -1);
xians@webrtc.orge46bc772014-10-10 08:36:56 +00001089}
1090
niklase@google.com470e71d2011-07-07 08:21:25 +00001091int AudioProcessingImpl::StopDebugRecording() {
peahdf3efa82015-11-28 12:35:15 -08001092 // Run in a single-threaded manner.
1093 rtc::CritScope cs_render(&crit_render_);
1094 rtc::CritScope cs_capture(&crit_capture_);
andrew@webrtc.org7bf26462011-12-03 00:03:31 +00001095
1096#ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP
niklase@google.com470e71d2011-07-07 08:21:25 +00001097 // We just return if recording hasn't started.
tommia6219cc2016-06-15 10:30:14 -07001098 debug_dump_.debug_file->CloseFile();
niklase@google.com470e71d2011-07-07 08:21:25 +00001099 return kNoError;
andrew@webrtc.org7bf26462011-12-03 00:03:31 +00001100#else
1101 return kUnsupportedFunctionError;
1102#endif // WEBRTC_AUDIOPROC_DEBUG_DUMP
niklase@google.com470e71d2011-07-07 08:21:25 +00001103}
1104
1105EchoCancellation* AudioProcessingImpl::echo_cancellation() const {
peahdf3efa82015-11-28 12:35:15 -08001106 // Adding a lock here has no effect as it allows any access to the submodule
1107 // from the returned pointer.
peahb624d8c2016-03-05 03:01:14 -08001108 return public_submodules_->echo_cancellation.get();
niklase@google.com470e71d2011-07-07 08:21:25 +00001109}
1110
1111EchoControlMobile* AudioProcessingImpl::echo_control_mobile() const {
peahdf3efa82015-11-28 12:35:15 -08001112 // Adding a lock here has no effect as it allows any access to the submodule
1113 // from the returned pointer.
peahbb9edbd2016-03-10 12:54:25 -08001114 return public_submodules_->echo_control_mobile.get();
niklase@google.com470e71d2011-07-07 08:21:25 +00001115}
1116
1117GainControl* AudioProcessingImpl::gain_control() const {
peahdf3efa82015-11-28 12:35:15 -08001118 // Adding a lock here has no effect as it allows any access to the submodule
1119 // from the returned pointer.
peahbe615622016-02-13 16:40:47 -08001120 if (constants_.use_experimental_agc) {
1121 return public_submodules_->gain_control_for_experimental_agc.get();
pbos@webrtc.org788acd12014-12-15 09:41:24 +00001122 }
peahbfa97112016-03-10 21:09:04 -08001123 return public_submodules_->gain_control.get();
niklase@google.com470e71d2011-07-07 08:21:25 +00001124}
1125
1126HighPassFilter* AudioProcessingImpl::high_pass_filter() const {
peahdf3efa82015-11-28 12:35:15 -08001127 // Adding a lock here has no effect as it allows any access to the submodule
1128 // from the returned pointer.
solenberg70f99032015-12-08 11:07:32 -08001129 return public_submodules_->high_pass_filter.get();
niklase@google.com470e71d2011-07-07 08:21:25 +00001130}
1131
1132LevelEstimator* AudioProcessingImpl::level_estimator() const {
peahdf3efa82015-11-28 12:35:15 -08001133 // Adding a lock here has no effect as it allows any access to the submodule
1134 // from the returned pointer.
solenberg949028f2015-12-15 11:39:38 -08001135 return public_submodules_->level_estimator.get();
niklase@google.com470e71d2011-07-07 08:21:25 +00001136}
1137
1138NoiseSuppression* AudioProcessingImpl::noise_suppression() const {
peahdf3efa82015-11-28 12:35:15 -08001139 // Adding a lock here has no effect as it allows any access to the submodule
1140 // from the returned pointer.
solenberg5e465c32015-12-08 13:22:33 -08001141 return public_submodules_->noise_suppression.get();
niklase@google.com470e71d2011-07-07 08:21:25 +00001142}
1143
1144VoiceDetection* AudioProcessingImpl::voice_detection() const {
peahdf3efa82015-11-28 12:35:15 -08001145 // Adding a lock here has no effect as it allows any access to the submodule
1146 // from the returned pointer.
solenberga29386c2015-12-16 03:31:12 -08001147 return public_submodules_->voice_detection.get();
niklase@google.com470e71d2011-07-07 08:21:25 +00001148}
1149
Alejandro Luebs63a2c132016-03-31 18:04:40 -07001150bool AudioProcessingImpl::is_fwd_processed() const {
peah253d8fa2016-02-22 02:00:09 -08001151 // The beamformer, noise suppressor and highpass filter
1152 // modify the data.
1153 if (capture_nonlocked_.beamformer_enabled ||
1154 public_submodules_->high_pass_filter->is_enabled() ||
peahb624d8c2016-03-05 03:01:14 -08001155 public_submodules_->noise_suppression->is_enabled() ||
peahbb9edbd2016-03-10 12:54:25 -08001156 public_submodules_->echo_cancellation->is_enabled() ||
peahbfa97112016-03-10 21:09:04 -08001157 public_submodules_->echo_control_mobile->is_enabled() ||
1158 public_submodules_->gain_control->is_enabled()) {
aluebs@webrtc.orgae643ce2014-12-19 19:57:34 +00001159 return true;
1160 }
1161
peah253d8fa2016-02-22 02:00:09 -08001162 // The capture data is otherwise unchanged.
1163 return false;
andrew@webrtc.org7bf26462011-12-03 00:03:31 +00001164}
1165
aluebsdf6416a2016-03-16 18:26:35 -07001166bool AudioProcessingImpl::output_copy_needed() const {
andrew@webrtc.org369166a2012-04-24 18:38:03 +00001167 // Check if we've upmixed or downmixed the audio.
peahdf3efa82015-11-28 12:35:15 -08001168 return ((formats_.api_format.output_stream().num_channels() !=
1169 formats_.api_format.input_stream().num_channels()) ||
peahca4cac72016-06-29 15:26:12 -07001170 is_fwd_processed() || capture_.transient_suppressor_enabled ||
1171 capture_nonlocked_.level_controller_enabled);
andrew@webrtc.org7bf26462011-12-03 00:03:31 +00001172}
1173
aluebsdf6416a2016-03-16 18:26:35 -07001174bool AudioProcessingImpl::fwd_synthesis_needed() const {
Alejandro Luebs63a2c132016-03-31 18:04:40 -07001175 return (is_fwd_processed() &&
aluebsdf6416a2016-03-16 18:26:35 -07001176 is_multi_band(capture_nonlocked_.fwd_proc_format.sample_rate_hz()));
andrew@webrtc.org369166a2012-04-24 18:38:03 +00001177}
1178
aluebsdf6416a2016-03-16 18:26:35 -07001179bool AudioProcessingImpl::fwd_analysis_needed() const {
Alejandro Luebs63a2c132016-03-31 18:04:40 -07001180 if (!is_fwd_processed() &&
peahdf3efa82015-11-28 12:35:15 -08001181 !public_submodules_->voice_detection->is_enabled() &&
1182 !capture_.transient_suppressor_enabled) {
1183 // Only public_submodules_->level_estimator is enabled.
andrew@webrtc.org7bf26462011-12-03 00:03:31 +00001184 return false;
aluebsdf6416a2016-03-16 18:26:35 -07001185 } else if (is_multi_band(
1186 capture_nonlocked_.fwd_proc_format.sample_rate_hz())) {
peahdf3efa82015-11-28 12:35:15 -08001187 // Something besides public_submodules_->level_estimator is enabled, and we
1188 // have super-wb.
andrew@webrtc.org7bf26462011-12-03 00:03:31 +00001189 return true;
1190 }
1191 return false;
1192}
1193
ekmeyerson60d9b332015-08-14 10:35:55 -07001194bool AudioProcessingImpl::is_rev_processed() const {
peah1bcfce52016-08-26 07:16:04 -07001195#if WEBRTC_INTELLIGIBILITY_ENHANCER
Alejandro Luebsc9b0c262016-05-16 15:32:38 -07001196 return capture_nonlocked_.intelligibility_enabled;
peah1bcfce52016-08-26 07:16:04 -07001197#else
1198 return false;
1199#endif
ekmeyerson60d9b332015-08-14 10:35:55 -07001200}
1201
aluebsdf6416a2016-03-16 18:26:35 -07001202bool AudioProcessingImpl::rev_synthesis_needed() const {
1203 return (is_rev_processed() &&
aluebseb3603b2016-04-20 15:27:58 -07001204 is_multi_band(formats_.rev_proc_format.sample_rate_hz()));
aluebsdf6416a2016-03-16 18:26:35 -07001205}
1206
1207bool AudioProcessingImpl::rev_analysis_needed() const {
aluebseb3603b2016-04-20 15:27:58 -07001208 return is_multi_band(formats_.rev_proc_format.sample_rate_hz()) &&
Alejandro Luebs63a2c132016-03-31 18:04:40 -07001209 (is_rev_processed() ||
peahdc2242d2016-04-06 09:30:58 -07001210 public_submodules_->echo_cancellation
1211 ->is_enabled_render_side_query() ||
1212 public_submodules_->echo_control_mobile
1213 ->is_enabled_render_side_query() ||
1214 public_submodules_->gain_control->is_enabled_render_side_query());
aluebsdf6416a2016-03-16 18:26:35 -07001215}
1216
peah81b9bfe2015-11-27 02:47:28 -08001217bool AudioProcessingImpl::render_check_rev_conversion_needed() const {
1218 return rev_conversion_needed();
1219}
1220
ekmeyerson60d9b332015-08-14 10:35:55 -07001221bool AudioProcessingImpl::rev_conversion_needed() const {
peahdf3efa82015-11-28 12:35:15 -08001222 return (formats_.api_format.reverse_input_stream() !=
1223 formats_.api_format.reverse_output_stream());
ekmeyerson60d9b332015-08-14 10:35:55 -07001224}
1225
Bjorn Volckeradc46c42015-04-15 11:42:40 +02001226void AudioProcessingImpl::InitializeExperimentalAgc() {
peahbe615622016-02-13 16:40:47 -08001227 if (constants_.use_experimental_agc) {
peahdf3efa82015-11-28 12:35:15 -08001228 if (!private_submodules_->agc_manager.get()) {
1229 private_submodules_->agc_manager.reset(new AgcManagerDirect(
peahbfa97112016-03-10 21:09:04 -08001230 public_submodules_->gain_control.get(),
peahbe615622016-02-13 16:40:47 -08001231 public_submodules_->gain_control_for_experimental_agc.get(),
peahdf3efa82015-11-28 12:35:15 -08001232 constants_.agc_startup_min_volume));
pbos@webrtc.org788acd12014-12-15 09:41:24 +00001233 }
peahdf3efa82015-11-28 12:35:15 -08001234 private_submodules_->agc_manager->Initialize();
1235 private_submodules_->agc_manager->SetCaptureMuted(
1236 capture_.output_will_be_muted);
pbos@webrtc.org788acd12014-12-15 09:41:24 +00001237 }
pbos@webrtc.org788acd12014-12-15 09:41:24 +00001238}
1239
Bjorn Volckeradc46c42015-04-15 11:42:40 +02001240void AudioProcessingImpl::InitializeTransient() {
peahdf3efa82015-11-28 12:35:15 -08001241 if (capture_.transient_suppressor_enabled) {
1242 if (!public_submodules_->transient_suppressor.get()) {
1243 public_submodules_->transient_suppressor.reset(new TransientSuppressor());
pbos@webrtc.org788acd12014-12-15 09:41:24 +00001244 }
peahdf3efa82015-11-28 12:35:15 -08001245 public_submodules_->transient_suppressor->Initialize(
1246 capture_nonlocked_.fwd_proc_format.sample_rate_hz(),
1247 capture_nonlocked_.split_rate,
aluebsb2328d12016-01-11 20:32:29 -08001248 num_proc_channels());
pbos@webrtc.org788acd12014-12-15 09:41:24 +00001249 }
pbos@webrtc.org788acd12014-12-15 09:41:24 +00001250}
1251
aluebs@webrtc.orgae643ce2014-12-19 19:57:34 +00001252void AudioProcessingImpl::InitializeBeamformer() {
aluebsb2328d12016-01-11 20:32:29 -08001253 if (capture_nonlocked_.beamformer_enabled) {
peahdf3efa82015-11-28 12:35:15 -08001254 if (!private_submodules_->beamformer) {
1255 private_submodules_->beamformer.reset(new NonlinearBeamformer(
Alejandro Luebsf4022ff2016-07-01 17:19:09 -07001256 capture_.array_geometry, 1u, capture_.target_direction));
aluebs@webrtc.orgd82f55d2015-01-15 18:07:21 +00001257 }
peahdf3efa82015-11-28 12:35:15 -08001258 private_submodules_->beamformer->Initialize(kChunkSizeMs,
1259 capture_nonlocked_.split_rate);
aluebs@webrtc.orgae643ce2014-12-19 19:57:34 +00001260 }
1261}
1262
ekmeyerson60d9b332015-08-14 10:35:55 -07001263void AudioProcessingImpl::InitializeIntelligibility() {
peah1bcfce52016-08-26 07:16:04 -07001264#if WEBRTC_INTELLIGIBILITY_ENHANCER
Alejandro Luebsc9b0c262016-05-16 15:32:38 -07001265 if (capture_nonlocked_.intelligibility_enabled) {
peahdf3efa82015-11-28 12:35:15 -08001266 public_submodules_->intelligibility_enhancer.reset(
Alejandro Luebs18fcbcf2016-02-22 15:57:38 -08001267 new IntelligibilityEnhancer(capture_nonlocked_.split_rate,
Alex Luebs57ae8292016-03-09 16:24:34 +01001268 render_.render_audio->num_channels(),
1269 NoiseSuppressionImpl::num_noise_bins()));
ekmeyerson60d9b332015-08-14 10:35:55 -07001270 }
peah1bcfce52016-08-26 07:16:04 -07001271#endif
ekmeyerson60d9b332015-08-14 10:35:55 -07001272}
1273
solenberg70f99032015-12-08 11:07:32 -08001274void AudioProcessingImpl::InitializeHighPassFilter() {
aluebsb2328d12016-01-11 20:32:29 -08001275 public_submodules_->high_pass_filter->Initialize(num_proc_channels(),
solenberg70f99032015-12-08 11:07:32 -08001276 proc_sample_rate_hz());
1277}
1278
solenberg5e465c32015-12-08 13:22:33 -08001279void AudioProcessingImpl::InitializeNoiseSuppression() {
aluebsb2328d12016-01-11 20:32:29 -08001280 public_submodules_->noise_suppression->Initialize(num_proc_channels(),
solenberg5e465c32015-12-08 13:22:33 -08001281 proc_sample_rate_hz());
1282}
1283
peahb624d8c2016-03-05 03:01:14 -08001284void AudioProcessingImpl::InitializeEchoCanceller() {
peahb58a1582016-03-15 09:34:24 -07001285 public_submodules_->echo_cancellation->Initialize(
1286 proc_sample_rate_hz(), num_reverse_channels(), num_output_channels(),
1287 num_proc_channels());
peahb624d8c2016-03-05 03:01:14 -08001288}
1289
peahbfa97112016-03-10 21:09:04 -08001290void AudioProcessingImpl::InitializeGainController() {
peahb8fbb542016-03-15 02:28:08 -07001291 public_submodules_->gain_control->Initialize(num_proc_channels(),
1292 proc_sample_rate_hz());
peahbfa97112016-03-10 21:09:04 -08001293}
1294
peahbb9edbd2016-03-10 12:54:25 -08001295void AudioProcessingImpl::InitializeEchoControlMobile() {
peah253534d2016-03-15 04:32:28 -07001296 public_submodules_->echo_control_mobile->Initialize(
aluebs776593b2016-03-15 14:04:58 -07001297 proc_split_sample_rate_hz(),
1298 num_reverse_channels(),
1299 num_output_channels());
peahbb9edbd2016-03-10 12:54:25 -08001300}
1301
solenberg949028f2015-12-15 11:39:38 -08001302void AudioProcessingImpl::InitializeLevelEstimator() {
1303 public_submodules_->level_estimator->Initialize();
1304}
1305
peahca4cac72016-06-29 15:26:12 -07001306void AudioProcessingImpl::InitializeLevelController() {
1307 private_submodules_->level_controller->Initialize(proc_sample_rate_hz());
1308}
1309
solenberga29386c2015-12-16 03:31:12 -08001310void AudioProcessingImpl::InitializeVoiceDetection() {
1311 public_submodules_->voice_detection->Initialize(proc_split_sample_rate_hz());
1312}
1313
Bjorn Volcker1ca324f2015-06-29 14:57:29 +02001314void AudioProcessingImpl::MaybeUpdateHistograms() {
Bjorn Volckerd92f2672015-07-05 10:46:01 +02001315 static const int kMinDiffDelayMs = 60;
Bjorn Volcker1ca324f2015-06-29 14:57:29 +02001316
1317 if (echo_cancellation()->is_enabled()) {
Bjorn Volcker4e7aa432015-07-07 11:50:05 +02001318 // Activate delay_jumps_ counters if we know echo_cancellation is runnning.
1319 // If a stream has echo we know that the echo_cancellation is in process.
peahdf3efa82015-11-28 12:35:15 -08001320 if (capture_.stream_delay_jumps == -1 &&
Bjorn Volcker4e7aa432015-07-07 11:50:05 +02001321 echo_cancellation()->stream_has_echo()) {
peahdf3efa82015-11-28 12:35:15 -08001322 capture_.stream_delay_jumps = 0;
1323 }
1324 if (capture_.aec_system_delay_jumps == -1 &&
1325 echo_cancellation()->stream_has_echo()) {
1326 capture_.aec_system_delay_jumps = 0;
Bjorn Volcker4e7aa432015-07-07 11:50:05 +02001327 }
1328
Bjorn Volcker1ca324f2015-06-29 14:57:29 +02001329 // Detect a jump in platform reported system delay and log the difference.
peahdf3efa82015-11-28 12:35:15 -08001330 const int diff_stream_delay_ms =
1331 capture_nonlocked_.stream_delay_ms - capture_.last_stream_delay_ms;
1332 if (diff_stream_delay_ms > kMinDiffDelayMs &&
1333 capture_.last_stream_delay_ms != 0) {
asaperssona2c58e22016-03-07 01:52:59 -08001334 RTC_HISTOGRAM_COUNTS("WebRTC.Audio.PlatformReportedStreamDelayJump",
1335 diff_stream_delay_ms, kMinDiffDelayMs, 1000, 100);
peahdf3efa82015-11-28 12:35:15 -08001336 if (capture_.stream_delay_jumps == -1) {
1337 capture_.stream_delay_jumps = 0; // Activate counter if needed.
Bjorn Volcker4e7aa432015-07-07 11:50:05 +02001338 }
peahdf3efa82015-11-28 12:35:15 -08001339 capture_.stream_delay_jumps++;
Bjorn Volcker1ca324f2015-06-29 14:57:29 +02001340 }
peahdf3efa82015-11-28 12:35:15 -08001341 capture_.last_stream_delay_ms = capture_nonlocked_.stream_delay_ms;
Bjorn Volcker1ca324f2015-06-29 14:57:29 +02001342
1343 // Detect a jump in AEC system delay and log the difference.
peah20028c42016-03-04 11:50:54 -08001344 const int samples_per_ms =
peahdf3efa82015-11-28 12:35:15 -08001345 rtc::CheckedDivExact(capture_nonlocked_.split_rate, 1000);
peah20028c42016-03-04 11:50:54 -08001346 RTC_DCHECK_LT(0, samples_per_ms);
Bjorn Volcker1ca324f2015-06-29 14:57:29 +02001347 const int aec_system_delay_ms =
peah20028c42016-03-04 11:50:54 -08001348 public_submodules_->echo_cancellation->GetSystemDelayInSamples() /
1349 samples_per_ms;
Michael Graczyk86c6d332015-07-23 11:41:39 -07001350 const int diff_aec_system_delay_ms =
peahdf3efa82015-11-28 12:35:15 -08001351 aec_system_delay_ms - capture_.last_aec_system_delay_ms;
Bjorn Volcker1ca324f2015-06-29 14:57:29 +02001352 if (diff_aec_system_delay_ms > kMinDiffDelayMs &&
peahdf3efa82015-11-28 12:35:15 -08001353 capture_.last_aec_system_delay_ms != 0) {
asaperssona2c58e22016-03-07 01:52:59 -08001354 RTC_HISTOGRAM_COUNTS("WebRTC.Audio.AecSystemDelayJump",
1355 diff_aec_system_delay_ms, kMinDiffDelayMs, 1000,
1356 100);
peahdf3efa82015-11-28 12:35:15 -08001357 if (capture_.aec_system_delay_jumps == -1) {
1358 capture_.aec_system_delay_jumps = 0; // Activate counter if needed.
Bjorn Volcker4e7aa432015-07-07 11:50:05 +02001359 }
peahdf3efa82015-11-28 12:35:15 -08001360 capture_.aec_system_delay_jumps++;
Bjorn Volcker1ca324f2015-06-29 14:57:29 +02001361 }
peahdf3efa82015-11-28 12:35:15 -08001362 capture_.last_aec_system_delay_ms = aec_system_delay_ms;
Bjorn Volcker1ca324f2015-06-29 14:57:29 +02001363 }
1364}
1365
Bjorn Volcker4e7aa432015-07-07 11:50:05 +02001366void AudioProcessingImpl::UpdateHistogramsOnCallEnd() {
peahdf3efa82015-11-28 12:35:15 -08001367 // Run in a single-threaded manner.
1368 rtc::CritScope cs_render(&crit_render_);
1369 rtc::CritScope cs_capture(&crit_capture_);
1370
1371 if (capture_.stream_delay_jumps > -1) {
asaperssona2c58e22016-03-07 01:52:59 -08001372 RTC_HISTOGRAM_ENUMERATION(
Bjorn Volcker4e7aa432015-07-07 11:50:05 +02001373 "WebRTC.Audio.NumOfPlatformReportedStreamDelayJumps",
peahdf3efa82015-11-28 12:35:15 -08001374 capture_.stream_delay_jumps, 51);
Bjorn Volcker4e7aa432015-07-07 11:50:05 +02001375 }
peahdf3efa82015-11-28 12:35:15 -08001376 capture_.stream_delay_jumps = -1;
1377 capture_.last_stream_delay_ms = 0;
Bjorn Volcker4e7aa432015-07-07 11:50:05 +02001378
peahdf3efa82015-11-28 12:35:15 -08001379 if (capture_.aec_system_delay_jumps > -1) {
asaperssona2c58e22016-03-07 01:52:59 -08001380 RTC_HISTOGRAM_ENUMERATION("WebRTC.Audio.NumOfAecSystemDelayJumps",
1381 capture_.aec_system_delay_jumps, 51);
Bjorn Volcker4e7aa432015-07-07 11:50:05 +02001382 }
peahdf3efa82015-11-28 12:35:15 -08001383 capture_.aec_system_delay_jumps = -1;
1384 capture_.last_aec_system_delay_ms = 0;
Bjorn Volcker4e7aa432015-07-07 11:50:05 +02001385}
1386
andrew@webrtc.org7bf26462011-12-03 00:03:31 +00001387#ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP
peahdf3efa82015-11-28 12:35:15 -08001388int AudioProcessingImpl::WriteMessageToDebugFile(
1389 FileWrapper* debug_file,
ivocd66b44d2016-01-15 03:06:36 -08001390 int64_t* filesize_limit_bytes,
peahdf3efa82015-11-28 12:35:15 -08001391 rtc::CriticalSection* crit_debug,
1392 ApmDebugDumpThreadState* debug_state) {
1393 int32_t size = debug_state->event_msg->ByteSize();
ajm@google.com808e0e02011-08-03 21:08:51 +00001394 if (size <= 0) {
1395 return kUnspecifiedError;
1396 }
andrew@webrtc.org621df672013-10-22 10:27:23 +00001397#if defined(WEBRTC_ARCH_BIG_ENDIAN)
Michael Graczyk86c6d332015-07-23 11:41:39 -07001398// TODO(ajm): Use little-endian "on the wire". For the moment, we can be
1399// pretty safe in assuming little-endian.
ajm@google.com808e0e02011-08-03 21:08:51 +00001400#endif
1401
peahdf3efa82015-11-28 12:35:15 -08001402 if (!debug_state->event_msg->SerializeToString(&debug_state->event_str)) {
ajm@google.com808e0e02011-08-03 21:08:51 +00001403 return kUnspecifiedError;
1404 }
1405
peahdf3efa82015-11-28 12:35:15 -08001406 {
1407 // Ensure atomic writes of the message.
ivocd66b44d2016-01-15 03:06:36 -08001408 rtc::CritScope cs_debug(crit_debug);
1409
tommia6219cc2016-06-15 10:30:14 -07001410 RTC_DCHECK(debug_file->is_open());
ivocd66b44d2016-01-15 03:06:36 -08001411 // Update the byte counter.
1412 if (*filesize_limit_bytes >= 0) {
1413 *filesize_limit_bytes -=
1414 (sizeof(int32_t) + debug_state->event_str.length());
1415 if (*filesize_limit_bytes < 0) {
1416 // Not enough bytes are left to write this message, so stop logging.
1417 debug_file->CloseFile();
1418 return kNoError;
1419 }
1420 }
peahdf3efa82015-11-28 12:35:15 -08001421 // Write message preceded by its size.
1422 if (!debug_file->Write(&size, sizeof(int32_t))) {
1423 return kFileError;
1424 }
1425 if (!debug_file->Write(debug_state->event_str.data(),
1426 debug_state->event_str.length())) {
1427 return kFileError;
1428 }
ajm@google.com808e0e02011-08-03 21:08:51 +00001429 }
1430
peahdf3efa82015-11-28 12:35:15 -08001431 debug_state->event_msg->Clear();
ajm@google.com808e0e02011-08-03 21:08:51 +00001432
andrew@webrtc.org17e40642014-03-04 20:58:13 +00001433 return kNoError;
ajm@google.com808e0e02011-08-03 21:08:51 +00001434}
1435
1436int AudioProcessingImpl::WriteInitMessage() {
peahdf3efa82015-11-28 12:35:15 -08001437 debug_dump_.capture.event_msg->set_type(audioproc::Event::INIT);
1438 audioproc::Init* msg = debug_dump_.capture.event_msg->mutable_init();
1439 msg->set_sample_rate(formats_.api_format.input_stream().sample_rate_hz());
ajm@google.com808e0e02011-08-03 21:08:51 +00001440
Peter Kasting69558702016-01-12 16:26:35 -08001441 msg->set_num_input_channels(static_cast<google::protobuf::int32>(
1442 formats_.api_format.input_stream().num_channels()));
1443 msg->set_num_output_channels(static_cast<google::protobuf::int32>(
1444 formats_.api_format.output_stream().num_channels()));
1445 msg->set_num_reverse_channels(static_cast<google::protobuf::int32>(
1446 formats_.api_format.reverse_input_stream().num_channels()));
peahdf3efa82015-11-28 12:35:15 -08001447 msg->set_reverse_sample_rate(
1448 formats_.api_format.reverse_input_stream().sample_rate_hz());
1449 msg->set_output_sample_rate(
1450 formats_.api_format.output_stream().sample_rate_hz());
peahc7bdf8a2016-04-11 07:05:53 -07001451 msg->set_reverse_output_sample_rate(
1452 formats_.api_format.reverse_output_stream().sample_rate_hz());
1453 msg->set_num_reverse_output_channels(
1454 formats_.api_format.reverse_output_stream().num_channels());
peahdf3efa82015-11-28 12:35:15 -08001455
1456 RETURN_ON_ERR(WriteMessageToDebugFile(debug_dump_.debug_file.get(),
ivocd66b44d2016-01-15 03:06:36 -08001457 &debug_dump_.num_bytes_left_for_log_,
peahdf3efa82015-11-28 12:35:15 -08001458 &crit_debug_, &debug_dump_.capture));
Minyue13b96ba2015-10-03 00:39:14 +02001459 return kNoError;
1460}
1461
1462int AudioProcessingImpl::WriteConfigMessage(bool forced) {
1463 audioproc::Config config;
1464
peahdf3efa82015-11-28 12:35:15 -08001465 config.set_aec_enabled(public_submodules_->echo_cancellation->is_enabled());
Minyue13b96ba2015-10-03 00:39:14 +02001466 config.set_aec_delay_agnostic_enabled(
peahdf3efa82015-11-28 12:35:15 -08001467 public_submodules_->echo_cancellation->is_delay_agnostic_enabled());
Minyue13b96ba2015-10-03 00:39:14 +02001468 config.set_aec_drift_compensation_enabled(
peahdf3efa82015-11-28 12:35:15 -08001469 public_submodules_->echo_cancellation->is_drift_compensation_enabled());
Minyue13b96ba2015-10-03 00:39:14 +02001470 config.set_aec_extended_filter_enabled(
peahdf3efa82015-11-28 12:35:15 -08001471 public_submodules_->echo_cancellation->is_extended_filter_enabled());
1472 config.set_aec_suppression_level(static_cast<int>(
1473 public_submodules_->echo_cancellation->suppression_level()));
Minyue13b96ba2015-10-03 00:39:14 +02001474
peahdf3efa82015-11-28 12:35:15 -08001475 config.set_aecm_enabled(
1476 public_submodules_->echo_control_mobile->is_enabled());
Minyue13b96ba2015-10-03 00:39:14 +02001477 config.set_aecm_comfort_noise_enabled(
peahdf3efa82015-11-28 12:35:15 -08001478 public_submodules_->echo_control_mobile->is_comfort_noise_enabled());
1479 config.set_aecm_routing_mode(static_cast<int>(
1480 public_submodules_->echo_control_mobile->routing_mode()));
Minyue13b96ba2015-10-03 00:39:14 +02001481
peahdf3efa82015-11-28 12:35:15 -08001482 config.set_agc_enabled(public_submodules_->gain_control->is_enabled());
1483 config.set_agc_mode(
1484 static_cast<int>(public_submodules_->gain_control->mode()));
1485 config.set_agc_limiter_enabled(
1486 public_submodules_->gain_control->is_limiter_enabled());
peahbe615622016-02-13 16:40:47 -08001487 config.set_noise_robust_agc_enabled(constants_.use_experimental_agc);
Minyue13b96ba2015-10-03 00:39:14 +02001488
peahdf3efa82015-11-28 12:35:15 -08001489 config.set_hpf_enabled(public_submodules_->high_pass_filter->is_enabled());
Minyue13b96ba2015-10-03 00:39:14 +02001490
peahdf3efa82015-11-28 12:35:15 -08001491 config.set_ns_enabled(public_submodules_->noise_suppression->is_enabled());
1492 config.set_ns_level(
1493 static_cast<int>(public_submodules_->noise_suppression->level()));
Minyue13b96ba2015-10-03 00:39:14 +02001494
peahdf3efa82015-11-28 12:35:15 -08001495 config.set_transient_suppression_enabled(
1496 capture_.transient_suppressor_enabled);
Alejandro Luebsc9b0c262016-05-16 15:32:38 -07001497 config.set_intelligibility_enhancer_enabled(
1498 capture_nonlocked_.intelligibility_enabled);
Minyue13b96ba2015-10-03 00:39:14 +02001499
peah7789fe72016-04-15 01:19:44 -07001500 std::string experiments_description =
1501 public_submodules_->echo_cancellation->GetExperimentsDescription();
1502 // TODO(peah): Add semicolon-separated concatenations of experiment
1503 // descriptions for other submodules.
peahca4cac72016-06-29 15:26:12 -07001504 if (capture_nonlocked_.level_controller_enabled) {
1505 experiments_description += "LevelController;";
1506 }
peah7789fe72016-04-15 01:19:44 -07001507 config.set_experiments_description(experiments_description);
1508
Minyue13b96ba2015-10-03 00:39:14 +02001509 std::string serialized_config = config.SerializeAsString();
peahdf3efa82015-11-28 12:35:15 -08001510 if (!forced &&
1511 debug_dump_.capture.last_serialized_config == serialized_config) {
Minyue13b96ba2015-10-03 00:39:14 +02001512 return kNoError;
ajm@google.com808e0e02011-08-03 21:08:51 +00001513 }
1514
peahdf3efa82015-11-28 12:35:15 -08001515 debug_dump_.capture.last_serialized_config = serialized_config;
Minyue13b96ba2015-10-03 00:39:14 +02001516
peahdf3efa82015-11-28 12:35:15 -08001517 debug_dump_.capture.event_msg->set_type(audioproc::Event::CONFIG);
1518 debug_dump_.capture.event_msg->mutable_config()->CopyFrom(config);
Minyue13b96ba2015-10-03 00:39:14 +02001519
peahdf3efa82015-11-28 12:35:15 -08001520 RETURN_ON_ERR(WriteMessageToDebugFile(debug_dump_.debug_file.get(),
ivocd66b44d2016-01-15 03:06:36 -08001521 &debug_dump_.num_bytes_left_for_log_,
peahdf3efa82015-11-28 12:35:15 -08001522 &crit_debug_, &debug_dump_.capture));
ajm@google.com808e0e02011-08-03 21:08:51 +00001523 return kNoError;
1524}
andrew@webrtc.org7bf26462011-12-03 00:03:31 +00001525#endif // WEBRTC_AUDIOPROC_DEBUG_DUMP
andrew@webrtc.orgddbb8a22014-04-22 21:00:04 +00001526
niklase@google.com470e71d2011-07-07 08:21:25 +00001527} // namespace webrtc