blob: 0e375c9b9bc95e529e7255fe7a504f2ad6f2fbec [file] [log] [blame]
niklase@google.com470e71d2011-07-07 08:21:25 +00001/*
andrew@webrtc.org40654032012-01-30 20:51:15 +00002 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
niklase@google.com470e71d2011-07-07 08:21:25 +00003 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020011#include "modules/audio_processing/audio_processing_impl.h"
niklase@google.com470e71d2011-07-07 08:21:25 +000012
Michael Graczyk86c6d332015-07-23 11:41:39 -070013#include <algorithm>
Yves Gerey988cc082018-10-23 12:03:01 +020014#include <cstdint>
Mirko Bonadei317a1f02019-09-17 17:06:18 +020015#include <memory>
alessiob3ec96df2017-05-22 06:57:06 -070016#include <string>
Yves Gerey988cc082018-10-23 12:03:01 +020017#include <type_traits>
18#include <utility>
niklase@google.com470e71d2011-07-07 08:21:25 +000019
Yves Gerey988cc082018-10-23 12:03:01 +020020#include "absl/types/optional.h"
21#include "api/array_view.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020022#include "common_audio/audio_converter.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020023#include "common_audio/include/audio_util.h"
Alex Loikob5c9a792018-04-16 16:31:22 +020024#include "modules/audio_processing/agc2/gain_applier.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020025#include "modules/audio_processing/audio_buffer.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020026#include "modules/audio_processing/common.h"
Yves Gerey988cc082018-10-23 12:03:01 +020027#include "modules/audio_processing/include/audio_frame_view.h"
Per Åhgren13735822018-02-12 21:42:56 +010028#include "modules/audio_processing/logging/apm_data_dumper.h"
Steve Anton10542f22019-01-11 09:11:00 -080029#include "rtc_base/atomic_ops.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020030#include "rtc_base/checks.h"
Steve Anton10542f22019-01-11 09:11:00 -080031#include "rtc_base/constructor_magic.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020032#include "rtc_base/logging.h"
Steve Anton10542f22019-01-11 09:11:00 -080033#include "rtc_base/ref_counted_object.h"
34#include "rtc_base/time_utils.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020035#include "rtc_base/trace_event.h"
Sam Zackrissonfeee1e42019-09-20 07:50:35 +020036#include "system_wrappers/include/field_trial.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020037#include "system_wrappers/include/metrics.h"
andrew@webrtc.org7bf26462011-12-03 00:03:31 +000038
Michael Graczyk86c6d332015-07-23 11:41:39 -070039#define RETURN_ON_ERR(expr) \
40 do { \
41 int err = (expr); \
42 if (err != kNoError) { \
43 return err; \
44 } \
andrew@webrtc.org60730cf2014-01-07 17:45:09 +000045 } while (0)
46
niklase@google.com470e71d2011-07-07 08:21:25 +000047namespace webrtc {
aluebsdf6416a2016-03-16 18:26:35 -070048
kwibergd59d3bb2016-09-13 07:49:33 -070049constexpr int AudioProcessing::kNativeSampleRatesHz[];
Alex Loiko73ec0192018-05-15 10:52:28 +020050constexpr int kRuntimeSettingQueueSize = 100;
aluebsdf6416a2016-03-16 18:26:35 -070051
Michael Graczyk86c6d332015-07-23 11:41:39 -070052namespace {
53
54static bool LayoutHasKeyboard(AudioProcessing::ChannelLayout layout) {
55 switch (layout) {
56 case AudioProcessing::kMono:
57 case AudioProcessing::kStereo:
58 return false;
59 case AudioProcessing::kMonoAndKeyboard:
60 case AudioProcessing::kStereoAndKeyboard:
61 return true;
62 }
63
kwiberg9e2be5f2016-09-14 05:23:22 -070064 RTC_NOTREACHED();
Michael Graczyk86c6d332015-07-23 11:41:39 -070065 return false;
66}
aluebsdf6416a2016-03-16 18:26:35 -070067
peah2ace3f92016-09-10 04:42:27 -070068bool SampleRateSupportsMultiBand(int sample_rate_hz) {
aluebsdf6416a2016-03-16 18:26:35 -070069 return sample_rate_hz == AudioProcessing::kSampleRate32kHz ||
70 sample_rate_hz == AudioProcessing::kSampleRate48kHz;
71}
72
Per Åhgren0cbb58e2019-10-29 22:59:44 +010073// Checks whether the legacy ns functionality should be enforced.
74bool DetectLegacyNsEnforcement() {
75 return field_trial::IsEnabled("WebRTC-NewNoiseSuppressionKillSwitch");
76}
77
Per Åhgrenc8626b62019-08-23 15:49:51 +020078// Identify the native processing rate that best handles a sample rate.
Per Åhgrenfcbe4072019-09-15 00:27:58 +020079int SuitableProcessRate(int minimum_rate,
80 int max_splitting_rate,
81 bool band_splitting_required) {
Per Åhgrenc8626b62019-08-23 15:49:51 +020082 const int uppermost_native_rate =
Per Åhgrenfcbe4072019-09-15 00:27:58 +020083 band_splitting_required ? max_splitting_rate : 48000;
Per Åhgrenc8626b62019-08-23 15:49:51 +020084 for (auto rate : {16000, 32000, 48000}) {
peah2ace3f92016-09-10 04:42:27 -070085 if (rate >= uppermost_native_rate) {
86 return uppermost_native_rate;
87 }
88 if (rate >= minimum_rate) {
aluebsdf6416a2016-03-16 18:26:35 -070089 return rate;
90 }
91 }
peah2ace3f92016-09-10 04:42:27 -070092 RTC_NOTREACHED();
93 return uppermost_native_rate;
aluebsdf6416a2016-03-16 18:26:35 -070094}
95
Sam Zackrisson23513132019-01-11 15:10:32 +010096NoiseSuppression::Level NsConfigLevelToInterfaceLevel(
97 AudioProcessing::Config::NoiseSuppression::Level level) {
98 using NsConfig = AudioProcessing::Config::NoiseSuppression;
99 switch (level) {
100 case NsConfig::kLow:
saza0bad15f2019-10-16 11:46:11 +0200101 return NoiseSuppression::Level::kLow;
Sam Zackrisson23513132019-01-11 15:10:32 +0100102 case NsConfig::kModerate:
saza0bad15f2019-10-16 11:46:11 +0200103 return NoiseSuppression::Level::kModerate;
Sam Zackrisson23513132019-01-11 15:10:32 +0100104 case NsConfig::kHigh:
saza0bad15f2019-10-16 11:46:11 +0200105 return NoiseSuppression::Level::kHigh;
Sam Zackrisson23513132019-01-11 15:10:32 +0100106 case NsConfig::kVeryHigh:
saza0bad15f2019-10-16 11:46:11 +0200107 return NoiseSuppression::Level::kVeryHigh;
Sam Zackrisson23513132019-01-11 15:10:32 +0100108 default:
109 RTC_NOTREACHED();
110 }
111}
112
Sam Zackrissonf0d1c032019-03-27 13:28:08 +0100113GainControl::Mode Agc1ConfigModeToInterfaceMode(
114 AudioProcessing::Config::GainController1::Mode mode) {
115 using Agc1Config = AudioProcessing::Config::GainController1;
116 switch (mode) {
117 case Agc1Config::kAdaptiveAnalog:
118 return GainControl::kAdaptiveAnalog;
119 case Agc1Config::kAdaptiveDigital:
120 return GainControl::kAdaptiveDigital;
121 case Agc1Config::kFixedDigital:
122 return GainControl::kFixedDigital;
123 }
124}
125
peah9e6a2902017-05-15 07:19:21 -0700126// Maximum lengths that frame of samples being passed from the render side to
127// the capture side can have (does not apply to AEC3).
128static const size_t kMaxAllowedValuesOfSamplesPerBand = 160;
129static const size_t kMaxAllowedValuesOfSamplesPerFrame = 480;
130
peah764e3642016-10-22 05:04:30 -0700131// Maximum number of frames to buffer in the render queue.
132// TODO(peah): Decrease this once we properly handle hugely unbalanced
133// reverse and forward call numbers.
134static const size_t kMaxNumFramesToBuffer = 100;
Michael Graczyk86c6d332015-07-23 11:41:39 -0700135} // namespace
andrew@webrtc.org60730cf2014-01-07 17:45:09 +0000136
137// Throughout webrtc, it's assumed that success is represented by zero.
kwiberg@webrtc.org2ebfac52015-01-14 10:51:54 +0000138static_assert(AudioProcessing::kNoError == 0, "kNoError must be zero");
andrew@webrtc.org60730cf2014-01-07 17:45:09 +0000139
saza1d600522019-10-18 13:29:43 +0200140AudioProcessingImpl::SubmoduleStates::SubmoduleStates(
Alex Loiko5825aa62017-12-18 16:02:40 +0100141 bool capture_post_processor_enabled,
Valeriia Nemychnikovaf06eb572018-08-29 10:37:09 +0200142 bool render_pre_processor_enabled,
143 bool capture_analyzer_enabled)
Alex Loiko5825aa62017-12-18 16:02:40 +0100144 : capture_post_processor_enabled_(capture_post_processor_enabled),
Valeriia Nemychnikovaf06eb572018-08-29 10:37:09 +0200145 render_pre_processor_enabled_(render_pre_processor_enabled),
146 capture_analyzer_enabled_(capture_analyzer_enabled) {}
peah2ace3f92016-09-10 04:42:27 -0700147
saza1d600522019-10-18 13:29:43 +0200148bool AudioProcessingImpl::SubmoduleStates::Update(
Sam Zackrissoncb1b5562018-09-28 14:15:09 +0200149 bool high_pass_filter_enabled,
peah2ace3f92016-09-10 04:42:27 -0700150 bool echo_canceller_enabled,
151 bool mobile_echo_controller_enabled,
ivoc9f4a4a02016-10-28 05:39:16 -0700152 bool residual_echo_detector_enabled,
peah2ace3f92016-09-10 04:42:27 -0700153 bool noise_suppressor_enabled,
peah2ace3f92016-09-10 04:42:27 -0700154 bool adaptive_gain_controller_enabled,
alessiob3ec96df2017-05-22 06:57:06 -0700155 bool gain_controller2_enabled,
Alex Loikob5c9a792018-04-16 16:31:22 +0200156 bool pre_amplifier_enabled,
Gustaf Ullbergce045ac2017-10-16 13:49:04 +0200157 bool echo_controller_enabled,
Sam Zackrisson0824c6f2019-10-07 14:03:56 +0200158 bool voice_detector_enabled,
peah2ace3f92016-09-10 04:42:27 -0700159 bool transient_suppressor_enabled) {
160 bool changed = false;
Sam Zackrissoncb1b5562018-09-28 14:15:09 +0200161 changed |= (high_pass_filter_enabled != high_pass_filter_enabled_);
peah2ace3f92016-09-10 04:42:27 -0700162 changed |= (echo_canceller_enabled != echo_canceller_enabled_);
163 changed |=
164 (mobile_echo_controller_enabled != mobile_echo_controller_enabled_);
ivoc9f4a4a02016-10-28 05:39:16 -0700165 changed |=
166 (residual_echo_detector_enabled != residual_echo_detector_enabled_);
peah2ace3f92016-09-10 04:42:27 -0700167 changed |= (noise_suppressor_enabled != noise_suppressor_enabled_);
168 changed |=
peah2ace3f92016-09-10 04:42:27 -0700169 (adaptive_gain_controller_enabled != adaptive_gain_controller_enabled_);
Per Åhgren6ee75fd2019-04-26 11:33:37 +0200170 changed |= (gain_controller2_enabled != gain_controller2_enabled_);
Alex Loikob5c9a792018-04-16 16:31:22 +0200171 changed |= (pre_amplifier_enabled_ != pre_amplifier_enabled);
Gustaf Ullbergce045ac2017-10-16 13:49:04 +0200172 changed |= (echo_controller_enabled != echo_controller_enabled_);
Sam Zackrisson0824c6f2019-10-07 14:03:56 +0200173 changed |= (voice_detector_enabled != voice_detector_enabled_);
peah2ace3f92016-09-10 04:42:27 -0700174 changed |= (transient_suppressor_enabled != transient_suppressor_enabled_);
175 if (changed) {
Sam Zackrissoncb1b5562018-09-28 14:15:09 +0200176 high_pass_filter_enabled_ = high_pass_filter_enabled;
peah2ace3f92016-09-10 04:42:27 -0700177 echo_canceller_enabled_ = echo_canceller_enabled;
178 mobile_echo_controller_enabled_ = mobile_echo_controller_enabled;
ivoc9f4a4a02016-10-28 05:39:16 -0700179 residual_echo_detector_enabled_ = residual_echo_detector_enabled;
peah2ace3f92016-09-10 04:42:27 -0700180 noise_suppressor_enabled_ = noise_suppressor_enabled;
peah2ace3f92016-09-10 04:42:27 -0700181 adaptive_gain_controller_enabled_ = adaptive_gain_controller_enabled;
alessiob3ec96df2017-05-22 06:57:06 -0700182 gain_controller2_enabled_ = gain_controller2_enabled;
Alex Loikob5c9a792018-04-16 16:31:22 +0200183 pre_amplifier_enabled_ = pre_amplifier_enabled;
Gustaf Ullbergce045ac2017-10-16 13:49:04 +0200184 echo_controller_enabled_ = echo_controller_enabled;
Sam Zackrisson0824c6f2019-10-07 14:03:56 +0200185 voice_detector_enabled_ = voice_detector_enabled;
peah2ace3f92016-09-10 04:42:27 -0700186 transient_suppressor_enabled_ = transient_suppressor_enabled;
187 }
188
189 changed |= first_update_;
190 first_update_ = false;
191 return changed;
192}
193
saza1d600522019-10-18 13:29:43 +0200194bool AudioProcessingImpl::SubmoduleStates::CaptureMultiBandSubModulesActive()
peah2ace3f92016-09-10 04:42:27 -0700195 const {
Gustaf Ullberg8675eee2019-10-09 13:34:36 +0200196 return CaptureMultiBandProcessingPresent() || voice_detector_enabled_;
peah2ace3f92016-09-10 04:42:27 -0700197}
198
saza1d600522019-10-18 13:29:43 +0200199bool AudioProcessingImpl::SubmoduleStates::CaptureMultiBandProcessingPresent()
200 const {
Gustaf Ullberg8675eee2019-10-09 13:34:36 +0200201 // If echo controller is present, assume it performs active processing.
202 return CaptureMultiBandProcessingActive(/*ec_processing_active=*/true);
203}
204
saza1d600522019-10-18 13:29:43 +0200205bool AudioProcessingImpl::SubmoduleStates::CaptureMultiBandProcessingActive(
Gustaf Ullberg8675eee2019-10-09 13:34:36 +0200206 bool ec_processing_active) const {
Sam Zackrissoncb1b5562018-09-28 14:15:09 +0200207 return high_pass_filter_enabled_ || echo_canceller_enabled_ ||
peah2ace3f92016-09-10 04:42:27 -0700208 mobile_echo_controller_enabled_ || noise_suppressor_enabled_ ||
Gustaf Ullberg8675eee2019-10-09 13:34:36 +0200209 adaptive_gain_controller_enabled_ ||
210 (echo_controller_enabled_ && ec_processing_active);
peah2ace3f92016-09-10 04:42:27 -0700211}
212
saza1d600522019-10-18 13:29:43 +0200213bool AudioProcessingImpl::SubmoduleStates::CaptureFullBandProcessingActive()
peah23ac8b42017-05-23 05:33:56 -0700214 const {
Alex Loikob5c9a792018-04-16 16:31:22 +0200215 return gain_controller2_enabled_ || capture_post_processor_enabled_ ||
216 pre_amplifier_enabled_;
peah23ac8b42017-05-23 05:33:56 -0700217}
218
saza1d600522019-10-18 13:29:43 +0200219bool AudioProcessingImpl::SubmoduleStates::CaptureAnalyzerActive() const {
Valeriia Nemychnikovaf06eb572018-08-29 10:37:09 +0200220 return capture_analyzer_enabled_;
221}
222
saza1d600522019-10-18 13:29:43 +0200223bool AudioProcessingImpl::SubmoduleStates::RenderMultiBandSubModulesActive()
peah2ace3f92016-09-10 04:42:27 -0700224 const {
225 return RenderMultiBandProcessingActive() || echo_canceller_enabled_ ||
ivoc20270be2016-11-15 05:24:35 -0800226 mobile_echo_controller_enabled_ || adaptive_gain_controller_enabled_ ||
Gustaf Ullbergce045ac2017-10-16 13:49:04 +0200227 echo_controller_enabled_;
peah2ace3f92016-09-10 04:42:27 -0700228}
229
saza1d600522019-10-18 13:29:43 +0200230bool AudioProcessingImpl::SubmoduleStates::RenderFullBandProcessingActive()
Alex Loiko5825aa62017-12-18 16:02:40 +0100231 const {
232 return render_pre_processor_enabled_;
233}
234
saza1d600522019-10-18 13:29:43 +0200235bool AudioProcessingImpl::SubmoduleStates::RenderMultiBandProcessingActive()
peah2ace3f92016-09-10 04:42:27 -0700236 const {
peah2ace3f92016-09-10 04:42:27 -0700237 return false;
peah2ace3f92016-09-10 04:42:27 -0700238}
239
saza1d600522019-10-18 13:29:43 +0200240bool AudioProcessingImpl::SubmoduleStates::HighPassFilteringRequired() const {
Sam Zackrissoncb1b5562018-09-28 14:15:09 +0200241 return high_pass_filter_enabled_ || echo_canceller_enabled_ ||
242 mobile_echo_controller_enabled_ || noise_suppressor_enabled_;
243}
244
Ivo Creusen5ec7e122017-12-22 11:35:59 +0100245AudioProcessingBuilder::AudioProcessingBuilder() = default;
246AudioProcessingBuilder::~AudioProcessingBuilder() = default;
247
248AudioProcessingBuilder& AudioProcessingBuilder::SetCapturePostProcessing(
249 std::unique_ptr<CustomProcessing> capture_post_processing) {
250 capture_post_processing_ = std::move(capture_post_processing);
251 return *this;
252}
253
254AudioProcessingBuilder& AudioProcessingBuilder::SetRenderPreProcessing(
255 std::unique_ptr<CustomProcessing> render_pre_processing) {
256 render_pre_processing_ = std::move(render_pre_processing);
257 return *this;
258}
259
Valeriia Nemychnikovaf06eb572018-08-29 10:37:09 +0200260AudioProcessingBuilder& AudioProcessingBuilder::SetCaptureAnalyzer(
261 std::unique_ptr<CustomAudioAnalyzer> capture_analyzer) {
262 capture_analyzer_ = std::move(capture_analyzer);
263 return *this;
264}
265
Ivo Creusen5ec7e122017-12-22 11:35:59 +0100266AudioProcessingBuilder& AudioProcessingBuilder::SetEchoControlFactory(
267 std::unique_ptr<EchoControlFactory> echo_control_factory) {
268 echo_control_factory_ = std::move(echo_control_factory);
269 return *this;
270}
271
Ivo Creusen09fa4b02018-01-11 16:08:54 +0100272AudioProcessingBuilder& AudioProcessingBuilder::SetEchoDetector(
Ivo Creusend1f970d2018-06-14 11:02:03 +0200273 rtc::scoped_refptr<EchoDetector> echo_detector) {
Ivo Creusen09fa4b02018-01-11 16:08:54 +0100274 echo_detector_ = std::move(echo_detector);
275 return *this;
276}
277
Ivo Creusen5ec7e122017-12-22 11:35:59 +0100278AudioProcessing* AudioProcessingBuilder::Create() {
279 webrtc::Config config;
280 return Create(config);
281}
282
283AudioProcessing* AudioProcessingBuilder::Create(const webrtc::Config& config) {
Ivo Creusen09fa4b02018-01-11 16:08:54 +0100284 AudioProcessingImpl* apm = new rtc::RefCountedObject<AudioProcessingImpl>(
285 config, std::move(capture_post_processing_),
286 std::move(render_pre_processing_), std::move(echo_control_factory_),
Valeriia Nemychnikovaf06eb572018-08-29 10:37:09 +0200287 std::move(echo_detector_), std::move(capture_analyzer_));
Ivo Creusen09fa4b02018-01-11 16:08:54 +0100288 if (apm->Initialize() != AudioProcessing::kNoError) {
289 delete apm;
290 apm = nullptr;
291 }
292 return apm;
Ivo Creusen5ec7e122017-12-22 11:35:59 +0100293}
294
peah88ac8532016-09-12 16:47:25 -0700295AudioProcessingImpl::AudioProcessingImpl(const webrtc::Config& config)
Sam Zackrissonfeee1e42019-09-20 07:50:35 +0200296 : AudioProcessingImpl(config,
297 /*capture_post_processor=*/nullptr,
298 /*render_pre_processor=*/nullptr,
299 /*echo_control_factory=*/nullptr,
300 /*echo_detector=*/nullptr,
301 /*capture_analyzer=*/nullptr) {}
aluebs@webrtc.orgd82f55d2015-01-15 18:07:21 +0000302
Per Åhgren13735822018-02-12 21:42:56 +0100303int AudioProcessingImpl::instance_count_ = 0;
304
Sam Zackrisson0beac582017-09-25 12:04:02 +0200305AudioProcessingImpl::AudioProcessingImpl(
306 const webrtc::Config& config,
Alex Loiko5825aa62017-12-18 16:02:40 +0100307 std::unique_ptr<CustomProcessing> capture_post_processor,
308 std::unique_ptr<CustomProcessing> render_pre_processor,
Gustaf Ullberg002ef282017-10-12 15:13:17 +0200309 std::unique_ptr<EchoControlFactory> echo_control_factory,
Valeriia Nemychnikovaf06eb572018-08-29 10:37:09 +0200310 rtc::scoped_refptr<EchoDetector> echo_detector,
311 std::unique_ptr<CustomAudioAnalyzer> capture_analyzer)
Per Åhgren13735822018-02-12 21:42:56 +0100312 : data_dumper_(
313 new ApmDataDumper(rtc::AtomicOps::Increment(&instance_count_))),
Per Åhgren0cbb58e2019-10-29 22:59:44 +0100314 enforced_usage_of_legacy_ns_(DetectLegacyNsEnforcement()),
Alex Loiko73ec0192018-05-15 10:52:28 +0200315 capture_runtime_settings_(kRuntimeSettingQueueSize),
316 render_runtime_settings_(kRuntimeSettingQueueSize),
317 capture_runtime_settings_enqueuer_(&capture_runtime_settings_),
318 render_runtime_settings_enqueuer_(&render_runtime_settings_),
Gustaf Ullberg002ef282017-10-12 15:13:17 +0200319 echo_control_factory_(std::move(echo_control_factory)),
Valeriia Nemychnikovaf06eb572018-08-29 10:37:09 +0200320 submodule_states_(!!capture_post_processor,
321 !!render_pre_processor,
322 !!capture_analyzer),
saza1d600522019-10-18 13:29:43 +0200323 submodules_(std::move(capture_post_processor),
324 std::move(render_pre_processor),
325 std::move(echo_detector),
Per Åhgren3daedb62019-11-22 12:11:40 +0100326 std::move(capture_analyzer)),
327 constants_(config.Get<ExperimentalAgc>().startup_min_volume,
328 config.Get<ExperimentalAgc>().clipped_level_min,
pbos@webrtc.org788acd12014-12-15 09:41:24 +0000329#if defined(WEBRTC_ANDROID) || defined(WEBRTC_IOS)
Per Åhgren3daedb62019-11-22 12:11:40 +0100330 /* enabled= */ false,
331 /* enabled_agc2_level_estimator= */ false,
332 /* digital_adaptive_disabled= */ false,
pbos@webrtc.org788acd12014-12-15 09:41:24 +0000333#else
Per Åhgren3daedb62019-11-22 12:11:40 +0100334 config.Get<ExperimentalAgc>().enabled,
335 config.Get<ExperimentalAgc>().enabled_agc2_level_estimator,
336 config.Get<ExperimentalAgc>().digital_adaptive_disabled,
pbos@webrtc.org788acd12014-12-15 09:41:24 +0000337#endif
Sam Zackrissonfeee1e42019-09-20 07:50:35 +0200338 !field_trial::IsEnabled(
339 "WebRTC-ApmExperimentalMultiChannelRenderKillSwitch"),
340 !field_trial::IsEnabled(
341 "WebRTC-ApmExperimentalMultiChannelCaptureKillSwitch")),
andrew1c7075f2015-06-24 18:14:14 -0700342#if defined(WEBRTC_ANDROID) || defined(WEBRTC_IOS)
Sam Zackrisson9394f6f2018-06-14 10:11:35 +0200343 capture_(false),
andrew1c7075f2015-06-24 18:14:14 -0700344#else
Sam Zackrisson9394f6f2018-06-14 10:11:35 +0200345 capture_(config.Get<ExperimentalNs>().enabled),
andrew1c7075f2015-06-24 18:14:14 -0700346#endif
Alessio Bazzicacc22f512018-08-30 13:01:34 +0200347 capture_nonlocked_() {
Sam Zackrisson72cc71c2019-10-21 12:54:02 +0200348 RTC_LOG(LS_INFO) << "Injected APM submodules:"
349 << "\nEcho control factory: " << !!echo_control_factory_
350 << "\nEcho detector: " << !!submodules_.echo_detector
351 << "\nCapture analyzer: " << !!submodules_.capture_analyzer
352 << "\nCapture post processor: "
353 << !!submodules_.capture_post_processor
354 << "\nRender pre processor: "
355 << !!submodules_.render_pre_processor;
356
Sam Zackrisson421c8592019-02-11 13:39:46 +0100357 // Mark Echo Controller enabled if a factory is injected.
358 capture_nonlocked_.echo_controller_enabled =
359 static_cast<bool>(echo_control_factory_);
niklase@google.com470e71d2011-07-07 08:21:25 +0000360
saza1d600522019-10-18 13:29:43 +0200361 submodules_.gain_control.reset(new GainControlImpl());
Gustaf Ullbergce045ac2017-10-16 13:49:04 +0200362
Sam Zackrisson421c8592019-02-11 13:39:46 +0100363 // If no echo detector is injected, use the ResidualEchoDetector.
saza1d600522019-10-18 13:29:43 +0200364 if (!submodules_.echo_detector) {
365 submodules_.echo_detector =
Sam Zackrisson421c8592019-02-11 13:39:46 +0100366 new rtc::RefCountedObject<ResidualEchoDetector>();
peahdf3efa82015-11-28 12:35:15 -0800367 }
pbos@webrtc.org788acd12014-12-15 09:41:24 +0000368
Sam Zackrisson421c8592019-02-11 13:39:46 +0100369 // TODO(alessiob): Move the injected gain controller once injection is
370 // implemented.
saza1d600522019-10-18 13:29:43 +0200371 submodules_.gain_controller2.reset(new GainController2());
Sam Zackrisson421c8592019-02-11 13:39:46 +0100372
andrew@webrtc.orge84978f2014-01-25 02:09:06 +0000373 SetExtraOptions(config);
niklase@google.com470e71d2011-07-07 08:21:25 +0000374}
375
Per Åhgren0e3198e2019-11-18 08:52:22 +0100376AudioProcessingImpl::~AudioProcessingImpl() = default;
niklase@google.com470e71d2011-07-07 08:21:25 +0000377
niklase@google.com470e71d2011-07-07 08:21:25 +0000378int AudioProcessingImpl::Initialize() {
peahdf3efa82015-11-28 12:35:15 -0800379 // Run in a single-threaded manner during initialization.
380 rtc::CritScope cs_render(&crit_render_);
381 rtc::CritScope cs_capture(&crit_capture_);
niklase@google.com470e71d2011-07-07 08:21:25 +0000382 return InitializeLocked();
383}
384
peahde65ddc2016-09-16 15:02:15 -0700385int AudioProcessingImpl::Initialize(int capture_input_sample_rate_hz,
386 int capture_output_sample_rate_hz,
387 int render_input_sample_rate_hz,
388 ChannelLayout capture_input_layout,
389 ChannelLayout capture_output_layout,
390 ChannelLayout render_input_layout) {
Michael Graczyk86c6d332015-07-23 11:41:39 -0700391 const ProcessingConfig processing_config = {
peahde65ddc2016-09-16 15:02:15 -0700392 {{capture_input_sample_rate_hz, ChannelsFromLayout(capture_input_layout),
393 LayoutHasKeyboard(capture_input_layout)},
394 {capture_output_sample_rate_hz,
395 ChannelsFromLayout(capture_output_layout),
396 LayoutHasKeyboard(capture_output_layout)},
397 {render_input_sample_rate_hz, ChannelsFromLayout(render_input_layout),
398 LayoutHasKeyboard(render_input_layout)},
399 {render_input_sample_rate_hz, ChannelsFromLayout(render_input_layout),
400 LayoutHasKeyboard(render_input_layout)}}};
Michael Graczyk86c6d332015-07-23 11:41:39 -0700401
402 return Initialize(processing_config);
403}
404
405int AudioProcessingImpl::Initialize(const ProcessingConfig& processing_config) {
peahdf3efa82015-11-28 12:35:15 -0800406 // Run in a single-threaded manner during initialization.
407 rtc::CritScope cs_render(&crit_render_);
408 rtc::CritScope cs_capture(&crit_capture_);
Michael Graczyk86c6d332015-07-23 11:41:39 -0700409 return InitializeLocked(processing_config);
andrew@webrtc.orga8b97372014-03-10 22:26:12 +0000410}
411
peahdf3efa82015-11-28 12:35:15 -0800412int AudioProcessingImpl::MaybeInitializeRender(
peah81b9bfe2015-11-27 02:47:28 -0800413 const ProcessingConfig& processing_config) {
peahdf3efa82015-11-28 12:35:15 -0800414 // Called from both threads. Thread check is therefore not possible.
Oskar Sundbom4b276482019-05-23 14:28:00 +0200415 if (processing_config == formats_.api_format) {
peah192164e2015-11-17 02:16:45 -0800416 return kNoError;
417 }
peahdf3efa82015-11-28 12:35:15 -0800418
419 rtc::CritScope cs_capture(&crit_capture_);
peah192164e2015-11-17 02:16:45 -0800420 return InitializeLocked(processing_config);
421}
422
niklase@google.com470e71d2011-07-07 08:21:25 +0000423int AudioProcessingImpl::InitializeLocked() {
Per Åhgren4bdced52017-06-27 16:00:38 +0200424 UpdateActiveSubmoduleStates();
425
Per Åhgrend47941e2019-08-22 11:51:13 +0200426 const int render_audiobuffer_sample_rate_hz =
peahdf3efa82015-11-28 12:35:15 -0800427 formats_.api_format.reverse_output_stream().num_frames() == 0
Per Åhgrend47941e2019-08-22 11:51:13 +0200428 ? formats_.render_processing_format.sample_rate_hz()
429 : formats_.api_format.reverse_output_stream().sample_rate_hz();
peahdf3efa82015-11-28 12:35:15 -0800430 if (formats_.api_format.reverse_input_stream().num_channels() > 0) {
431 render_.render_audio.reset(new AudioBuffer(
Per Åhgrend47941e2019-08-22 11:51:13 +0200432 formats_.api_format.reverse_input_stream().sample_rate_hz(),
peahdf3efa82015-11-28 12:35:15 -0800433 formats_.api_format.reverse_input_stream().num_channels(),
Per Åhgrend47941e2019-08-22 11:51:13 +0200434 formats_.render_processing_format.sample_rate_hz(),
peahde65ddc2016-09-16 15:02:15 -0700435 formats_.render_processing_format.num_channels(),
Per Åhgrend47941e2019-08-22 11:51:13 +0200436 render_audiobuffer_sample_rate_hz,
437 formats_.render_processing_format.num_channels()));
peah2ace3f92016-09-10 04:42:27 -0700438 if (formats_.api_format.reverse_input_stream() !=
439 formats_.api_format.reverse_output_stream()) {
kwibergc2b785d2016-02-24 05:22:32 -0800440 render_.render_converter = AudioConverter::Create(
peahdf3efa82015-11-28 12:35:15 -0800441 formats_.api_format.reverse_input_stream().num_channels(),
442 formats_.api_format.reverse_input_stream().num_frames(),
443 formats_.api_format.reverse_output_stream().num_channels(),
kwibergc2b785d2016-02-24 05:22:32 -0800444 formats_.api_format.reverse_output_stream().num_frames());
ekmeyerson60d9b332015-08-14 10:35:55 -0700445 } else {
peahdf3efa82015-11-28 12:35:15 -0800446 render_.render_converter.reset(nullptr);
ekmeyerson60d9b332015-08-14 10:35:55 -0700447 }
Michael Graczyk86c6d332015-07-23 11:41:39 -0700448 } else {
peahdf3efa82015-11-28 12:35:15 -0800449 render_.render_audio.reset(nullptr);
450 render_.render_converter.reset(nullptr);
Michael Graczyk86c6d332015-07-23 11:41:39 -0700451 }
peahce4d9152017-05-19 01:28:05 -0700452
Per Åhgrend47941e2019-08-22 11:51:13 +0200453 capture_.capture_audio.reset(new AudioBuffer(
454 formats_.api_format.input_stream().sample_rate_hz(),
455 formats_.api_format.input_stream().num_channels(),
456 capture_nonlocked_.capture_processing_format.sample_rate_hz(),
457 formats_.api_format.output_stream().num_channels(),
458 formats_.api_format.output_stream().sample_rate_hz(),
459 formats_.api_format.output_stream().num_channels()));
niklase@google.com470e71d2011-07-07 08:21:25 +0000460
Gustaf Ullberg422b9e02019-10-09 13:02:14 +0200461 if (capture_nonlocked_.capture_processing_format.sample_rate_hz() <
462 formats_.api_format.output_stream().sample_rate_hz() &&
463 formats_.api_format.output_stream().sample_rate_hz() == 48000) {
464 capture_.capture_fullband_audio.reset(
465 new AudioBuffer(formats_.api_format.input_stream().sample_rate_hz(),
466 formats_.api_format.input_stream().num_channels(),
467 formats_.api_format.output_stream().sample_rate_hz(),
468 formats_.api_format.output_stream().num_channels(),
469 formats_.api_format.output_stream().sample_rate_hz(),
470 formats_.api_format.output_stream().num_channels()));
471 } else {
472 capture_.capture_fullband_audio.reset();
473 }
474
peah764e3642016-10-22 05:04:30 -0700475 AllocateRenderQueue();
476
saza1d600522019-10-18 13:29:43 +0200477 submodules_.gain_control->Initialize(num_proc_channels(),
478 proc_sample_rate_hz());
Per Åhgren3daedb62019-11-22 12:11:40 +0100479 if (constants_.use_experimental_agc) {
480 if (!submodules_.agc_manager.get() ||
481 submodules_.agc_manager->num_channels() !=
482 static_cast<int>(num_proc_channels()) ||
483 submodules_.agc_manager->sample_rate_hz() !=
484 capture_nonlocked_.split_rate) {
Per Åhgren2a6b3b12019-11-26 19:34:26 +0100485 int stream_analog_level = -1;
486 const bool re_creation = !!submodules_.agc_manager;
487 if (re_creation) {
488 stream_analog_level = submodules_.agc_manager->stream_analog_level();
489 }
Per Åhgren3daedb62019-11-22 12:11:40 +0100490 submodules_.agc_manager.reset(new AgcManagerDirect(
491 num_proc_channels(), constants_.agc_startup_min_volume,
492 constants_.agc_clipped_level_min,
493 constants_.use_experimental_agc_agc2_level_estimation,
494 constants_.use_experimental_agc_agc2_digital_adaptive,
495 capture_nonlocked_.split_rate));
Per Åhgren2a6b3b12019-11-26 19:34:26 +0100496 if (re_creation) {
497 submodules_.agc_manager->set_stream_analog_level(stream_analog_level);
498 }
Per Åhgren3daedb62019-11-22 12:11:40 +0100499 }
saza1d600522019-10-18 13:29:43 +0200500 submodules_.agc_manager->Initialize();
Per Åhgren3daedb62019-11-22 12:11:40 +0100501 submodules_.agc_manager->SetupDigitalGainControl(
Per Åhgren0e3198e2019-11-18 08:52:22 +0100502 submodules_.gain_control.get());
saza1d600522019-10-18 13:29:43 +0200503 submodules_.agc_manager->SetCaptureMuted(capture_.output_will_be_muted);
peahde65ddc2016-09-16 15:02:15 -0700504 }
Bjorn Volckeradc46c42015-04-15 11:42:40 +0200505 InitializeTransient();
Per Åhgren0aefbf02019-08-23 21:29:17 +0200506 InitializeHighPassFilter();
Sam Zackrisson0824c6f2019-10-07 14:03:56 +0200507 InitializeVoiceDetector();
ivoc9f4a4a02016-10-28 05:39:16 -0700508 InitializeResidualEchoDetector();
Gustaf Ullberg8eb9c7d2017-10-14 08:28:46 +0200509 InitializeEchoController();
alessiob3ec96df2017-05-22 06:57:06 -0700510 InitializeGainController2();
saza0bad15f2019-10-16 11:46:11 +0200511 InitializeNoiseSuppressor();
Valeriia Nemychnikovaf06eb572018-08-29 10:37:09 +0200512 InitializeAnalyzer();
Sam Zackrisson0beac582017-09-25 12:04:02 +0200513 InitializePostProcessor();
Alex Loiko5825aa62017-12-18 16:02:40 +0100514 InitializePreProcessor();
solenberg70f99032015-12-08 11:07:32 -0800515
aleloi868f32f2017-05-23 07:20:05 -0700516 if (aec_dump_) {
Minyue Li656d6092018-08-10 15:38:52 +0200517 aec_dump_->WriteInitMessage(formats_.api_format, rtc::TimeUTCMillis());
aleloi868f32f2017-05-23 07:20:05 -0700518 }
niklase@google.com470e71d2011-07-07 08:21:25 +0000519 return kNoError;
520}
521
Michael Graczyk86c6d332015-07-23 11:41:39 -0700522int AudioProcessingImpl::InitializeLocked(const ProcessingConfig& config) {
Per Åhgren4bdced52017-06-27 16:00:38 +0200523 UpdateActiveSubmoduleStates();
524
Michael Graczyk86c6d332015-07-23 11:41:39 -0700525 for (const auto& stream : config.streams) {
Michael Graczyk86c6d332015-07-23 11:41:39 -0700526 if (stream.num_channels() > 0 && stream.sample_rate_hz() <= 0) {
527 return kBadSampleRateError;
528 }
andrew@webrtc.orga8b97372014-03-10 22:26:12 +0000529 }
Michael Graczyk86c6d332015-07-23 11:41:39 -0700530
Peter Kasting69558702016-01-12 16:26:35 -0800531 const size_t num_in_channels = config.input_stream().num_channels();
532 const size_t num_out_channels = config.output_stream().num_channels();
Michael Graczyk86c6d332015-07-23 11:41:39 -0700533
534 // Need at least one input channel.
535 // Need either one output channel or as many outputs as there are inputs.
536 if (num_in_channels == 0 ||
537 !(num_out_channels == 1 || num_out_channels == num_in_channels)) {
Michael Graczykc2047542015-07-22 21:06:11 -0700538 return kBadNumberChannelsError;
539 }
540
peahdf3efa82015-11-28 12:35:15 -0800541 formats_.api_format = config;
andrew@webrtc.orgddbb8a22014-04-22 21:00:04 +0000542
Per Åhgrenfcbe4072019-09-15 00:27:58 +0200543 // Choose maximum rate to use for the split filtering.
544 RTC_DCHECK(config_.pipeline.maximum_internal_processing_rate == 48000 ||
545 config_.pipeline.maximum_internal_processing_rate == 32000);
546 int max_splitting_rate = 48000;
547 if (config_.pipeline.maximum_internal_processing_rate == 32000) {
548 max_splitting_rate = config_.pipeline.maximum_internal_processing_rate;
549 }
550
Per Åhgrenc8626b62019-08-23 15:49:51 +0200551 int capture_processing_rate = SuitableProcessRate(
peah423d2362016-04-09 16:06:52 -0700552 std::min(formats_.api_format.input_stream().sample_rate_hz(),
peah2ace3f92016-09-10 04:42:27 -0700553 formats_.api_format.output_stream().sample_rate_hz()),
Per Åhgrenfcbe4072019-09-15 00:27:58 +0200554 max_splitting_rate,
peah2ace3f92016-09-10 04:42:27 -0700555 submodule_states_.CaptureMultiBandSubModulesActive() ||
556 submodule_states_.RenderMultiBandSubModulesActive());
Per Åhgrenc8626b62019-08-23 15:49:51 +0200557 RTC_DCHECK_NE(8000, capture_processing_rate);
andrew@webrtc.orga8b97372014-03-10 22:26:12 +0000558
peahde65ddc2016-09-16 15:02:15 -0700559 capture_nonlocked_.capture_processing_format =
560 StreamConfig(capture_processing_rate);
peah2ace3f92016-09-10 04:42:27 -0700561
peah2ce640f2017-04-07 03:57:48 -0700562 int render_processing_rate;
Gustaf Ullbergbd83b912017-10-18 12:32:42 +0200563 if (!capture_nonlocked_.echo_controller_enabled) {
Per Åhgrenc8626b62019-08-23 15:49:51 +0200564 render_processing_rate = SuitableProcessRate(
peah2ce640f2017-04-07 03:57:48 -0700565 std::min(formats_.api_format.reverse_input_stream().sample_rate_hz(),
566 formats_.api_format.reverse_output_stream().sample_rate_hz()),
Per Åhgrenfcbe4072019-09-15 00:27:58 +0200567 max_splitting_rate,
peah2ce640f2017-04-07 03:57:48 -0700568 submodule_states_.CaptureMultiBandSubModulesActive() ||
569 submodule_states_.RenderMultiBandSubModulesActive());
570 } else {
571 render_processing_rate = capture_processing_rate;
572 }
573
peahde65ddc2016-09-16 15:02:15 -0700574 // If the forward sample rate is 8 kHz, the render stream is also processed
aluebseb3603b2016-04-20 15:27:58 -0700575 // at this rate.
peahde65ddc2016-09-16 15:02:15 -0700576 if (capture_nonlocked_.capture_processing_format.sample_rate_hz() ==
577 kSampleRate8kHz) {
578 render_processing_rate = kSampleRate8kHz;
andrew@webrtc.orga8b97372014-03-10 22:26:12 +0000579 } else {
peahde65ddc2016-09-16 15:02:15 -0700580 render_processing_rate =
581 std::max(render_processing_rate, static_cast<int>(kSampleRate16kHz));
andrew@webrtc.orgddbb8a22014-04-22 21:00:04 +0000582 }
583
Per Åhgrenc8626b62019-08-23 15:49:51 +0200584 RTC_DCHECK_NE(8000, render_processing_rate);
585
peahce4d9152017-05-19 01:28:05 -0700586 if (submodule_states_.RenderMultiBandSubModulesActive()) {
Sam Zackrissonfeee1e42019-09-20 07:50:35 +0200587 // By default, downmix the render stream to mono for analysis. This has been
588 // demonstrated to work well for AEC in most practical scenarios.
Per Åhgrene14cb992019-11-27 09:34:22 +0100589 const bool multi_channel_render = config_.pipeline.multi_channel_render &&
590 constants_.multi_channel_render_support;
Sam Zackrissonfeee1e42019-09-20 07:50:35 +0200591 int render_processing_num_channels =
Per Åhgrene14cb992019-11-27 09:34:22 +0100592 multi_channel_render
Sam Zackrissonfeee1e42019-09-20 07:50:35 +0200593 ? formats_.api_format.reverse_input_stream().num_channels()
594 : 1;
595 formats_.render_processing_format =
596 StreamConfig(render_processing_rate, render_processing_num_channels);
peahce4d9152017-05-19 01:28:05 -0700597 } else {
598 formats_.render_processing_format = StreamConfig(
599 formats_.api_format.reverse_input_stream().sample_rate_hz(),
600 formats_.api_format.reverse_input_stream().num_channels());
601 }
andrew@webrtc.orgddbb8a22014-04-22 21:00:04 +0000602
peahde65ddc2016-09-16 15:02:15 -0700603 if (capture_nonlocked_.capture_processing_format.sample_rate_hz() ==
604 kSampleRate32kHz ||
605 capture_nonlocked_.capture_processing_format.sample_rate_hz() ==
606 kSampleRate48kHz) {
peahdf3efa82015-11-28 12:35:15 -0800607 capture_nonlocked_.split_rate = kSampleRate16kHz;
andrew@webrtc.orgddbb8a22014-04-22 21:00:04 +0000608 } else {
peahdf3efa82015-11-28 12:35:15 -0800609 capture_nonlocked_.split_rate =
peahde65ddc2016-09-16 15:02:15 -0700610 capture_nonlocked_.capture_processing_format.sample_rate_hz();
andrew@webrtc.orga8b97372014-03-10 22:26:12 +0000611 }
612
613 return InitializeLocked();
614}
615
peah88ac8532016-09-12 16:47:25 -0700616void AudioProcessingImpl::ApplyConfig(const AudioProcessing::Config& config) {
Sam Zackrisson72cc71c2019-10-21 12:54:02 +0200617 RTC_LOG(LS_INFO) << "AudioProcessing::ApplyConfig: " << config.ToString();
618
peah88ac8532016-09-12 16:47:25 -0700619 // Run in a single-threaded manner when applying the settings.
620 rtc::CritScope cs_render(&crit_render_);
621 rtc::CritScope cs_capture(&crit_capture_);
622
Sam Zackrissonfeee1e42019-09-20 07:50:35 +0200623 const bool pipeline_config_changed =
Per Åhgrene14cb992019-11-27 09:34:22 +0100624 config_.pipeline.multi_channel_render !=
625 config.pipeline.multi_channel_render ||
626 config_.pipeline.multi_channel_capture !=
627 config.pipeline.multi_channel_capture;
Sam Zackrissonfeee1e42019-09-20 07:50:35 +0200628
Per Åhgren200feba2019-03-06 04:16:46 +0100629 const bool aec_config_changed =
630 config_.echo_canceller.enabled != config.echo_canceller.enabled ||
631 config_.echo_canceller.use_legacy_aec !=
632 config.echo_canceller.use_legacy_aec ||
633 config_.echo_canceller.mobile_mode != config.echo_canceller.mobile_mode ||
634 (config_.echo_canceller.enabled && config.echo_canceller.use_legacy_aec &&
635 config_.echo_canceller.legacy_moderate_suppression_level !=
636 config.echo_canceller.legacy_moderate_suppression_level);
637
Sam Zackrissonf0d1c032019-03-27 13:28:08 +0100638 const bool agc1_config_changed =
639 config_.gain_controller1.enabled != config.gain_controller1.enabled ||
640 config_.gain_controller1.mode != config.gain_controller1.mode ||
641 config_.gain_controller1.target_level_dbfs !=
642 config.gain_controller1.target_level_dbfs ||
643 config_.gain_controller1.compression_gain_db !=
644 config.gain_controller1.compression_gain_db ||
645 config_.gain_controller1.enable_limiter !=
646 config.gain_controller1.enable_limiter ||
647 config_.gain_controller1.analog_level_minimum !=
648 config.gain_controller1.analog_level_minimum ||
649 config_.gain_controller1.analog_level_maximum !=
650 config.gain_controller1.analog_level_maximum;
651
Sam Zackrisson0824c6f2019-10-07 14:03:56 +0200652 const bool voice_detection_config_changed =
653 config_.voice_detection.enabled != config.voice_detection.enabled;
654
saza0bad15f2019-10-16 11:46:11 +0200655 const bool ns_config_changed =
656 config_.noise_suppression.enabled != config.noise_suppression.enabled ||
657 config_.noise_suppression.level != config.noise_suppression.level;
658
Yves Gerey499bc6c2018-10-10 18:29:07 +0200659 config_ = config;
660
Per Åhgren200feba2019-03-06 04:16:46 +0100661 if (aec_config_changed) {
662 InitializeEchoController();
663 }
Sam Zackrissoncdf0e6d2018-09-17 11:05:17 +0200664
saza0bad15f2019-10-16 11:46:11 +0200665 if (ns_config_changed) {
666 InitializeNoiseSuppressor();
667 }
Sam Zackrisson23513132019-01-11 15:10:32 +0100668
Per Åhgren0aefbf02019-08-23 21:29:17 +0200669 InitializeHighPassFilter();
peah8271d042016-11-22 07:24:52 -0800670
Sam Zackrissonf0d1c032019-03-27 13:28:08 +0100671 if (agc1_config_changed) {
672 ApplyAgc1Config(config_.gain_controller1);
673 }
674
Sam Zackrissonab1aee02018-03-05 15:59:06 +0100675 const bool config_ok = GainController2::Validate(config_.gain_controller2);
alessiob3ec96df2017-05-22 06:57:06 -0700676 if (!config_ok) {
Jonas Olsson645b0272018-02-15 15:16:27 +0100677 RTC_LOG(LS_ERROR) << "AudioProcessing module config error\n"
678 "Gain Controller 2: "
Mirko Bonadei675513b2017-11-09 11:09:25 +0100679 << GainController2::ToString(config_.gain_controller2)
Jonas Olsson645b0272018-02-15 15:16:27 +0100680 << "\nReverting to default parameter set";
alessiob3ec96df2017-05-22 06:57:06 -0700681 config_.gain_controller2 = AudioProcessing::Config::GainController2();
682 }
Alessio Bazzica270f7b52017-10-13 11:05:17 +0200683 InitializeGainController2();
Alex Loikob5c9a792018-04-16 16:31:22 +0200684 InitializePreAmplifier();
saza1d600522019-10-18 13:29:43 +0200685 submodules_.gain_controller2->ApplyConfig(config_.gain_controller2);
Sam Zackrissonb24c00f2018-11-26 16:18:25 +0100686
saza1d600522019-10-18 13:29:43 +0200687 if (config_.level_estimation.enabled && !submodules_.output_level_estimator) {
688 submodules_.output_level_estimator = std::make_unique<LevelEstimator>();
Sam Zackrissonb24c00f2018-11-26 16:18:25 +0100689 }
Sam Zackrisson4db667b2018-12-21 16:29:27 +0100690
Sam Zackrisson0824c6f2019-10-07 14:03:56 +0200691 if (voice_detection_config_changed) {
692 InitializeVoiceDetector();
Sam Zackrisson4db667b2018-12-21 16:29:27 +0100693 }
Sam Zackrissonfeee1e42019-09-20 07:50:35 +0200694
695 // Reinitialization must happen after all submodule configuration to avoid
696 // additional reinitializations on the next capture / render processing call.
697 if (pipeline_config_changed) {
698 InitializeLocked(formats_.api_format);
699 }
peah88ac8532016-09-12 16:47:25 -0700700}
701
Sam Zackrissonf0d1c032019-03-27 13:28:08 +0100702void AudioProcessingImpl::ApplyAgc1Config(
703 const Config::GainController1& config) {
Per Åhgren0e3198e2019-11-18 08:52:22 +0100704 int error = submodules_.gain_control->Enable(config.enabled);
Sam Zackrissonf0d1c032019-03-27 13:28:08 +0100705 RTC_DCHECK_EQ(kNoError, error);
Sam Zackrissonf0d1c032019-03-27 13:28:08 +0100706
Per Åhgren0e3198e2019-11-18 08:52:22 +0100707 if (!submodules_.agc_manager) {
708 error = submodules_.gain_control->set_mode(
709 Agc1ConfigModeToInterfaceMode(config.mode));
710 RTC_DCHECK_EQ(kNoError, error);
711 error = submodules_.gain_control->set_target_level_dbfs(
712 config.target_level_dbfs);
713 RTC_DCHECK_EQ(kNoError, error);
714 error = submodules_.gain_control->set_compression_gain_db(
715 config.compression_gain_db);
716 RTC_DCHECK_EQ(kNoError, error);
717 error = submodules_.gain_control->enable_limiter(config.enable_limiter);
718 RTC_DCHECK_EQ(kNoError, error);
719 error = submodules_.gain_control->set_analog_level_limits(
720 config.analog_level_minimum, config.analog_level_maximum);
721 RTC_DCHECK_EQ(kNoError, error);
Sam Zackrissonf0d1c032019-03-27 13:28:08 +0100722 }
Sam Zackrissonf0d1c032019-03-27 13:28:08 +0100723}
724
peah88ac8532016-09-12 16:47:25 -0700725void AudioProcessingImpl::SetExtraOptions(const webrtc::Config& config) {
peahdf3efa82015-11-28 12:35:15 -0800726 // Run in a single-threaded manner when setting the extra options.
727 rtc::CritScope cs_render(&crit_render_);
728 rtc::CritScope cs_capture(&crit_capture_);
pbos@webrtc.org788acd12014-12-15 09:41:24 +0000729
Per Åhgrenf204faf2019-04-25 15:18:06 +0200730 capture_nonlocked_.use_aec2_extended_filter =
731 config.Get<ExtendedFilter>().enabled;
732 capture_nonlocked_.use_aec2_delay_agnostic =
733 config.Get<DelayAgnostic>().enabled;
734 capture_nonlocked_.use_aec2_refined_adaptive_filter =
735 config.Get<RefinedAdaptiveFilter>().enabled;
peahb624d8c2016-03-05 03:01:14 -0800736
peahdf3efa82015-11-28 12:35:15 -0800737 if (capture_.transient_suppressor_enabled !=
738 config.Get<ExperimentalNs>().enabled) {
739 capture_.transient_suppressor_enabled =
740 config.Get<ExperimentalNs>().enabled;
pbos@webrtc.org788acd12014-12-15 09:41:24 +0000741 InitializeTransient();
742 }
andrew@webrtc.org61e596f2013-07-25 18:28:29 +0000743}
744
andrew@webrtc.orgddbb8a22014-04-22 21:00:04 +0000745int AudioProcessingImpl::proc_sample_rate_hz() const {
peahdf3efa82015-11-28 12:35:15 -0800746 // Used as callback from submodules, hence locking is not allowed.
peahde65ddc2016-09-16 15:02:15 -0700747 return capture_nonlocked_.capture_processing_format.sample_rate_hz();
niklase@google.com470e71d2011-07-07 08:21:25 +0000748}
749
Gustaf Ullberg422b9e02019-10-09 13:02:14 +0200750int AudioProcessingImpl::proc_fullband_sample_rate_hz() const {
751 return capture_.capture_fullband_audio
752 ? capture_.capture_fullband_audio->num_frames() * 100
753 : capture_nonlocked_.capture_processing_format.sample_rate_hz();
754}
755
andrew@webrtc.orgddbb8a22014-04-22 21:00:04 +0000756int AudioProcessingImpl::proc_split_sample_rate_hz() const {
peahdf3efa82015-11-28 12:35:15 -0800757 // Used as callback from submodules, hence locking is not allowed.
758 return capture_nonlocked_.split_rate;
niklase@google.com470e71d2011-07-07 08:21:25 +0000759}
760
Peter Kasting69558702016-01-12 16:26:35 -0800761size_t AudioProcessingImpl::num_reverse_channels() const {
peahdf3efa82015-11-28 12:35:15 -0800762 // Used as callback from submodules, hence locking is not allowed.
peahde65ddc2016-09-16 15:02:15 -0700763 return formats_.render_processing_format.num_channels();
niklase@google.com470e71d2011-07-07 08:21:25 +0000764}
765
Peter Kasting69558702016-01-12 16:26:35 -0800766size_t AudioProcessingImpl::num_input_channels() const {
peahdf3efa82015-11-28 12:35:15 -0800767 // Used as callback from submodules, hence locking is not allowed.
768 return formats_.api_format.input_stream().num_channels();
niklase@google.com470e71d2011-07-07 08:21:25 +0000769}
770
Peter Kasting69558702016-01-12 16:26:35 -0800771size_t AudioProcessingImpl::num_proc_channels() const {
aluebsb2328d12016-01-11 20:32:29 -0800772 // Used as callback from submodules, hence locking is not allowed.
Per Åhgrene14cb992019-11-27 09:34:22 +0100773 const bool multi_channel_capture = config_.pipeline.multi_channel_capture &&
774 constants_.multi_channel_capture_support;
775 if (capture_nonlocked_.echo_controller_enabled && !multi_channel_capture) {
Sam Zackrissonfeee1e42019-09-20 07:50:35 +0200776 return 1;
777 }
778 return num_output_channels();
aluebsb2328d12016-01-11 20:32:29 -0800779}
780
Peter Kasting69558702016-01-12 16:26:35 -0800781size_t AudioProcessingImpl::num_output_channels() const {
peahdf3efa82015-11-28 12:35:15 -0800782 // Used as callback from submodules, hence locking is not allowed.
783 return formats_.api_format.output_stream().num_channels();
niklase@google.com470e71d2011-07-07 08:21:25 +0000784}
785
andrew@webrtc.org17342e52014-02-12 22:28:31 +0000786void AudioProcessingImpl::set_output_will_be_muted(bool muted) {
peahdf3efa82015-11-28 12:35:15 -0800787 rtc::CritScope cs(&crit_capture_);
788 capture_.output_will_be_muted = muted;
saza1d600522019-10-18 13:29:43 +0200789 if (submodules_.agc_manager.get()) {
790 submodules_.agc_manager->SetCaptureMuted(capture_.output_will_be_muted);
pbos@webrtc.org788acd12014-12-15 09:41:24 +0000791 }
andrew@webrtc.org17342e52014-02-12 22:28:31 +0000792}
793
Alessio Bazzicac054e782018-04-16 12:10:09 +0200794void AudioProcessingImpl::SetRuntimeSetting(RuntimeSetting setting) {
Alex Loiko73ec0192018-05-15 10:52:28 +0200795 switch (setting.type()) {
796 case RuntimeSetting::Type::kCustomRenderProcessingRuntimeSetting:
Alessio Bazzica7c19a702019-11-07 13:22:00 +0100797 case RuntimeSetting::Type::kPlayoutAudioDeviceChange:
Alex Loiko73ec0192018-05-15 10:52:28 +0200798 render_runtime_settings_enqueuer_.Enqueue(setting);
799 return;
Alex Loiko73ec0192018-05-15 10:52:28 +0200800 case RuntimeSetting::Type::kCapturePreGain:
Sam Zackrissonf0d1c032019-03-27 13:28:08 +0100801 case RuntimeSetting::Type::kCaptureCompressionGain:
Per Åhgren6ee75fd2019-04-26 11:33:37 +0200802 case RuntimeSetting::Type::kCaptureFixedPostGain:
Alessio Bazzica7c19a702019-11-07 13:22:00 +0100803 capture_runtime_settings_enqueuer_.Enqueue(setting);
804 return;
Fredrik Hernqvistca362852019-05-10 15:50:02 +0200805 case RuntimeSetting::Type::kPlayoutVolumeChange:
Alex Loiko73ec0192018-05-15 10:52:28 +0200806 capture_runtime_settings_enqueuer_.Enqueue(setting);
Alessio Bazzica7c19a702019-11-07 13:22:00 +0100807 render_runtime_settings_enqueuer_.Enqueue(setting);
808 return;
809 case RuntimeSetting::Type::kNotSpecified:
810 RTC_NOTREACHED();
Alex Loiko73ec0192018-05-15 10:52:28 +0200811 return;
812 }
813 // The language allows the enum to have a non-enumerator
814 // value. Check that this doesn't happen.
815 RTC_NOTREACHED();
Alessio Bazzicac054e782018-04-16 12:10:09 +0200816}
817
818AudioProcessingImpl::RuntimeSettingEnqueuer::RuntimeSettingEnqueuer(
819 SwapQueue<RuntimeSetting>* runtime_settings)
Alessio Bazzica33444dc2018-04-20 13:16:55 +0200820 : runtime_settings_(*runtime_settings) {
821 RTC_DCHECK(runtime_settings);
Alessio Bazzicac054e782018-04-16 12:10:09 +0200822}
823
824AudioProcessingImpl::RuntimeSettingEnqueuer::~RuntimeSettingEnqueuer() =
825 default;
826
827void AudioProcessingImpl::RuntimeSettingEnqueuer::Enqueue(
828 RuntimeSetting setting) {
829 size_t remaining_attempts = 10;
Alessio Bazzica33444dc2018-04-20 13:16:55 +0200830 while (!runtime_settings_.Insert(&setting) && remaining_attempts-- > 0) {
Alessio Bazzicac054e782018-04-16 12:10:09 +0200831 RuntimeSetting setting_to_discard;
Alessio Bazzica33444dc2018-04-20 13:16:55 +0200832 if (runtime_settings_.Remove(&setting_to_discard))
Alessio Bazzicac054e782018-04-16 12:10:09 +0200833 RTC_LOG(LS_ERROR)
834 << "The runtime settings queue is full. Oldest setting discarded.";
835 }
836 if (remaining_attempts == 0)
837 RTC_LOG(LS_ERROR) << "Cannot enqueue a new runtime setting.";
838}
andrew@webrtc.org17342e52014-02-12 22:28:31 +0000839
andrew@webrtc.orgddbb8a22014-04-22 21:00:04 +0000840int AudioProcessingImpl::ProcessStream(const float* const* src,
Michael Graczyk86c6d332015-07-23 11:41:39 -0700841 const StreamConfig& input_config,
842 const StreamConfig& output_config,
843 float* const* dest) {
peah369f8282015-12-17 06:42:29 -0800844 TRACE_EVENT0("webrtc", "AudioProcessing::ProcessStream_StreamConfig");
peahdf3efa82015-11-28 12:35:15 -0800845 ProcessingConfig processing_config;
peah2ace3f92016-09-10 04:42:27 -0700846 bool reinitialization_required = false;
peahdf3efa82015-11-28 12:35:15 -0800847 {
848 // Acquire the capture lock in order to safely call the function
849 // that retrieves the render side data. This function accesses apm
850 // getters that need the capture lock held when being called.
851 rtc::CritScope cs_capture(&crit_capture_);
peah764e3642016-10-22 05:04:30 -0700852 EmptyQueuedRenderAudio();
peahdf3efa82015-11-28 12:35:15 -0800853
854 if (!src || !dest) {
855 return kNullPointerError;
856 }
857
858 processing_config = formats_.api_format;
peah2ace3f92016-09-10 04:42:27 -0700859 reinitialization_required = UpdateActiveSubmoduleStates();
niklase@google.com470e71d2011-07-07 08:21:25 +0000860 }
andrew@webrtc.org17e40642014-03-04 20:58:13 +0000861
Oskar Sundbom4b276482019-05-23 14:28:00 +0200862 if (processing_config.input_stream() != input_config) {
863 processing_config.input_stream() = input_config;
864 reinitialization_required = true;
peahdf3efa82015-11-28 12:35:15 -0800865 }
Oskar Sundbom4b276482019-05-23 14:28:00 +0200866
867 if (processing_config.output_stream() != output_config) {
868 processing_config.output_stream() = output_config;
869 reinitialization_required = true;
870 }
871
872 if (reinitialization_required) {
873 // Reinitialize.
874 rtc::CritScope cs_render(&crit_render_);
875 rtc::CritScope cs_capture(&crit_capture_);
876 RETURN_ON_ERR(InitializeLocked(processing_config));
877 }
878
peahdf3efa82015-11-28 12:35:15 -0800879 rtc::CritScope cs_capture(&crit_capture_);
kwiberg9e2be5f2016-09-14 05:23:22 -0700880 RTC_DCHECK_EQ(processing_config.input_stream().num_frames(),
881 formats_.api_format.input_stream().num_frames());
andrew@webrtc.org17e40642014-03-04 20:58:13 +0000882
aleloi868f32f2017-05-23 07:20:05 -0700883 if (aec_dump_) {
884 RecordUnprocessedCaptureStream(src);
885 }
886
Per Åhgrena1351272019-08-15 12:15:46 +0200887 capture_.keyboard_info.Extract(src, formats_.api_format.input_stream());
peahdf3efa82015-11-28 12:35:15 -0800888 capture_.capture_audio->CopyFrom(src, formats_.api_format.input_stream());
Gustaf Ullberg8675eee2019-10-09 13:34:36 +0200889 if (capture_.capture_fullband_audio) {
890 capture_.capture_fullband_audio->CopyFrom(
891 src, formats_.api_format.input_stream());
892 }
peahde65ddc2016-09-16 15:02:15 -0700893 RETURN_ON_ERR(ProcessCaptureStreamLocked());
Gustaf Ullberg422b9e02019-10-09 13:02:14 +0200894 if (capture_.capture_fullband_audio) {
895 capture_.capture_fullband_audio->CopyTo(formats_.api_format.output_stream(),
896 dest);
897 } else {
898 capture_.capture_audio->CopyTo(formats_.api_format.output_stream(), dest);
899 }
andrew@webrtc.org17e40642014-03-04 20:58:13 +0000900
aleloi868f32f2017-05-23 07:20:05 -0700901 if (aec_dump_) {
902 RecordProcessedCaptureStream(dest);
903 }
andrew@webrtc.org17e40642014-03-04 20:58:13 +0000904 return kNoError;
905}
906
Alex Loiko73ec0192018-05-15 10:52:28 +0200907void AudioProcessingImpl::HandleCaptureRuntimeSettings() {
Alessio Bazzicac054e782018-04-16 12:10:09 +0200908 RuntimeSetting setting;
Alex Loiko73ec0192018-05-15 10:52:28 +0200909 while (capture_runtime_settings_.Remove(&setting)) {
Alex Loiko62347222018-09-10 10:18:07 +0200910 if (aec_dump_) {
911 aec_dump_->WriteRuntimeSetting(setting);
912 }
Alessio Bazzicac054e782018-04-16 12:10:09 +0200913 switch (setting.type()) {
914 case RuntimeSetting::Type::kCapturePreGain:
Alex Loikob5c9a792018-04-16 16:31:22 +0200915 if (config_.pre_amplifier.enabled) {
916 float value;
917 setting.GetFloat(&value);
Sam Zackrisson21bfa402019-10-23 09:43:01 +0200918 config_.pre_amplifier.fixed_gain_factor = value;
saza1d600522019-10-18 13:29:43 +0200919 submodules_.pre_amplifier->SetGainFactor(value);
Alex Loikob5c9a792018-04-16 16:31:22 +0200920 }
921 // TODO(bugs.chromium.org/9138): Log setting handling by Aec Dump.
Alessio Bazzicac054e782018-04-16 12:10:09 +0200922 break;
Sam Zackrissonf0d1c032019-03-27 13:28:08 +0100923 case RuntimeSetting::Type::kCaptureCompressionGain: {
Per Åhgren0e3198e2019-11-18 08:52:22 +0100924 if (!submodules_.agc_manager) {
925 float value;
926 setting.GetFloat(&value);
927 int int_value = static_cast<int>(value + .5f);
928 config_.gain_controller1.compression_gain_db = int_value;
929 int error =
930 submodules_.gain_control->set_compression_gain_db(int_value);
931 RTC_DCHECK_EQ(kNoError, error);
932 }
Sam Zackrissonf0d1c032019-03-27 13:28:08 +0100933 break;
934 }
Per Åhgren6ee75fd2019-04-26 11:33:37 +0200935 case RuntimeSetting::Type::kCaptureFixedPostGain: {
936 if (config_.gain_controller2.enabled) {
937 float value;
938 setting.GetFloat(&value);
939 config_.gain_controller2.fixed_digital.gain_db = value;
saza1d600522019-10-18 13:29:43 +0200940 submodules_.gain_controller2->ApplyConfig(config_.gain_controller2);
Per Åhgren6ee75fd2019-04-26 11:33:37 +0200941 }
942 break;
943 }
Fredrik Hernqvistca362852019-05-10 15:50:02 +0200944 case RuntimeSetting::Type::kPlayoutVolumeChange: {
945 int value;
946 setting.GetInt(&value);
947 capture_.playout_volume = value;
948 break;
949 }
Alessio Bazzica7c19a702019-11-07 13:22:00 +0100950 case RuntimeSetting::Type::kPlayoutAudioDeviceChange:
951 RTC_NOTREACHED();
952 break;
Alex Loiko73ec0192018-05-15 10:52:28 +0200953 case RuntimeSetting::Type::kCustomRenderProcessingRuntimeSetting:
954 RTC_NOTREACHED();
955 break;
956 case RuntimeSetting::Type::kNotSpecified:
957 RTC_NOTREACHED();
958 break;
959 }
960 }
961}
962
963void AudioProcessingImpl::HandleRenderRuntimeSettings() {
964 RuntimeSetting setting;
965 while (render_runtime_settings_.Remove(&setting)) {
Alex Loiko62347222018-09-10 10:18:07 +0200966 if (aec_dump_) {
967 aec_dump_->WriteRuntimeSetting(setting);
968 }
Alex Loiko73ec0192018-05-15 10:52:28 +0200969 switch (setting.type()) {
Alessio Bazzica7c19a702019-11-07 13:22:00 +0100970 case RuntimeSetting::Type::kPlayoutAudioDeviceChange: // fall-through
Alessio Bazzica7587de42019-11-11 13:32:20 +0100971 case RuntimeSetting::Type::kPlayoutVolumeChange: // fall-through
Alex Loiko73ec0192018-05-15 10:52:28 +0200972 case RuntimeSetting::Type::kCustomRenderProcessingRuntimeSetting:
saza1d600522019-10-18 13:29:43 +0200973 if (submodules_.render_pre_processor) {
974 submodules_.render_pre_processor->SetRuntimeSetting(setting);
Alex Loiko73ec0192018-05-15 10:52:28 +0200975 }
976 break;
Sam Zackrissonf0d1c032019-03-27 13:28:08 +0100977 case RuntimeSetting::Type::kCapturePreGain: // fall-through
978 case RuntimeSetting::Type::kCaptureCompressionGain: // fall-through
Per Åhgren6ee75fd2019-04-26 11:33:37 +0200979 case RuntimeSetting::Type::kCaptureFixedPostGain: // fall-through
Alessio Bazzica33444dc2018-04-20 13:16:55 +0200980 case RuntimeSetting::Type::kNotSpecified:
Alessio Bazzicac054e782018-04-16 12:10:09 +0200981 RTC_NOTREACHED();
982 break;
983 }
984 }
985}
986
peah9e6a2902017-05-15 07:19:21 -0700987void AudioProcessingImpl::QueueBandedRenderAudio(AudioBuffer* audio) {
kwibergaf476c72016-11-28 15:21:39 -0800988 RTC_DCHECK_GE(160, audio->num_frames_per_band());
peah764e3642016-10-22 05:04:30 -0700989
990 // Insert the samples into the queue.
saza1d600522019-10-18 13:29:43 +0200991 if (submodules_.echo_cancellation) {
Per Åhgrenf204faf2019-04-25 15:18:06 +0200992 RTC_DCHECK(aec_render_signal_queue_);
Per Åhgrenb6e24d72019-04-29 12:14:50 +0200993 EchoCancellationImpl::PackRenderAudioBuffer(audio, num_output_channels(),
994 num_reverse_channels(),
995 &aec_render_queue_buffer_);
996
Per Åhgrenf204faf2019-04-25 15:18:06 +0200997 if (!aec_render_signal_queue_->Insert(&aec_render_queue_buffer_)) {
998 // The data queue is full and needs to be emptied.
999 EmptyQueuedRenderAudio();
peah764e3642016-10-22 05:04:30 -07001000
Per Åhgrenf204faf2019-04-25 15:18:06 +02001001 // Retry the insert (should always work).
1002 bool result = aec_render_signal_queue_->Insert(&aec_render_queue_buffer_);
1003 RTC_DCHECK(result);
1004 }
peaha0624602016-10-25 04:45:24 -07001005 }
1006
saza1d600522019-10-18 13:29:43 +02001007 if (submodules_.echo_control_mobile) {
Per Åhgrenb6e24d72019-04-29 12:14:50 +02001008 EchoControlMobileImpl::PackRenderAudioBuffer(audio, num_output_channels(),
1009 num_reverse_channels(),
1010 &aecm_render_queue_buffer_);
1011 RTC_DCHECK(aecm_render_signal_queue_);
1012 // Insert the samples into the queue.
1013 if (!aecm_render_signal_queue_->Insert(&aecm_render_queue_buffer_)) {
1014 // The data queue is full and needs to be emptied.
1015 EmptyQueuedRenderAudio();
peaha0624602016-10-25 04:45:24 -07001016
Per Åhgrenb6e24d72019-04-29 12:14:50 +02001017 // Retry the insert (should always work).
1018 bool result =
1019 aecm_render_signal_queue_->Insert(&aecm_render_queue_buffer_);
1020 RTC_DCHECK(result);
1021 }
peah764e3642016-10-22 05:04:30 -07001022 }
peah701d6282016-10-25 05:42:20 -07001023
Per Åhgren0e3198e2019-11-18 08:52:22 +01001024 if (!submodules_.agc_manager) {
Per Åhgrene35b32c2019-11-22 18:22:04 +01001025 GainControlImpl::PackRenderAudioBuffer(*audio, &agc_render_queue_buffer_);
peah701d6282016-10-25 05:42:20 -07001026 // Insert the samples into the queue.
1027 if (!agc_render_signal_queue_->Insert(&agc_render_queue_buffer_)) {
1028 // The data queue is full and needs to be emptied.
1029 EmptyQueuedRenderAudio();
1030
1031 // Retry the insert (should always work).
1032 bool result = agc_render_signal_queue_->Insert(&agc_render_queue_buffer_);
1033 RTC_DCHECK(result);
1034 }
1035 }
peah9e6a2902017-05-15 07:19:21 -07001036}
ivoc9f4a4a02016-10-28 05:39:16 -07001037
peah9e6a2902017-05-15 07:19:21 -07001038void AudioProcessingImpl::QueueNonbandedRenderAudio(AudioBuffer* audio) {
ivoc9f4a4a02016-10-28 05:39:16 -07001039 ResidualEchoDetector::PackRenderAudioBuffer(audio, &red_render_queue_buffer_);
1040
1041 // Insert the samples into the queue.
1042 if (!red_render_signal_queue_->Insert(&red_render_queue_buffer_)) {
1043 // The data queue is full and needs to be emptied.
1044 EmptyQueuedRenderAudio();
1045
1046 // Retry the insert (should always work).
1047 bool result = red_render_signal_queue_->Insert(&red_render_queue_buffer_);
1048 RTC_DCHECK(result);
1049 }
peah764e3642016-10-22 05:04:30 -07001050}
1051
1052void AudioProcessingImpl::AllocateRenderQueue() {
peah701d6282016-10-25 05:42:20 -07001053 const size_t new_agc_render_queue_element_max_size =
peah9e6a2902017-05-15 07:19:21 -07001054 std::max(static_cast<size_t>(1), kMaxAllowedValuesOfSamplesPerBand);
peah701d6282016-10-25 05:42:20 -07001055
ivoc9f4a4a02016-10-28 05:39:16 -07001056 const size_t new_red_render_queue_element_max_size =
1057 std::max(static_cast<size_t>(1), kMaxAllowedValuesOfSamplesPerFrame);
1058
peaha0624602016-10-25 04:45:24 -07001059 // Reallocate the queues if the queue item sizes are too small to fit the
1060 // data to put in the queues.
peah701d6282016-10-25 05:42:20 -07001061
1062 if (agc_render_queue_element_max_size_ <
1063 new_agc_render_queue_element_max_size) {
1064 agc_render_queue_element_max_size_ = new_agc_render_queue_element_max_size;
1065
1066 std::vector<int16_t> template_queue_element(
1067 agc_render_queue_element_max_size_);
1068
1069 agc_render_signal_queue_.reset(
1070 new SwapQueue<std::vector<int16_t>, RenderQueueItemVerifier<int16_t>>(
1071 kMaxNumFramesToBuffer, template_queue_element,
1072 RenderQueueItemVerifier<int16_t>(
1073 agc_render_queue_element_max_size_)));
1074
1075 agc_render_queue_buffer_.resize(agc_render_queue_element_max_size_);
1076 agc_capture_queue_buffer_.resize(agc_render_queue_element_max_size_);
1077 } else {
1078 agc_render_signal_queue_->Clear();
peah764e3642016-10-22 05:04:30 -07001079 }
ivoc9f4a4a02016-10-28 05:39:16 -07001080
1081 if (red_render_queue_element_max_size_ <
1082 new_red_render_queue_element_max_size) {
1083 red_render_queue_element_max_size_ = new_red_render_queue_element_max_size;
1084
1085 std::vector<float> template_queue_element(
1086 red_render_queue_element_max_size_);
1087
1088 red_render_signal_queue_.reset(
1089 new SwapQueue<std::vector<float>, RenderQueueItemVerifier<float>>(
1090 kMaxNumFramesToBuffer, template_queue_element,
1091 RenderQueueItemVerifier<float>(
1092 red_render_queue_element_max_size_)));
1093
1094 red_render_queue_buffer_.resize(red_render_queue_element_max_size_);
1095 red_capture_queue_buffer_.resize(red_render_queue_element_max_size_);
1096 } else {
1097 red_render_signal_queue_->Clear();
1098 }
peah764e3642016-10-22 05:04:30 -07001099}
1100
1101void AudioProcessingImpl::EmptyQueuedRenderAudio() {
1102 rtc::CritScope cs_capture(&crit_capture_);
saza1d600522019-10-18 13:29:43 +02001103 if (submodules_.echo_cancellation) {
Per Åhgrenf204faf2019-04-25 15:18:06 +02001104 RTC_DCHECK(aec_render_signal_queue_);
1105 while (aec_render_signal_queue_->Remove(&aec_capture_queue_buffer_)) {
saza1d600522019-10-18 13:29:43 +02001106 submodules_.echo_cancellation->ProcessRenderAudio(
Per Åhgrenf204faf2019-04-25 15:18:06 +02001107 aec_capture_queue_buffer_);
1108 }
peaha0624602016-10-25 04:45:24 -07001109 }
1110
saza1d600522019-10-18 13:29:43 +02001111 if (submodules_.echo_control_mobile) {
Per Åhgrenb6e24d72019-04-29 12:14:50 +02001112 RTC_DCHECK(aecm_render_signal_queue_);
1113 while (aecm_render_signal_queue_->Remove(&aecm_capture_queue_buffer_)) {
saza1d600522019-10-18 13:29:43 +02001114 submodules_.echo_control_mobile->ProcessRenderAudio(
Per Åhgrenb6e24d72019-04-29 12:14:50 +02001115 aecm_capture_queue_buffer_);
1116 }
peah701d6282016-10-25 05:42:20 -07001117 }
1118
1119 while (agc_render_signal_queue_->Remove(&agc_capture_queue_buffer_)) {
saza1d600522019-10-18 13:29:43 +02001120 submodules_.gain_control->ProcessRenderAudio(agc_capture_queue_buffer_);
peah764e3642016-10-22 05:04:30 -07001121 }
ivoc9f4a4a02016-10-28 05:39:16 -07001122
1123 while (red_render_signal_queue_->Remove(&red_capture_queue_buffer_)) {
saza1d600522019-10-18 13:29:43 +02001124 RTC_DCHECK(submodules_.echo_detector);
1125 submodules_.echo_detector->AnalyzeRenderAudio(red_capture_queue_buffer_);
ivoc9f4a4a02016-10-28 05:39:16 -07001126 }
peah764e3642016-10-22 05:04:30 -07001127}
1128
andrew@webrtc.org17e40642014-03-04 20:58:13 +00001129int AudioProcessingImpl::ProcessStream(AudioFrame* frame) {
peah369f8282015-12-17 06:42:29 -08001130 TRACE_EVENT0("webrtc", "AudioProcessing::ProcessStream_AudioFrame");
peahdf3efa82015-11-28 12:35:15 -08001131 {
1132 // Acquire the capture lock in order to safely call the function
Alessio Bazzicad2b97402018-08-09 14:23:11 +02001133 // that retrieves the render side data. This function accesses APM
peahdf3efa82015-11-28 12:35:15 -08001134 // getters that need the capture lock held when being called.
peahdf3efa82015-11-28 12:35:15 -08001135 rtc::CritScope cs_capture(&crit_capture_);
peah764e3642016-10-22 05:04:30 -07001136 EmptyQueuedRenderAudio();
peahdf3efa82015-11-28 12:35:15 -08001137 }
peahfa6228e2015-11-16 16:27:42 -08001138
andrew@webrtc.org17e40642014-03-04 20:58:13 +00001139 if (!frame) {
1140 return kNullPointerError;
1141 }
andrew@webrtc.orgddbb8a22014-04-22 21:00:04 +00001142 // Must be a native rate.
1143 if (frame->sample_rate_hz_ != kSampleRate8kHz &&
1144 frame->sample_rate_hz_ != kSampleRate16kHz &&
aluebs@webrtc.org087da132014-11-17 23:01:23 +00001145 frame->sample_rate_hz_ != kSampleRate32kHz &&
1146 frame->sample_rate_hz_ != kSampleRate48kHz) {
andrew@webrtc.orgddbb8a22014-04-22 21:00:04 +00001147 return kBadSampleRateError;
1148 }
peah192164e2015-11-17 02:16:45 -08001149
peahdf3efa82015-11-28 12:35:15 -08001150 ProcessingConfig processing_config;
peah2ace3f92016-09-10 04:42:27 -07001151 bool reinitialization_required = false;
peahdf3efa82015-11-28 12:35:15 -08001152 {
1153 // Aquire lock for the access of api_format.
1154 // The lock is released immediately due to the conditional
1155 // reinitialization.
1156 rtc::CritScope cs_capture(&crit_capture_);
1157 // TODO(ajm): The input and output rates and channels are currently
1158 // constrained to be identical in the int16 interface.
1159 processing_config = formats_.api_format;
peah2ace3f92016-09-10 04:42:27 -07001160
1161 reinitialization_required = UpdateActiveSubmoduleStates();
peahdf3efa82015-11-28 12:35:15 -08001162 }
Michael Graczyk86c6d332015-07-23 11:41:39 -07001163
Oskar Sundbom4b276482019-05-23 14:28:00 +02001164 reinitialization_required =
1165 reinitialization_required ||
1166 processing_config.input_stream().sample_rate_hz() !=
1167 frame->sample_rate_hz_ ||
1168 processing_config.input_stream().num_channels() != frame->num_channels_ ||
1169 processing_config.output_stream().sample_rate_hz() !=
1170 frame->sample_rate_hz_ ||
1171 processing_config.output_stream().num_channels() != frame->num_channels_;
1172
1173 if (reinitialization_required) {
1174 processing_config.input_stream().set_sample_rate_hz(frame->sample_rate_hz_);
1175 processing_config.input_stream().set_num_channels(frame->num_channels_);
1176 processing_config.output_stream().set_sample_rate_hz(
1177 frame->sample_rate_hz_);
1178 processing_config.output_stream().set_num_channels(frame->num_channels_);
1179
1180 // Reinitialize.
peahdf3efa82015-11-28 12:35:15 -08001181 rtc::CritScope cs_render(&crit_render_);
Oskar Sundbom4b276482019-05-23 14:28:00 +02001182 rtc::CritScope cs_capture(&crit_capture_);
1183 RETURN_ON_ERR(InitializeLocked(processing_config));
peahdf3efa82015-11-28 12:35:15 -08001184 }
Oskar Sundbom4b276482019-05-23 14:28:00 +02001185
peahdf3efa82015-11-28 12:35:15 -08001186 rtc::CritScope cs_capture(&crit_capture_);
peah192164e2015-11-17 02:16:45 -08001187 if (frame->samples_per_channel_ !=
peahdf3efa82015-11-28 12:35:15 -08001188 formats_.api_format.input_stream().num_frames()) {
niklase@google.com470e71d2011-07-07 08:21:25 +00001189 return kBadDataLengthError;
1190 }
1191
aleloi868f32f2017-05-23 07:20:05 -07001192 if (aec_dump_) {
1193 RecordUnprocessedCaptureStream(*frame);
1194 }
1195
Per Åhgrend47941e2019-08-22 11:51:13 +02001196 capture_.capture_audio->CopyFrom(frame);
Gustaf Ullberg3c918b12019-10-11 13:14:44 +02001197 if (capture_.capture_fullband_audio) {
1198 capture_.capture_fullband_audio->CopyFrom(frame);
1199 }
peahde65ddc2016-09-16 15:02:15 -07001200 RETURN_ON_ERR(ProcessCaptureStreamLocked());
Gustaf Ullberg8675eee2019-10-09 13:34:36 +02001201 if (submodule_states_.CaptureMultiBandProcessingPresent() ||
Per Åhgrena1351272019-08-15 12:15:46 +02001202 submodule_states_.CaptureFullBandProcessingActive()) {
Gustaf Ullberg422b9e02019-10-09 13:02:14 +02001203 if (capture_.capture_fullband_audio) {
1204 capture_.capture_fullband_audio->CopyTo(frame);
1205 } else {
1206 capture_.capture_audio->CopyTo(frame);
1207 }
Per Åhgrena1351272019-08-15 12:15:46 +02001208 }
Sam Zackrisson0824c6f2019-10-07 14:03:56 +02001209 if (capture_.stats.voice_detected) {
1210 frame->vad_activity_ = *capture_.stats.voice_detected
1211 ? AudioFrame::kVadActive
1212 : AudioFrame::kVadPassive;
1213 }
andrew@webrtc.org17e40642014-03-04 20:58:13 +00001214
aleloi868f32f2017-05-23 07:20:05 -07001215 if (aec_dump_) {
1216 RecordProcessedCaptureStream(*frame);
1217 }
andrew@webrtc.org17e40642014-03-04 20:58:13 +00001218
1219 return kNoError;
1220}
1221
peahde65ddc2016-09-16 15:02:15 -07001222int AudioProcessingImpl::ProcessCaptureStreamLocked() {
Alex Loiko73ec0192018-05-15 10:52:28 +02001223 HandleCaptureRuntimeSettings();
Alessio Bazzicac054e782018-04-16 12:10:09 +02001224
peahb58a1582016-03-15 09:34:24 -07001225 // Ensure that not both the AEC and AECM are active at the same time.
Sam Zackrisson2a959d92018-07-23 14:48:07 +00001226 // TODO(peah): Simplify once the public API Enable functions for these
1227 // are moved to APM.
saza1d600522019-10-18 13:29:43 +02001228 RTC_DCHECK_LE(!!submodules_.echo_controller +
1229 !!submodules_.echo_cancellation +
1230 !!submodules_.echo_control_mobile,
Per Åhgrenb6e24d72019-04-29 12:14:50 +02001231 1);
peahb58a1582016-03-15 09:34:24 -07001232
peahde65ddc2016-09-16 15:02:15 -07001233 AudioBuffer* capture_buffer = capture_.capture_audio.get(); // For brevity.
ekmeyerson60d9b332015-08-14 10:35:55 -07001234
saza1d600522019-10-18 13:29:43 +02001235 if (submodules_.pre_amplifier) {
1236 submodules_.pre_amplifier->ApplyGain(AudioFrameView<float>(
Per Åhgrend47941e2019-08-22 11:51:13 +02001237 capture_buffer->channels(), capture_buffer->num_channels(),
Alex Loikob5c9a792018-04-16 16:31:22 +02001238 capture_buffer->num_frames()));
1239 }
1240
Per Åhgren928146f2019-08-20 09:19:21 +02001241 capture_input_rms_.Analyze(rtc::ArrayView<const float>(
Per Åhgrend47941e2019-08-22 11:51:13 +02001242 capture_buffer->channels_const()[0],
henrik.lundin290d43a2016-11-29 08:09:09 -08001243 capture_nonlocked_.capture_processing_format.num_frames()));
peah1b08dc32016-12-20 13:45:58 -08001244 const bool log_rms = ++capture_rms_interval_counter_ >= 1000;
1245 if (log_rms) {
1246 capture_rms_interval_counter_ = 0;
1247 RmsLevel::Levels levels = capture_input_rms_.AverageAndPeak();
henrik.lundin45bb5132016-12-06 04:28:04 -08001248 RTC_HISTOGRAM_COUNTS_LINEAR("WebRTC.Audio.ApmCaptureInputLevelAverageRms",
1249 levels.average, 1, RmsLevel::kMinLevelDb, 64);
1250 RTC_HISTOGRAM_COUNTS_LINEAR("WebRTC.Audio.ApmCaptureInputLevelPeakRms",
1251 levels.peak, 1, RmsLevel::kMinLevelDb, 64);
henrik.lundin290d43a2016-11-29 08:09:09 -08001252 }
1253
saza1d600522019-10-18 13:29:43 +02001254 if (submodules_.echo_controller) {
Per Åhgren88cf0502018-07-16 17:08:41 +02001255 // Detect and flag any change in the analog gain.
Per Åhgren0e3198e2019-11-18 08:52:22 +01001256 int analog_mic_level = recommended_stream_analog_level();
Per Åhgren88cf0502018-07-16 17:08:41 +02001257 capture_.echo_path_gain_change =
1258 capture_.prev_analog_mic_level != analog_mic_level &&
1259 capture_.prev_analog_mic_level != -1;
1260 capture_.prev_analog_mic_level = analog_mic_level;
1261
Per Åhgrend2650d12018-10-02 17:00:59 +02001262 // Detect and flag any change in the pre-amplifier gain.
saza1d600522019-10-18 13:29:43 +02001263 if (submodules_.pre_amplifier) {
1264 float pre_amp_gain = submodules_.pre_amplifier->GetGainFactor();
Per Åhgrend2650d12018-10-02 17:00:59 +02001265 capture_.echo_path_gain_change =
1266 capture_.echo_path_gain_change ||
1267 (capture_.prev_pre_amp_gain != pre_amp_gain &&
Per Åhgrene8a55692018-10-02 23:10:38 +02001268 capture_.prev_pre_amp_gain >= 0.f);
Per Åhgrend2650d12018-10-02 17:00:59 +02001269 capture_.prev_pre_amp_gain = pre_amp_gain;
1270 }
Fredrik Hernqvistca362852019-05-10 15:50:02 +02001271
1272 // Detect volume change.
1273 capture_.echo_path_gain_change =
1274 capture_.echo_path_gain_change ||
1275 (capture_.prev_playout_volume != capture_.playout_volume &&
1276 capture_.prev_playout_volume >= 0);
1277 capture_.prev_playout_volume = capture_.playout_volume;
1278
saza1d600522019-10-18 13:29:43 +02001279 submodules_.echo_controller->AnalyzeCapture(capture_buffer);
peahe0eae3c2016-12-14 01:16:23 -08001280 }
1281
Per Åhgren3daedb62019-11-22 12:11:40 +01001282 if (constants_.use_experimental_agc &&
1283 submodules_.gain_control->is_enabled()) {
1284 submodules_.agc_manager->AnalyzePreProcess(capture_buffer);
pbos@webrtc.org788acd12014-12-15 09:41:24 +00001285 }
1286
peah2ace3f92016-09-10 04:42:27 -07001287 if (submodule_states_.CaptureMultiBandSubModulesActive() &&
1288 SampleRateSupportsMultiBand(
peahde65ddc2016-09-16 15:02:15 -07001289 capture_nonlocked_.capture_processing_format.sample_rate_hz())) {
1290 capture_buffer->SplitIntoFrequencyBands();
niklase@google.com470e71d2011-07-07 08:21:25 +00001291 }
1292
Per Åhgrene14cb992019-11-27 09:34:22 +01001293 const bool multi_channel_capture = config_.pipeline.multi_channel_capture &&
1294 constants_.multi_channel_capture_support;
1295 if (submodules_.echo_controller && !multi_channel_capture) {
peah522d71b2017-02-23 05:16:26 -08001296 // Force down-mixing of the number of channels after the detection of
1297 // capture signal saturation.
1298 // TODO(peah): Look into ensuring that this kind of tampering with the
1299 // AudioBuffer functionality should not be needed.
1300 capture_buffer->set_num_channels(1);
1301 }
1302
saza1d600522019-10-18 13:29:43 +02001303 if (submodules_.high_pass_filter) {
1304 submodules_.high_pass_filter->Process(capture_buffer);
peah8271d042016-11-22 07:24:52 -08001305 }
Per Åhgren0cbb58e2019-10-29 22:59:44 +01001306
Per Åhgrene35b32c2019-11-22 18:22:04 +01001307 RETURN_ON_ERR(submodules_.gain_control->AnalyzeCaptureAudio(*capture_buffer));
Per Åhgren0cbb58e2019-10-29 22:59:44 +01001308 RTC_DCHECK(
1309 !(submodules_.legacy_noise_suppressor && submodules_.noise_suppressor));
saza1d600522019-10-18 13:29:43 +02001310 if (submodules_.noise_suppressor) {
Per Åhgren0cbb58e2019-10-29 22:59:44 +01001311 submodules_.noise_suppressor->Analyze(*capture_buffer);
1312 } else if (submodules_.legacy_noise_suppressor) {
1313 submodules_.legacy_noise_suppressor->AnalyzeCaptureAudio(capture_buffer);
saza0bad15f2019-10-16 11:46:11 +02001314 }
peahb58a1582016-03-15 09:34:24 -07001315
saza1d600522019-10-18 13:29:43 +02001316 if (submodules_.echo_control_mobile) {
Per Åhgrenb6e24d72019-04-29 12:14:50 +02001317 // Ensure that the stream delay was set before the call to the
1318 // AECM ProcessCaptureAudio function.
1319 if (!was_stream_delay_set()) {
1320 return AudioProcessing::kStreamParameterNotSetError;
Per Åhgrend0fa8202018-04-18 09:35:13 +02001321 }
1322
saza1d600522019-10-18 13:29:43 +02001323 if (submodules_.noise_suppressor) {
Per Åhgren0cbb58e2019-10-29 22:59:44 +01001324 submodules_.noise_suppressor->Process(capture_buffer);
1325 } else if (submodules_.legacy_noise_suppressor) {
saza1d600522019-10-18 13:29:43 +02001326 submodules_.echo_control_mobile->CopyLowPassReference(capture_buffer);
Per Åhgren0cbb58e2019-10-29 22:59:44 +01001327 submodules_.legacy_noise_suppressor->ProcessCaptureAudio(capture_buffer);
Per Åhgrenb6e24d72019-04-29 12:14:50 +02001328 }
peahe0eae3c2016-12-14 01:16:23 -08001329
saza1d600522019-10-18 13:29:43 +02001330 RETURN_ON_ERR(submodules_.echo_control_mobile->ProcessCaptureAudio(
Per Åhgren46537a32017-06-07 10:08:10 +02001331 capture_buffer, stream_delay_ms()));
Per Åhgrenb6e24d72019-04-29 12:14:50 +02001332 } else {
saza1d600522019-10-18 13:29:43 +02001333 if (submodules_.echo_controller) {
Per Åhgrenb6e24d72019-04-29 12:14:50 +02001334 data_dumper_->DumpRaw("stream_delay", stream_delay_ms());
1335
1336 if (was_stream_delay_set()) {
saza1d600522019-10-18 13:29:43 +02001337 submodules_.echo_controller->SetAudioBufferDelay(stream_delay_ms());
Per Åhgrenb6e24d72019-04-29 12:14:50 +02001338 }
1339
Per Åhgrenc20a19c2019-11-13 11:12:29 +01001340 AudioBuffer* linear_aec_buffer = capture_.linear_aec_output.get();
saza1d600522019-10-18 13:29:43 +02001341 submodules_.echo_controller->ProcessCapture(
Per Åhgrenc20a19c2019-11-13 11:12:29 +01001342 capture_buffer, linear_aec_buffer, capture_.echo_path_gain_change);
saza1d600522019-10-18 13:29:43 +02001343 } else if (submodules_.echo_cancellation) {
Per Åhgrenb6e24d72019-04-29 12:14:50 +02001344 // Ensure that the stream delay was set before the call to the
1345 // AEC ProcessCaptureAudio function.
1346 if (!was_stream_delay_set()) {
1347 return AudioProcessing::kStreamParameterNotSetError;
1348 }
1349
saza1d600522019-10-18 13:29:43 +02001350 RETURN_ON_ERR(submodules_.echo_cancellation->ProcessCaptureAudio(
Per Åhgrenb6e24d72019-04-29 12:14:50 +02001351 capture_buffer, stream_delay_ms()));
1352 }
1353
saza1d600522019-10-18 13:29:43 +02001354 if (submodules_.noise_suppressor) {
Per Åhgren0cbb58e2019-10-29 22:59:44 +01001355 submodules_.noise_suppressor->Process(capture_buffer);
1356 } else if (submodules_.legacy_noise_suppressor) {
1357 submodules_.legacy_noise_suppressor->ProcessCaptureAudio(capture_buffer);
saza0bad15f2019-10-16 11:46:11 +02001358 }
Per Åhgren46537a32017-06-07 10:08:10 +02001359 }
ivoc9f4a4a02016-10-28 05:39:16 -07001360
Sam Zackrisson4db667b2018-12-21 16:29:27 +01001361 if (config_.voice_detection.enabled) {
Sam Zackrisson4db667b2018-12-21 16:29:27 +01001362 capture_.stats.voice_detected =
saza1d600522019-10-18 13:29:43 +02001363 submodules_.voice_detector->ProcessCaptureAudio(capture_buffer);
Sam Zackrisson4db667b2018-12-21 16:29:27 +01001364 } else {
1365 capture_.stats.voice_detected = absl::nullopt;
1366 }
pbos@webrtc.org788acd12014-12-15 09:41:24 +00001367
Per Åhgren3daedb62019-11-22 12:11:40 +01001368 if (constants_.use_experimental_agc &&
1369 submodules_.gain_control->is_enabled()) {
1370 submodules_.agc_manager->Process(capture_buffer);
1371
1372 absl::optional<int> new_digital_gain =
1373 submodules_.agc_manager->GetDigitalComressionGain();
1374 if (new_digital_gain) {
1375 submodules_.gain_control->set_compression_gain_db(*new_digital_gain);
1376 }
pbos@webrtc.org788acd12014-12-15 09:41:24 +00001377 }
Per Åhgren200feba2019-03-06 04:16:46 +01001378 // TODO(peah): Add reporting from AEC3 whether there is echo.
saza1d600522019-10-18 13:29:43 +02001379 RETURN_ON_ERR(submodules_.gain_control->ProcessCaptureAudio(
1380 capture_buffer, submodules_.echo_cancellation &&
1381 submodules_.echo_cancellation->stream_has_echo()));
niklase@google.com470e71d2011-07-07 08:21:25 +00001382
Gustaf Ullberg8675eee2019-10-09 13:34:36 +02001383 if (submodule_states_.CaptureMultiBandProcessingPresent() &&
peah2ace3f92016-09-10 04:42:27 -07001384 SampleRateSupportsMultiBand(
peahde65ddc2016-09-16 15:02:15 -07001385 capture_nonlocked_.capture_processing_format.sample_rate_hz())) {
1386 capture_buffer->MergeFrequencyBands();
niklase@google.com470e71d2011-07-07 08:21:25 +00001387 }
1388
Gustaf Ullberg422b9e02019-10-09 13:02:14 +02001389 if (capture_.capture_fullband_audio) {
saza1d600522019-10-18 13:29:43 +02001390 const auto& ec = submodules_.echo_controller;
Gustaf Ullberg8675eee2019-10-09 13:34:36 +02001391 bool ec_active = ec ? ec->ActiveProcessing() : false;
1392 // Only update the fullband buffer if the multiband processing has changed
1393 // the signal. Keep the original signal otherwise.
1394 if (submodule_states_.CaptureMultiBandProcessingActive(ec_active)) {
1395 capture_buffer->CopyTo(capture_.capture_fullband_audio.get());
1396 }
Gustaf Ullberg422b9e02019-10-09 13:02:14 +02001397 capture_buffer = capture_.capture_fullband_audio.get();
1398 }
1399
peah9e6a2902017-05-15 07:19:21 -07001400 if (config_.residual_echo_detector.enabled) {
saza1d600522019-10-18 13:29:43 +02001401 RTC_DCHECK(submodules_.echo_detector);
1402 submodules_.echo_detector->AnalyzeCaptureAudio(rtc::ArrayView<const float>(
1403 capture_buffer->channels()[0], capture_buffer->num_frames()));
peah9e6a2902017-05-15 07:19:21 -07001404 }
1405
pbos@webrtc.org788acd12014-12-15 09:41:24 +00001406 // TODO(aluebs): Investigate if the transient suppression placement should be
1407 // before or after the AGC.
peahdf3efa82015-11-28 12:35:15 -08001408 if (capture_.transient_suppressor_enabled) {
saza1d600522019-10-18 13:29:43 +02001409 float voice_probability = submodules_.agc_manager.get()
1410 ? submodules_.agc_manager->voice_probability()
1411 : 1.f;
pbos@webrtc.org788acd12014-12-15 09:41:24 +00001412
saza1d600522019-10-18 13:29:43 +02001413 submodules_.transient_suppressor->Suppress(
Per Åhgrend47941e2019-08-22 11:51:13 +02001414 capture_buffer->channels()[0], capture_buffer->num_frames(),
peahde65ddc2016-09-16 15:02:15 -07001415 capture_buffer->num_channels(),
Per Åhgrend47941e2019-08-22 11:51:13 +02001416 capture_buffer->split_bands_const(0)[kBand0To8kHz],
Per Åhgrena1351272019-08-15 12:15:46 +02001417 capture_buffer->num_frames_per_band(),
1418 capture_.keyboard_info.keyboard_data,
1419 capture_.keyboard_info.num_keyboard_frames, voice_probability,
peahdf3efa82015-11-28 12:35:15 -08001420 capture_.key_pressed);
pbos@webrtc.org788acd12014-12-15 09:41:24 +00001421 }
1422
Valeriia Nemychnikovaf06eb572018-08-29 10:37:09 +02001423 // Experimental APM sub-module that analyzes |capture_buffer|.
saza1d600522019-10-18 13:29:43 +02001424 if (submodules_.capture_analyzer) {
1425 submodules_.capture_analyzer->Analyze(capture_buffer);
Valeriia Nemychnikovaf06eb572018-08-29 10:37:09 +02001426 }
1427
Alessio Bazzica270f7b52017-10-13 11:05:17 +02001428 if (config_.gain_controller2.enabled) {
saza1d600522019-10-18 13:29:43 +02001429 submodules_.gain_controller2->NotifyAnalogLevel(
Per Åhgren0e3198e2019-11-18 08:52:22 +01001430 recommended_stream_analog_level());
saza1d600522019-10-18 13:29:43 +02001431 submodules_.gain_controller2->Process(capture_buffer);
alessiob3ec96df2017-05-22 06:57:06 -07001432 }
1433
saza1d600522019-10-18 13:29:43 +02001434 if (submodules_.capture_post_processor) {
1435 submodules_.capture_post_processor->Process(capture_buffer);
Sam Zackrisson0beac582017-09-25 12:04:02 +02001436 }
1437
andrew@webrtc.org755b04a2011-11-15 16:57:56 +00001438 // The level estimator operates on the recombined data.
Sam Zackrissonb24c00f2018-11-26 16:18:25 +01001439 if (config_.level_estimation.enabled) {
saza1d600522019-10-18 13:29:43 +02001440 submodules_.output_level_estimator->ProcessStream(*capture_buffer);
1441 capture_.stats.output_rms_dbfs = submodules_.output_level_estimator->RMS();
Sam Zackrissonb24c00f2018-11-26 16:18:25 +01001442 } else {
1443 capture_.stats.output_rms_dbfs = absl::nullopt;
1444 }
ajm@google.com808e0e02011-08-03 21:08:51 +00001445
Per Åhgren928146f2019-08-20 09:19:21 +02001446 capture_output_rms_.Analyze(rtc::ArrayView<const float>(
Per Åhgrend47941e2019-08-22 11:51:13 +02001447 capture_buffer->channels_const()[0],
peah1b08dc32016-12-20 13:45:58 -08001448 capture_nonlocked_.capture_processing_format.num_frames()));
1449 if (log_rms) {
1450 RmsLevel::Levels levels = capture_output_rms_.AverageAndPeak();
1451 RTC_HISTOGRAM_COUNTS_LINEAR("WebRTC.Audio.ApmCaptureOutputLevelAverageRms",
1452 levels.average, 1, RmsLevel::kMinLevelDb, 64);
1453 RTC_HISTOGRAM_COUNTS_LINEAR("WebRTC.Audio.ApmCaptureOutputLevelPeakRms",
1454 levels.peak, 1, RmsLevel::kMinLevelDb, 64);
1455 }
1456
Per Åhgren0e3198e2019-11-18 08:52:22 +01001457 if (submodules_.agc_manager) {
1458 int level = recommended_stream_analog_level();
1459 data_dumper_->DumpRaw("experimental_gain_control_stream_analog_level", 1,
1460 &level);
1461 }
1462
peahdf3efa82015-11-28 12:35:15 -08001463 capture_.was_stream_delay_set = false;
niklase@google.com470e71d2011-07-07 08:21:25 +00001464 return kNoError;
1465}
1466
Gustaf Ullberg8c51f2e2019-10-22 15:21:31 +02001467int AudioProcessingImpl::AnalyzeReverseStream(
1468 const float* const* data,
1469 const StreamConfig& reverse_config) {
1470 TRACE_EVENT0("webrtc", "AudioProcessing::AnalyzeReverseStream_StreamConfig");
1471 rtc::CritScope cs(&crit_render_);
1472 return AnalyzeReverseStreamLocked(data, reverse_config, reverse_config);
1473}
1474
peahde65ddc2016-09-16 15:02:15 -07001475int AudioProcessingImpl::ProcessReverseStream(const float* const* src,
1476 const StreamConfig& input_config,
1477 const StreamConfig& output_config,
1478 float* const* dest) {
peah369f8282015-12-17 06:42:29 -08001479 TRACE_EVENT0("webrtc", "AudioProcessing::ProcessReverseStream_StreamConfig");
peahdf3efa82015-11-28 12:35:15 -08001480 rtc::CritScope cs(&crit_render_);
peahde65ddc2016-09-16 15:02:15 -07001481 RETURN_ON_ERR(AnalyzeReverseStreamLocked(src, input_config, output_config));
Alex Loiko5825aa62017-12-18 16:02:40 +01001482 if (submodule_states_.RenderMultiBandProcessingActive() ||
1483 submodule_states_.RenderFullBandProcessingActive()) {
peahdf3efa82015-11-28 12:35:15 -08001484 render_.render_audio->CopyTo(formats_.api_format.reverse_output_stream(),
1485 dest);
peah2ace3f92016-09-10 04:42:27 -07001486 } else if (formats_.api_format.reverse_input_stream() !=
1487 formats_.api_format.reverse_output_stream()) {
peahde65ddc2016-09-16 15:02:15 -07001488 render_.render_converter->Convert(src, input_config.num_samples(), dest,
1489 output_config.num_samples());
ekmeyerson60d9b332015-08-14 10:35:55 -07001490 } else {
peahde65ddc2016-09-16 15:02:15 -07001491 CopyAudioIfNeeded(src, input_config.num_frames(),
1492 input_config.num_channels(), dest);
ekmeyerson60d9b332015-08-14 10:35:55 -07001493 }
1494
1495 return kNoError;
Michael Graczyk86c6d332015-07-23 11:41:39 -07001496}
1497
peahdf3efa82015-11-28 12:35:15 -08001498int AudioProcessingImpl::AnalyzeReverseStreamLocked(
ekmeyerson60d9b332015-08-14 10:35:55 -07001499 const float* const* src,
peahde65ddc2016-09-16 15:02:15 -07001500 const StreamConfig& input_config,
1501 const StreamConfig& output_config) {
peahdf3efa82015-11-28 12:35:15 -08001502 if (src == nullptr) {
andrew@webrtc.org17e40642014-03-04 20:58:13 +00001503 return kNullPointerError;
1504 }
andrew@webrtc.org17e40642014-03-04 20:58:13 +00001505
peahde65ddc2016-09-16 15:02:15 -07001506 if (input_config.num_channels() == 0) {
Michael Graczyk86c6d332015-07-23 11:41:39 -07001507 return kBadNumberChannelsError;
andrew@webrtc.org17e40642014-03-04 20:58:13 +00001508 }
1509
peahdf3efa82015-11-28 12:35:15 -08001510 ProcessingConfig processing_config = formats_.api_format;
peahde65ddc2016-09-16 15:02:15 -07001511 processing_config.reverse_input_stream() = input_config;
1512 processing_config.reverse_output_stream() = output_config;
Michael Graczyk86c6d332015-07-23 11:41:39 -07001513
peahdf3efa82015-11-28 12:35:15 -08001514 RETURN_ON_ERR(MaybeInitializeRender(processing_config));
Fredrik Solenbergbbf21a32018-04-12 22:44:09 +02001515 RTC_DCHECK_EQ(input_config.num_frames(),
1516 formats_.api_format.reverse_input_stream().num_frames());
Michael Graczyk86c6d332015-07-23 11:41:39 -07001517
aleloi868f32f2017-05-23 07:20:05 -07001518 if (aec_dump_) {
1519 const size_t channel_size =
1520 formats_.api_format.reverse_input_stream().num_frames();
1521 const size_t num_channels =
1522 formats_.api_format.reverse_input_stream().num_channels();
1523 aec_dump_->WriteRenderStreamMessage(
Alex Loikoe36e8bb2018-02-16 11:54:07 +01001524 AudioFrameView<const float>(src, num_channels, channel_size));
aleloi868f32f2017-05-23 07:20:05 -07001525 }
peahdf3efa82015-11-28 12:35:15 -08001526 render_.render_audio->CopyFrom(src,
1527 formats_.api_format.reverse_input_stream());
peahde65ddc2016-09-16 15:02:15 -07001528 return ProcessRenderStreamLocked();
ekmeyerson60d9b332015-08-14 10:35:55 -07001529}
1530
1531int AudioProcessingImpl::ProcessReverseStream(AudioFrame* frame) {
peah369f8282015-12-17 06:42:29 -08001532 TRACE_EVENT0("webrtc", "AudioProcessing::ProcessReverseStream_AudioFrame");
peahdf3efa82015-11-28 12:35:15 -08001533 rtc::CritScope cs(&crit_render_);
peahdf3efa82015-11-28 12:35:15 -08001534 if (frame == nullptr) {
niklase@google.com470e71d2011-07-07 08:21:25 +00001535 return kNullPointerError;
1536 }
andrew@webrtc.orgddbb8a22014-04-22 21:00:04 +00001537 // Must be a native rate.
1538 if (frame->sample_rate_hz_ != kSampleRate8kHz &&
1539 frame->sample_rate_hz_ != kSampleRate16kHz &&
aluebs@webrtc.org087da132014-11-17 23:01:23 +00001540 frame->sample_rate_hz_ != kSampleRate32kHz &&
1541 frame->sample_rate_hz_ != kSampleRate48kHz) {
andrew@webrtc.orgddbb8a22014-04-22 21:00:04 +00001542 return kBadSampleRateError;
1543 }
andrew@webrtc.orga8b97372014-03-10 22:26:12 +00001544
Michael Graczyk86c6d332015-07-23 11:41:39 -07001545 if (frame->num_channels_ <= 0) {
1546 return kBadNumberChannelsError;
1547 }
1548
peahdf3efa82015-11-28 12:35:15 -08001549 ProcessingConfig processing_config = formats_.api_format;
ekmeyerson60d9b332015-08-14 10:35:55 -07001550 processing_config.reverse_input_stream().set_sample_rate_hz(
1551 frame->sample_rate_hz_);
1552 processing_config.reverse_input_stream().set_num_channels(
1553 frame->num_channels_);
1554 processing_config.reverse_output_stream().set_sample_rate_hz(
1555 frame->sample_rate_hz_);
1556 processing_config.reverse_output_stream().set_num_channels(
1557 frame->num_channels_);
Michael Graczyk86c6d332015-07-23 11:41:39 -07001558
peahdf3efa82015-11-28 12:35:15 -08001559 RETURN_ON_ERR(MaybeInitializeRender(processing_config));
Michael Graczyk86c6d332015-07-23 11:41:39 -07001560 if (frame->samples_per_channel_ !=
peahdf3efa82015-11-28 12:35:15 -08001561 formats_.api_format.reverse_input_stream().num_frames()) {
andrew@webrtc.org17e40642014-03-04 20:58:13 +00001562 return kBadDataLengthError;
1563 }
niklase@google.com470e71d2011-07-07 08:21:25 +00001564
aleloi868f32f2017-05-23 07:20:05 -07001565 if (aec_dump_) {
1566 aec_dump_->WriteRenderStreamMessage(*frame);
1567 }
1568
Per Åhgrend47941e2019-08-22 11:51:13 +02001569 render_.render_audio->CopyFrom(frame);
peahde65ddc2016-09-16 15:02:15 -07001570 RETURN_ON_ERR(ProcessRenderStreamLocked());
Per Åhgrena1351272019-08-15 12:15:46 +02001571 if (submodule_states_.RenderMultiBandProcessingActive() ||
1572 submodule_states_.RenderFullBandProcessingActive()) {
Per Åhgrend47941e2019-08-22 11:51:13 +02001573 render_.render_audio->CopyTo(frame);
Per Åhgrena1351272019-08-15 12:15:46 +02001574 }
aluebsb0319552016-03-17 20:39:53 -07001575 return kNoError;
andrew@webrtc.org17e40642014-03-04 20:58:13 +00001576}
niklase@google.com470e71d2011-07-07 08:21:25 +00001577
peahde65ddc2016-09-16 15:02:15 -07001578int AudioProcessingImpl::ProcessRenderStreamLocked() {
1579 AudioBuffer* render_buffer = render_.render_audio.get(); // For brevity.
peah9e6a2902017-05-15 07:19:21 -07001580
Alex Loiko73ec0192018-05-15 10:52:28 +02001581 HandleRenderRuntimeSettings();
1582
saza1d600522019-10-18 13:29:43 +02001583 if (submodules_.render_pre_processor) {
1584 submodules_.render_pre_processor->Process(render_buffer);
Alex Loiko5825aa62017-12-18 16:02:40 +01001585 }
1586
Alessio Bazzicad2b97402018-08-09 14:23:11 +02001587 QueueNonbandedRenderAudio(render_buffer);
1588
peah2ace3f92016-09-10 04:42:27 -07001589 if (submodule_states_.RenderMultiBandSubModulesActive() &&
peahde65ddc2016-09-16 15:02:15 -07001590 SampleRateSupportsMultiBand(
1591 formats_.render_processing_format.sample_rate_hz())) {
1592 render_buffer->SplitIntoFrequencyBands();
niklase@google.com470e71d2011-07-07 08:21:25 +00001593 }
1594
peahce4d9152017-05-19 01:28:05 -07001595 if (submodule_states_.RenderMultiBandSubModulesActive()) {
1596 QueueBandedRenderAudio(render_buffer);
1597 }
1598
Alessio Bazzicad2b97402018-08-09 14:23:11 +02001599 // TODO(peah): Perform the queuing inside QueueRenderAudiuo().
saza1d600522019-10-18 13:29:43 +02001600 if (submodules_.echo_controller) {
1601 submodules_.echo_controller->AnalyzeRender(render_buffer);
peahe0eae3c2016-12-14 01:16:23 -08001602 }
niklase@google.com470e71d2011-07-07 08:21:25 +00001603
peah2ace3f92016-09-10 04:42:27 -07001604 if (submodule_states_.RenderMultiBandProcessingActive() &&
peahde65ddc2016-09-16 15:02:15 -07001605 SampleRateSupportsMultiBand(
1606 formats_.render_processing_format.sample_rate_hz())) {
1607 render_buffer->MergeFrequencyBands();
ekmeyerson60d9b332015-08-14 10:35:55 -07001608 }
1609
andrew@webrtc.org17e40642014-03-04 20:58:13 +00001610 return kNoError;
niklase@google.com470e71d2011-07-07 08:21:25 +00001611}
1612
1613int AudioProcessingImpl::set_stream_delay_ms(int delay) {
peahdf3efa82015-11-28 12:35:15 -08001614 rtc::CritScope cs(&crit_capture_);
andrew@webrtc.org5f23d642012-05-29 21:14:06 +00001615 Error retval = kNoError;
peahdf3efa82015-11-28 12:35:15 -08001616 capture_.was_stream_delay_set = true;
1617 delay += capture_.delay_offset_ms;
andrew@webrtc.org6f9f8172012-03-06 19:03:39 +00001618
niklase@google.com470e71d2011-07-07 08:21:25 +00001619 if (delay < 0) {
andrew@webrtc.org5f23d642012-05-29 21:14:06 +00001620 delay = 0;
1621 retval = kBadStreamParameterWarning;
niklase@google.com470e71d2011-07-07 08:21:25 +00001622 }
1623
1624 // TODO(ajm): the max is rather arbitrarily chosen; investigate.
1625 if (delay > 500) {
andrew@webrtc.org5f23d642012-05-29 21:14:06 +00001626 delay = 500;
1627 retval = kBadStreamParameterWarning;
niklase@google.com470e71d2011-07-07 08:21:25 +00001628 }
1629
peahdf3efa82015-11-28 12:35:15 -08001630 capture_nonlocked_.stream_delay_ms = delay;
andrew@webrtc.org5f23d642012-05-29 21:14:06 +00001631 return retval;
niklase@google.com470e71d2011-07-07 08:21:25 +00001632}
1633
Per Åhgrenc20a19c2019-11-13 11:12:29 +01001634bool AudioProcessingImpl::GetLinearAecOutput(
1635 rtc::ArrayView<std::array<float, 160>> linear_output) const {
1636 rtc::CritScope cs(&crit_capture_);
1637 AudioBuffer* linear_aec_buffer = capture_.linear_aec_output.get();
1638
1639 RTC_DCHECK(linear_aec_buffer);
1640 if (linear_aec_buffer) {
1641 RTC_DCHECK_EQ(1, linear_aec_buffer->num_bands());
1642 RTC_DCHECK_EQ(linear_output.size(), linear_aec_buffer->num_channels());
1643
1644 for (size_t ch = 0; ch < linear_aec_buffer->num_channels(); ++ch) {
1645 RTC_DCHECK_EQ(linear_output[ch].size(), linear_aec_buffer->num_frames());
1646 rtc::ArrayView<const float> channel_view =
1647 rtc::ArrayView<const float>(linear_aec_buffer->channels_const()[ch],
1648 linear_aec_buffer->num_frames());
1649 std::copy(channel_view.begin(), channel_view.end(),
1650 linear_output[ch].begin());
1651 }
1652 return true;
1653 }
1654 RTC_LOG(LS_ERROR) << "No linear AEC output available";
1655 RTC_NOTREACHED();
1656 return false;
1657}
1658
niklase@google.com470e71d2011-07-07 08:21:25 +00001659int AudioProcessingImpl::stream_delay_ms() const {
peahdf3efa82015-11-28 12:35:15 -08001660 // Used as callback from submodules, hence locking is not allowed.
1661 return capture_nonlocked_.stream_delay_ms;
niklase@google.com470e71d2011-07-07 08:21:25 +00001662}
1663
1664bool AudioProcessingImpl::was_stream_delay_set() const {
peahdf3efa82015-11-28 12:35:15 -08001665 // Used as callback from submodules, hence locking is not allowed.
1666 return capture_.was_stream_delay_set;
niklase@google.com470e71d2011-07-07 08:21:25 +00001667}
1668
andrew@webrtc.org17e40642014-03-04 20:58:13 +00001669void AudioProcessingImpl::set_stream_key_pressed(bool key_pressed) {
peahdf3efa82015-11-28 12:35:15 -08001670 rtc::CritScope cs(&crit_capture_);
1671 capture_.key_pressed = key_pressed;
andrew@webrtc.org17e40642014-03-04 20:58:13 +00001672}
1673
andrew@webrtc.org6f9f8172012-03-06 19:03:39 +00001674void AudioProcessingImpl::set_delay_offset_ms(int offset) {
peahdf3efa82015-11-28 12:35:15 -08001675 rtc::CritScope cs(&crit_capture_);
1676 capture_.delay_offset_ms = offset;
andrew@webrtc.org6f9f8172012-03-06 19:03:39 +00001677}
1678
1679int AudioProcessingImpl::delay_offset_ms() const {
peahdf3efa82015-11-28 12:35:15 -08001680 rtc::CritScope cs(&crit_capture_);
1681 return capture_.delay_offset_ms;
andrew@webrtc.org6f9f8172012-03-06 19:03:39 +00001682}
1683
Sam Zackrissonf0d1c032019-03-27 13:28:08 +01001684void AudioProcessingImpl::set_stream_analog_level(int level) {
1685 rtc::CritScope cs_capture(&crit_capture_);
Per Åhgren0e3198e2019-11-18 08:52:22 +01001686
1687 if (submodules_.agc_manager) {
1688 submodules_.agc_manager->set_stream_analog_level(level);
1689 data_dumper_->DumpRaw("experimental_gain_control_set_stream_analog_level",
1690 1, &level);
1691 } else {
1692 int error = submodules_.gain_control->set_stream_analog_level(level);
1693 RTC_DCHECK_EQ(kNoError, error);
1694 }
Sam Zackrissonf0d1c032019-03-27 13:28:08 +01001695}
1696
1697int AudioProcessingImpl::recommended_stream_analog_level() const {
1698 rtc::CritScope cs_capture(&crit_capture_);
Per Åhgren0e3198e2019-11-18 08:52:22 +01001699 if (submodules_.agc_manager) {
1700 return submodules_.agc_manager->stream_analog_level();
1701 }
1702 return submodules_.gain_control->stream_analog_level();
Sam Zackrissonf0d1c032019-03-27 13:28:08 +01001703}
1704
aleloi868f32f2017-05-23 07:20:05 -07001705void AudioProcessingImpl::AttachAecDump(std::unique_ptr<AecDump> aec_dump) {
1706 RTC_DCHECK(aec_dump);
1707 rtc::CritScope cs_render(&crit_render_);
1708 rtc::CritScope cs_capture(&crit_capture_);
1709
1710 // The previously attached AecDump will be destroyed with the
1711 // 'aec_dump' parameter, which is after locks are released.
1712 aec_dump_.swap(aec_dump);
1713 WriteAecDumpConfigMessage(true);
Minyue Li656d6092018-08-10 15:38:52 +02001714 aec_dump_->WriteInitMessage(formats_.api_format, rtc::TimeUTCMillis());
aleloi868f32f2017-05-23 07:20:05 -07001715}
1716
1717void AudioProcessingImpl::DetachAecDump() {
1718 // The d-tor of a task-queue based AecDump blocks until all pending
1719 // tasks are done. This construction avoids blocking while holding
1720 // the render and capture locks.
1721 std::unique_ptr<AecDump> aec_dump = nullptr;
1722 {
1723 rtc::CritScope cs_render(&crit_render_);
1724 rtc::CritScope cs_capture(&crit_capture_);
1725 aec_dump = std::move(aec_dump_);
1726 }
1727}
1728
Sam Zackrisson4d364492018-03-02 16:03:21 +01001729void AudioProcessingImpl::AttachPlayoutAudioGenerator(
1730 std::unique_ptr<AudioGenerator> audio_generator) {
1731 // TODO(bugs.webrtc.org/8882) Stub.
1732 // Reset internal audio generator with audio_generator.
1733}
1734
1735void AudioProcessingImpl::DetachPlayoutAudioGenerator() {
1736 // TODO(bugs.webrtc.org/8882) Stub.
1737 // Delete audio generator, if one is attached.
1738}
1739
Ivo Creusen56d46092017-11-24 17:29:59 +01001740AudioProcessingStats AudioProcessingImpl::GetStatistics(
Ivo Creusenae026092017-11-20 13:07:16 +01001741 bool has_remote_tracks) const {
Sam Zackrissonb24c00f2018-11-26 16:18:25 +01001742 rtc::CritScope cs_capture(&crit_capture_);
1743 if (!has_remote_tracks) {
1744 return capture_.stats;
1745 }
1746 AudioProcessingStats stats = capture_.stats;
1747 EchoCancellationImpl::Metrics metrics;
saza1d600522019-10-18 13:29:43 +02001748 if (submodules_.echo_controller) {
1749 auto ec_metrics = submodules_.echo_controller->GetMetrics();
Sam Zackrissonb24c00f2018-11-26 16:18:25 +01001750 stats.echo_return_loss = ec_metrics.echo_return_loss;
1751 stats.echo_return_loss_enhancement =
1752 ec_metrics.echo_return_loss_enhancement;
1753 stats.delay_ms = ec_metrics.delay_ms;
Sam Zackrissonb24c00f2018-11-26 16:18:25 +01001754 }
1755 if (config_.residual_echo_detector.enabled) {
saza1d600522019-10-18 13:29:43 +02001756 RTC_DCHECK(submodules_.echo_detector);
1757 auto ed_metrics = submodules_.echo_detector->GetMetrics();
Sam Zackrissonb24c00f2018-11-26 16:18:25 +01001758 stats.residual_echo_likelihood = ed_metrics.echo_likelihood;
1759 stats.residual_echo_likelihood_recent_max =
1760 ed_metrics.echo_likelihood_recent_max;
1761 }
Ivo Creusenae026092017-11-20 13:07:16 +01001762 return stats;
1763}
1764
peah8271d042016-11-22 07:24:52 -08001765void AudioProcessingImpl::MutateConfig(
1766 rtc::FunctionView<void(AudioProcessing::Config*)> mutator) {
1767 rtc::CritScope cs_render(&crit_render_);
1768 rtc::CritScope cs_capture(&crit_capture_);
1769 mutator(&config_);
1770 ApplyConfig(config_);
1771}
1772
1773AudioProcessing::Config AudioProcessingImpl::GetConfig() const {
1774 rtc::CritScope cs_render(&crit_render_);
1775 rtc::CritScope cs_capture(&crit_capture_);
1776 return config_;
1777}
1778
peah2ace3f92016-09-10 04:42:27 -07001779bool AudioProcessingImpl::UpdateActiveSubmoduleStates() {
1780 return submodule_states_.Update(
saza1d600522019-10-18 13:29:43 +02001781 config_.high_pass_filter.enabled, !!submodules_.echo_cancellation,
1782 !!submodules_.echo_control_mobile, config_.residual_echo_detector.enabled,
Per Åhgren0cbb58e2019-10-29 22:59:44 +01001783 !!submodules_.legacy_noise_suppressor || !!submodules_.noise_suppressor,
1784 submodules_.gain_control->is_enabled(), config_.gain_controller2.enabled,
1785 config_.pre_amplifier.enabled, capture_nonlocked_.echo_controller_enabled,
saza0bad15f2019-10-16 11:46:11 +02001786 config_.voice_detection.enabled, capture_.transient_suppressor_enabled);
ekmeyerson60d9b332015-08-14 10:35:55 -07001787}
1788
Bjorn Volckeradc46c42015-04-15 11:42:40 +02001789void AudioProcessingImpl::InitializeTransient() {
peahdf3efa82015-11-28 12:35:15 -08001790 if (capture_.transient_suppressor_enabled) {
saza1d600522019-10-18 13:29:43 +02001791 if (!submodules_.transient_suppressor.get()) {
1792 submodules_.transient_suppressor.reset(new TransientSuppressor());
pbos@webrtc.org788acd12014-12-15 09:41:24 +00001793 }
saza1d600522019-10-18 13:29:43 +02001794 submodules_.transient_suppressor->Initialize(proc_fullband_sample_rate_hz(),
1795 capture_nonlocked_.split_rate,
1796 num_proc_channels());
pbos@webrtc.org788acd12014-12-15 09:41:24 +00001797 }
pbos@webrtc.org788acd12014-12-15 09:41:24 +00001798}
1799
Per Åhgren0aefbf02019-08-23 21:29:17 +02001800void AudioProcessingImpl::InitializeHighPassFilter() {
1801 if (submodule_states_.HighPassFilteringRequired()) {
saza1d600522019-10-18 13:29:43 +02001802 submodules_.high_pass_filter.reset(new HighPassFilter(num_proc_channels()));
peah8271d042016-11-22 07:24:52 -08001803 } else {
saza1d600522019-10-18 13:29:43 +02001804 submodules_.high_pass_filter.reset();
peah8271d042016-11-22 07:24:52 -08001805 }
1806}
alessiob3ec96df2017-05-22 06:57:06 -07001807
Sam Zackrisson0824c6f2019-10-07 14:03:56 +02001808void AudioProcessingImpl::InitializeVoiceDetector() {
1809 if (config_.voice_detection.enabled) {
saza1d600522019-10-18 13:29:43 +02001810 submodules_.voice_detector = std::make_unique<VoiceDetection>(
Sam Zackrisson0824c6f2019-10-07 14:03:56 +02001811 proc_split_sample_rate_hz(), VoiceDetection::kVeryLowLikelihood);
1812 } else {
saza1d600522019-10-18 13:29:43 +02001813 submodules_.voice_detector.reset();
Sam Zackrisson0824c6f2019-10-07 14:03:56 +02001814 }
1815}
Gustaf Ullberg8eb9c7d2017-10-14 08:28:46 +02001816void AudioProcessingImpl::InitializeEchoController() {
Per Åhgrenb6e24d72019-04-29 12:14:50 +02001817 bool use_echo_controller =
1818 echo_control_factory_ ||
Per Åhgren200feba2019-03-06 04:16:46 +01001819 (config_.echo_canceller.enabled && !config_.echo_canceller.mobile_mode &&
Per Åhgrenb6e24d72019-04-29 12:14:50 +02001820 !config_.echo_canceller.use_legacy_aec);
1821
1822 if (use_echo_controller) {
1823 // Create and activate the echo controller.
Per Åhgren200feba2019-03-06 04:16:46 +01001824 if (echo_control_factory_) {
Per Åhgren4e5c7092019-11-01 20:44:11 +01001825 submodules_.echo_controller = echo_control_factory_->Create(
1826 proc_sample_rate_hz(), num_reverse_channels(), num_proc_channels());
Gustaf Ullberg2c6f3732019-11-07 17:15:12 +01001827 RTC_DCHECK(submodules_.echo_controller);
Per Åhgren200feba2019-03-06 04:16:46 +01001828 } else {
saza1d600522019-10-18 13:29:43 +02001829 submodules_.echo_controller = std::make_unique<EchoCanceller3>(
Sam Zackrissonfeee1e42019-09-20 07:50:35 +02001830 EchoCanceller3Config(), proc_sample_rate_hz(), num_reverse_channels(),
1831 num_proc_channels());
Per Åhgren200feba2019-03-06 04:16:46 +01001832 }
1833
Per Åhgrenc20a19c2019-11-13 11:12:29 +01001834 // Setup the storage for returning the linear AEC output.
1835 if (config_.echo_canceller.export_linear_aec_output) {
1836 constexpr int kLinearOutputRateHz = 16000;
1837 capture_.linear_aec_output = std::make_unique<AudioBuffer>(
1838 kLinearOutputRateHz, num_proc_channels(), kLinearOutputRateHz,
1839 num_proc_channels(), kLinearOutputRateHz, num_proc_channels());
1840 } else {
1841 capture_.linear_aec_output.reset();
1842 }
1843
Per Åhgren200feba2019-03-06 04:16:46 +01001844 capture_nonlocked_.echo_controller_enabled = true;
Per Åhgren200feba2019-03-06 04:16:46 +01001845
saza1d600522019-10-18 13:29:43 +02001846 submodules_.echo_cancellation.reset();
Per Åhgrenf204faf2019-04-25 15:18:06 +02001847 aec_render_signal_queue_.reset();
saza1d600522019-10-18 13:29:43 +02001848 submodules_.echo_control_mobile.reset();
Per Åhgrenb6e24d72019-04-29 12:14:50 +02001849 aecm_render_signal_queue_.reset();
Per Åhgrenf204faf2019-04-25 15:18:06 +02001850 return;
peahe0eae3c2016-12-14 01:16:23 -08001851 }
Per Åhgrenf204faf2019-04-25 15:18:06 +02001852
saza1d600522019-10-18 13:29:43 +02001853 submodules_.echo_controller.reset();
Per Åhgrenf204faf2019-04-25 15:18:06 +02001854 capture_nonlocked_.echo_controller_enabled = false;
Per Åhgrenc20a19c2019-11-13 11:12:29 +01001855 capture_.linear_aec_output.reset();
Per Åhgrenf204faf2019-04-25 15:18:06 +02001856
1857 if (!config_.echo_canceller.enabled) {
saza1d600522019-10-18 13:29:43 +02001858 submodules_.echo_cancellation.reset();
Per Åhgrenf204faf2019-04-25 15:18:06 +02001859 aec_render_signal_queue_.reset();
saza1d600522019-10-18 13:29:43 +02001860 submodules_.echo_control_mobile.reset();
Per Åhgrenb6e24d72019-04-29 12:14:50 +02001861 aecm_render_signal_queue_.reset();
Per Åhgrenf204faf2019-04-25 15:18:06 +02001862 return;
1863 }
1864
1865 if (config_.echo_canceller.mobile_mode) {
1866 // Create and activate AECM.
Per Åhgrenb6e24d72019-04-29 12:14:50 +02001867 size_t max_element_size =
1868 std::max(static_cast<size_t>(1),
1869 kMaxAllowedValuesOfSamplesPerBand *
1870 EchoControlMobileImpl::NumCancellersRequired(
1871 num_output_channels(), num_reverse_channels()));
1872
1873 std::vector<int16_t> template_queue_element(max_element_size);
1874
1875 aecm_render_signal_queue_.reset(
1876 new SwapQueue<std::vector<int16_t>, RenderQueueItemVerifier<int16_t>>(
1877 kMaxNumFramesToBuffer, template_queue_element,
1878 RenderQueueItemVerifier<int16_t>(max_element_size)));
1879
1880 aecm_render_queue_buffer_.resize(max_element_size);
1881 aecm_capture_queue_buffer_.resize(max_element_size);
1882
saza1d600522019-10-18 13:29:43 +02001883 submodules_.echo_control_mobile.reset(new EchoControlMobileImpl());
Per Åhgrenb6e24d72019-04-29 12:14:50 +02001884
saza1d600522019-10-18 13:29:43 +02001885 submodules_.echo_control_mobile->Initialize(proc_split_sample_rate_hz(),
1886 num_reverse_channels(),
1887 num_output_channels());
Per Åhgrenb6e24d72019-04-29 12:14:50 +02001888
saza1d600522019-10-18 13:29:43 +02001889 submodules_.echo_cancellation.reset();
Per Åhgrenf204faf2019-04-25 15:18:06 +02001890 aec_render_signal_queue_.reset();
1891 return;
1892 }
1893
saza1d600522019-10-18 13:29:43 +02001894 submodules_.echo_control_mobile.reset();
Per Åhgrenb6e24d72019-04-29 12:14:50 +02001895 aecm_render_signal_queue_.reset();
1896
Per Åhgrenf204faf2019-04-25 15:18:06 +02001897 // Create and activate AEC2.
saza1d600522019-10-18 13:29:43 +02001898 submodules_.echo_cancellation.reset(new EchoCancellationImpl());
1899 submodules_.echo_cancellation->SetExtraOptions(
Per Åhgrenf204faf2019-04-25 15:18:06 +02001900 capture_nonlocked_.use_aec2_extended_filter,
1901 capture_nonlocked_.use_aec2_delay_agnostic,
1902 capture_nonlocked_.use_aec2_refined_adaptive_filter);
1903
Per Åhgrenb6e24d72019-04-29 12:14:50 +02001904 size_t element_max_size =
Per Åhgrenf204faf2019-04-25 15:18:06 +02001905 std::max(static_cast<size_t>(1),
1906 kMaxAllowedValuesOfSamplesPerBand *
1907 EchoCancellationImpl::NumCancellersRequired(
1908 num_output_channels(), num_reverse_channels()));
1909
Per Åhgrenb6e24d72019-04-29 12:14:50 +02001910 std::vector<float> template_queue_element(element_max_size);
Per Åhgrenf204faf2019-04-25 15:18:06 +02001911
Per Åhgrenb6e24d72019-04-29 12:14:50 +02001912 aec_render_signal_queue_.reset(
1913 new SwapQueue<std::vector<float>, RenderQueueItemVerifier<float>>(
1914 kMaxNumFramesToBuffer, template_queue_element,
1915 RenderQueueItemVerifier<float>(element_max_size)));
Per Åhgrenf204faf2019-04-25 15:18:06 +02001916
Per Åhgrenb6e24d72019-04-29 12:14:50 +02001917 aec_render_queue_buffer_.resize(element_max_size);
1918 aec_capture_queue_buffer_.resize(element_max_size);
Per Åhgrenf204faf2019-04-25 15:18:06 +02001919
saza1d600522019-10-18 13:29:43 +02001920 submodules_.echo_cancellation->Initialize(
Per Åhgrenf204faf2019-04-25 15:18:06 +02001921 proc_sample_rate_hz(), num_reverse_channels(), num_output_channels(),
1922 num_proc_channels());
1923
saza1d600522019-10-18 13:29:43 +02001924 submodules_.echo_cancellation->set_suppression_level(
Per Åhgrenf204faf2019-04-25 15:18:06 +02001925 config_.echo_canceller.legacy_moderate_suppression_level
1926 ? EchoCancellationImpl::SuppressionLevel::kModerateSuppression
1927 : EchoCancellationImpl::SuppressionLevel::kHighSuppression);
peahe0eae3c2016-12-14 01:16:23 -08001928}
peah8271d042016-11-22 07:24:52 -08001929
alessiob3ec96df2017-05-22 06:57:06 -07001930void AudioProcessingImpl::InitializeGainController2() {
Alessio Bazzica270f7b52017-10-13 11:05:17 +02001931 if (config_.gain_controller2.enabled) {
saza1d600522019-10-18 13:29:43 +02001932 submodules_.gain_controller2->Initialize(proc_fullband_sample_rate_hz());
alessiob3ec96df2017-05-22 06:57:06 -07001933 }
1934}
1935
saza0bad15f2019-10-16 11:46:11 +02001936void AudioProcessingImpl::InitializeNoiseSuppressor() {
Per Åhgren0cbb58e2019-10-29 22:59:44 +01001937 submodules_.legacy_noise_suppressor.reset();
1938 submodules_.noise_suppressor.reset();
1939
saza0bad15f2019-10-16 11:46:11 +02001940 if (config_.noise_suppression.enabled) {
Per Åhgren0cbb58e2019-10-29 22:59:44 +01001941 const bool use_legacy_ns =
1942 config_.noise_suppression.use_legacy_ns || enforced_usage_of_legacy_ns_;
1943
1944 if (!use_legacy_ns) {
1945 auto map_level =
1946 [](AudioProcessing::Config::NoiseSuppression::Level level) {
1947 using NoiseSuppresionConfig =
1948 AudioProcessing::Config::NoiseSuppression;
1949 switch (level) {
1950 case NoiseSuppresionConfig::kLow:
1951 return NsConfig::SuppressionLevel::k6dB;
1952 case NoiseSuppresionConfig::kModerate:
1953 return NsConfig::SuppressionLevel::k12dB;
1954 case NoiseSuppresionConfig::kHigh:
1955 return NsConfig::SuppressionLevel::k18dB;
1956 case NoiseSuppresionConfig::kVeryHigh:
1957 return NsConfig::SuppressionLevel::k21dB;
1958 default:
1959 RTC_NOTREACHED();
1960 }
1961 };
1962
1963 NsConfig cfg;
1964 cfg.target_level = map_level(config_.noise_suppression.level);
1965 submodules_.noise_suppressor = std::make_unique<NoiseSuppressor>(
1966 cfg, proc_sample_rate_hz(), num_proc_channels());
1967 } else {
1968 auto ns_level =
1969 NsConfigLevelToInterfaceLevel(config_.noise_suppression.level);
1970 submodules_.legacy_noise_suppressor = std::make_unique<NoiseSuppression>(
1971 num_proc_channels(), proc_sample_rate_hz(), ns_level);
1972 }
saza0bad15f2019-10-16 11:46:11 +02001973 }
1974}
1975
Alex Loikob5c9a792018-04-16 16:31:22 +02001976void AudioProcessingImpl::InitializePreAmplifier() {
1977 if (config_.pre_amplifier.enabled) {
saza1d600522019-10-18 13:29:43 +02001978 submodules_.pre_amplifier.reset(
Alex Loikob5c9a792018-04-16 16:31:22 +02001979 new GainApplier(true, config_.pre_amplifier.fixed_gain_factor));
1980 } else {
saza1d600522019-10-18 13:29:43 +02001981 submodules_.pre_amplifier.reset();
Alex Loikob5c9a792018-04-16 16:31:22 +02001982 }
1983}
1984
ivoc9f4a4a02016-10-28 05:39:16 -07001985void AudioProcessingImpl::InitializeResidualEchoDetector() {
saza1d600522019-10-18 13:29:43 +02001986 RTC_DCHECK(submodules_.echo_detector);
1987 submodules_.echo_detector->Initialize(
Gustaf Ullberg422b9e02019-10-09 13:02:14 +02001988 proc_fullband_sample_rate_hz(), 1,
Ivo Creusenb1facc12018-04-12 16:15:58 +02001989 formats_.render_processing_format.sample_rate_hz(), 1);
ivoc9f4a4a02016-10-28 05:39:16 -07001990}
1991
Valeriia Nemychnikovaf06eb572018-08-29 10:37:09 +02001992void AudioProcessingImpl::InitializeAnalyzer() {
saza1d600522019-10-18 13:29:43 +02001993 if (submodules_.capture_analyzer) {
1994 submodules_.capture_analyzer->Initialize(proc_fullband_sample_rate_hz(),
1995 num_proc_channels());
Valeriia Nemychnikovaf06eb572018-08-29 10:37:09 +02001996 }
1997}
1998
Sam Zackrisson0beac582017-09-25 12:04:02 +02001999void AudioProcessingImpl::InitializePostProcessor() {
saza1d600522019-10-18 13:29:43 +02002000 if (submodules_.capture_post_processor) {
2001 submodules_.capture_post_processor->Initialize(
Gustaf Ullberg422b9e02019-10-09 13:02:14 +02002002 proc_fullband_sample_rate_hz(), num_proc_channels());
Sam Zackrisson0beac582017-09-25 12:04:02 +02002003 }
2004}
2005
Alex Loiko5825aa62017-12-18 16:02:40 +01002006void AudioProcessingImpl::InitializePreProcessor() {
saza1d600522019-10-18 13:29:43 +02002007 if (submodules_.render_pre_processor) {
2008 submodules_.render_pre_processor->Initialize(
Alex Loiko5825aa62017-12-18 16:02:40 +01002009 formats_.render_processing_format.sample_rate_hz(),
2010 formats_.render_processing_format.num_channels());
2011 }
2012}
2013
Per Åhgrenea4c5df2019-05-03 09:00:08 +02002014void AudioProcessingImpl::UpdateHistogramsOnCallEnd() {}
Bjorn Volcker4e7aa432015-07-07 11:50:05 +02002015
aleloi868f32f2017-05-23 07:20:05 -07002016void AudioProcessingImpl::WriteAecDumpConfigMessage(bool forced) {
2017 if (!aec_dump_) {
2018 return;
2019 }
Per Åhgrenf204faf2019-04-25 15:18:06 +02002020
2021 std::string experiments_description = "";
saza1d600522019-10-18 13:29:43 +02002022 if (submodules_.echo_cancellation) {
Per Åhgrenf204faf2019-04-25 15:18:06 +02002023 experiments_description +=
saza1d600522019-10-18 13:29:43 +02002024 submodules_.echo_cancellation->GetExperimentsDescription();
Per Åhgrenf204faf2019-04-25 15:18:06 +02002025 }
aleloi868f32f2017-05-23 07:20:05 -07002026 // TODO(peah): Add semicolon-separated concatenations of experiment
2027 // descriptions for other submodules.
aleloi868f32f2017-05-23 07:20:05 -07002028 if (constants_.agc_clipped_level_min != kClippedLevelMin) {
2029 experiments_description += "AgcClippingLevelExperiment;";
2030 }
Gustaf Ullbergce045ac2017-10-16 13:49:04 +02002031 if (capture_nonlocked_.echo_controller_enabled) {
2032 experiments_description += "EchoController;";
aleloi868f32f2017-05-23 07:20:05 -07002033 }
Alessio Bazzica270f7b52017-10-13 11:05:17 +02002034 if (config_.gain_controller2.enabled) {
2035 experiments_description += "GainController2;";
2036 }
aleloi868f32f2017-05-23 07:20:05 -07002037
2038 InternalAPMConfig apm_config;
2039
Per Åhgren200feba2019-03-06 04:16:46 +01002040 apm_config.aec_enabled = config_.echo_canceller.enabled;
aleloi868f32f2017-05-23 07:20:05 -07002041 apm_config.aec_delay_agnostic_enabled =
saza1d600522019-10-18 13:29:43 +02002042 submodules_.echo_cancellation &&
2043 submodules_.echo_cancellation->is_delay_agnostic_enabled();
aleloi868f32f2017-05-23 07:20:05 -07002044 apm_config.aec_drift_compensation_enabled =
saza1d600522019-10-18 13:29:43 +02002045 submodules_.echo_cancellation &&
2046 submodules_.echo_cancellation->is_drift_compensation_enabled();
aleloi868f32f2017-05-23 07:20:05 -07002047 apm_config.aec_extended_filter_enabled =
saza1d600522019-10-18 13:29:43 +02002048 submodules_.echo_cancellation &&
2049 submodules_.echo_cancellation->is_extended_filter_enabled();
Per Åhgrenf204faf2019-04-25 15:18:06 +02002050 apm_config.aec_suppression_level =
saza1d600522019-10-18 13:29:43 +02002051 submodules_.echo_cancellation
2052 ? static_cast<int>(submodules_.echo_cancellation->suppression_level())
Per Åhgrenf204faf2019-04-25 15:18:06 +02002053 : 0;
aleloi868f32f2017-05-23 07:20:05 -07002054
saza1d600522019-10-18 13:29:43 +02002055 apm_config.aecm_enabled = !!submodules_.echo_control_mobile;
aleloi868f32f2017-05-23 07:20:05 -07002056 apm_config.aecm_comfort_noise_enabled =
saza1d600522019-10-18 13:29:43 +02002057 submodules_.echo_control_mobile &&
2058 submodules_.echo_control_mobile->is_comfort_noise_enabled();
Per Åhgrenb6e24d72019-04-29 12:14:50 +02002059 apm_config.aecm_routing_mode =
saza1d600522019-10-18 13:29:43 +02002060 submodules_.echo_control_mobile
2061 ? static_cast<int>(submodules_.echo_control_mobile->routing_mode())
Per Åhgrenb6e24d72019-04-29 12:14:50 +02002062 : 0;
aleloi868f32f2017-05-23 07:20:05 -07002063
saza1d600522019-10-18 13:29:43 +02002064 apm_config.agc_enabled = submodules_.gain_control->is_enabled();
2065 apm_config.agc_mode = static_cast<int>(submodules_.gain_control->mode());
aleloi868f32f2017-05-23 07:20:05 -07002066 apm_config.agc_limiter_enabled =
saza1d600522019-10-18 13:29:43 +02002067 submodules_.gain_control->is_limiter_enabled();
Per Åhgren0e3198e2019-11-18 08:52:22 +01002068 apm_config.noise_robust_agc_enabled = !!submodules_.agc_manager;
aleloi868f32f2017-05-23 07:20:05 -07002069
2070 apm_config.hpf_enabled = config_.high_pass_filter.enabled;
2071
saza0bad15f2019-10-16 11:46:11 +02002072 apm_config.ns_enabled = config_.noise_suppression.enabled;
2073 apm_config.ns_level = static_cast<int>(config_.noise_suppression.level);
aleloi868f32f2017-05-23 07:20:05 -07002074
2075 apm_config.transient_suppression_enabled =
2076 capture_.transient_suppressor_enabled;
aleloi868f32f2017-05-23 07:20:05 -07002077 apm_config.experiments_description = experiments_description;
Alex Loiko5feb30e2018-04-16 13:52:32 +02002078 apm_config.pre_amplifier_enabled = config_.pre_amplifier.enabled;
2079 apm_config.pre_amplifier_fixed_gain_factor =
2080 config_.pre_amplifier.fixed_gain_factor;
aleloi868f32f2017-05-23 07:20:05 -07002081
2082 if (!forced && apm_config == apm_config_for_aec_dump_) {
2083 return;
2084 }
2085 aec_dump_->WriteConfig(apm_config);
2086 apm_config_for_aec_dump_ = apm_config;
2087}
2088
2089void AudioProcessingImpl::RecordUnprocessedCaptureStream(
2090 const float* const* src) {
2091 RTC_DCHECK(aec_dump_);
2092 WriteAecDumpConfigMessage(false);
2093
2094 const size_t channel_size = formats_.api_format.input_stream().num_frames();
2095 const size_t num_channels = formats_.api_format.input_stream().num_channels();
2096 aec_dump_->AddCaptureStreamInput(
Alex Loikoe36e8bb2018-02-16 11:54:07 +01002097 AudioFrameView<const float>(src, num_channels, channel_size));
aleloi868f32f2017-05-23 07:20:05 -07002098 RecordAudioProcessingState();
2099}
2100
2101void AudioProcessingImpl::RecordUnprocessedCaptureStream(
2102 const AudioFrame& capture_frame) {
2103 RTC_DCHECK(aec_dump_);
2104 WriteAecDumpConfigMessage(false);
2105
2106 aec_dump_->AddCaptureStreamInput(capture_frame);
2107 RecordAudioProcessingState();
2108}
2109
2110void AudioProcessingImpl::RecordProcessedCaptureStream(
2111 const float* const* processed_capture_stream) {
2112 RTC_DCHECK(aec_dump_);
2113
2114 const size_t channel_size = formats_.api_format.output_stream().num_frames();
2115 const size_t num_channels =
2116 formats_.api_format.output_stream().num_channels();
Alex Loikoe36e8bb2018-02-16 11:54:07 +01002117 aec_dump_->AddCaptureStreamOutput(AudioFrameView<const float>(
2118 processed_capture_stream, num_channels, channel_size));
aleloi868f32f2017-05-23 07:20:05 -07002119 aec_dump_->WriteCaptureStreamMessage();
2120}
2121
2122void AudioProcessingImpl::RecordProcessedCaptureStream(
2123 const AudioFrame& processed_capture_frame) {
2124 RTC_DCHECK(aec_dump_);
2125
2126 aec_dump_->AddCaptureStreamOutput(processed_capture_frame);
2127 aec_dump_->WriteCaptureStreamMessage();
2128}
2129
2130void AudioProcessingImpl::RecordAudioProcessingState() {
2131 RTC_DCHECK(aec_dump_);
2132 AecDump::AudioProcessingState audio_proc_state;
2133 audio_proc_state.delay = capture_nonlocked_.stream_delay_ms;
2134 audio_proc_state.drift =
saza1d600522019-10-18 13:29:43 +02002135 submodules_.echo_cancellation
2136 ? submodules_.echo_cancellation->stream_drift_samples()
Per Åhgrenf204faf2019-04-25 15:18:06 +02002137 : 0;
Per Åhgren0e3198e2019-11-18 08:52:22 +01002138 audio_proc_state.level = recommended_stream_analog_level();
aleloi868f32f2017-05-23 07:20:05 -07002139 audio_proc_state.keypress = capture_.key_pressed;
2140 aec_dump_->AddAudioProcessingState(audio_proc_state);
2141}
2142
kwiberg83ffe452016-08-29 14:46:07 -07002143AudioProcessingImpl::ApmCaptureState::ApmCaptureState(
Sam Zackrisson9394f6f2018-06-14 10:11:35 +02002144 bool transient_suppressor_enabled)
Per Åhgrenea4c5df2019-05-03 09:00:08 +02002145 : delay_offset_ms(0),
kwiberg83ffe452016-08-29 14:46:07 -07002146 was_stream_delay_set(false),
kwiberg83ffe452016-08-29 14:46:07 -07002147 output_will_be_muted(false),
2148 key_pressed(false),
2149 transient_suppressor_enabled(transient_suppressor_enabled),
peahde65ddc2016-09-16 15:02:15 -07002150 capture_processing_format(kSampleRate16kHz),
peah67995532017-04-10 14:12:41 -07002151 split_rate(kSampleRate16kHz),
Per Åhgren88cf0502018-07-16 17:08:41 +02002152 echo_path_gain_change(false),
Per Åhgrend2650d12018-10-02 17:00:59 +02002153 prev_analog_mic_level(-1),
Fredrik Hernqvistca362852019-05-10 15:50:02 +02002154 prev_pre_amp_gain(-1.f),
2155 playout_volume(-1),
2156 prev_playout_volume(-1) {}
kwiberg83ffe452016-08-29 14:46:07 -07002157
2158AudioProcessingImpl::ApmCaptureState::~ApmCaptureState() = default;
2159
Per Åhgrena1351272019-08-15 12:15:46 +02002160void AudioProcessingImpl::ApmCaptureState::KeyboardInfo::Extract(
2161 const float* const* data,
2162 const StreamConfig& stream_config) {
2163 if (stream_config.has_keyboard()) {
2164 keyboard_data = data[stream_config.num_channels()];
2165 } else {
2166 keyboard_data = NULL;
2167 }
2168 num_keyboard_frames = stream_config.num_frames();
2169}
2170
kwiberg83ffe452016-08-29 14:46:07 -07002171AudioProcessingImpl::ApmRenderState::ApmRenderState() = default;
2172
2173AudioProcessingImpl::ApmRenderState::~ApmRenderState() = default;
2174
niklase@google.com470e71d2011-07-07 08:21:25 +00002175} // namespace webrtc