blob: 3b66ee532ceb62984a5e8db7092a797e79e4a47f [file] [log] [blame]
niklase@google.com470e71d2011-07-07 08:21:25 +00001/*
andrew@webrtc.org40654032012-01-30 20:51:15 +00002 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
niklase@google.com470e71d2011-07-07 08:21:25 +00003 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020011#include "modules/audio_processing/audio_processing_impl.h"
niklase@google.com470e71d2011-07-07 08:21:25 +000012
Michael Graczyk86c6d332015-07-23 11:41:39 -070013#include <algorithm>
Yves Gerey988cc082018-10-23 12:03:01 +020014#include <cstdint>
Mirko Bonadei317a1f02019-09-17 17:06:18 +020015#include <memory>
alessiob3ec96df2017-05-22 06:57:06 -070016#include <string>
Yves Gerey988cc082018-10-23 12:03:01 +020017#include <type_traits>
18#include <utility>
niklase@google.com470e71d2011-07-07 08:21:25 +000019
Yves Gerey988cc082018-10-23 12:03:01 +020020#include "absl/types/optional.h"
21#include "api/array_view.h"
Per Åhgren645f24c2020-03-16 12:06:02 +010022#include "api/audio/audio_frame.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020023#include "common_audio/audio_converter.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020024#include "common_audio/include/audio_util.h"
Per Åhgren09e9a832020-05-11 11:03:47 +020025#include "modules/audio_processing/aec_dump/aec_dump_factory.h"
Alex Loikob5c9a792018-04-16 16:31:22 +020026#include "modules/audio_processing/agc2/gain_applier.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020027#include "modules/audio_processing/audio_buffer.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020028#include "modules/audio_processing/common.h"
Yves Gerey988cc082018-10-23 12:03:01 +020029#include "modules/audio_processing/include/audio_frame_view.h"
Per Åhgren13735822018-02-12 21:42:56 +010030#include "modules/audio_processing/logging/apm_data_dumper.h"
Sam Zackrissonb37e59d2020-04-27 08:39:33 +020031#include "modules/audio_processing/optionally_built_submodule_creators.h"
Steve Anton10542f22019-01-11 09:11:00 -080032#include "rtc_base/atomic_ops.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020033#include "rtc_base/checks.h"
Steve Anton10542f22019-01-11 09:11:00 -080034#include "rtc_base/constructor_magic.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020035#include "rtc_base/logging.h"
Steve Anton10542f22019-01-11 09:11:00 -080036#include "rtc_base/ref_counted_object.h"
37#include "rtc_base/time_utils.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020038#include "rtc_base/trace_event.h"
Sam Zackrissonfeee1e42019-09-20 07:50:35 +020039#include "system_wrappers/include/field_trial.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020040#include "system_wrappers/include/metrics.h"
andrew@webrtc.org7bf26462011-12-03 00:03:31 +000041
Michael Graczyk86c6d332015-07-23 11:41:39 -070042#define RETURN_ON_ERR(expr) \
43 do { \
44 int err = (expr); \
45 if (err != kNoError) { \
46 return err; \
47 } \
andrew@webrtc.org60730cf2014-01-07 17:45:09 +000048 } while (0)
49
niklase@google.com470e71d2011-07-07 08:21:25 +000050namespace webrtc {
aluebsdf6416a2016-03-16 18:26:35 -070051
Alex Loiko73ec0192018-05-15 10:52:28 +020052constexpr int kRuntimeSettingQueueSize = 100;
aluebsdf6416a2016-03-16 18:26:35 -070053
Michael Graczyk86c6d332015-07-23 11:41:39 -070054namespace {
55
56static bool LayoutHasKeyboard(AudioProcessing::ChannelLayout layout) {
57 switch (layout) {
58 case AudioProcessing::kMono:
59 case AudioProcessing::kStereo:
60 return false;
61 case AudioProcessing::kMonoAndKeyboard:
62 case AudioProcessing::kStereoAndKeyboard:
63 return true;
64 }
65
kwiberg9e2be5f2016-09-14 05:23:22 -070066 RTC_NOTREACHED();
Michael Graczyk86c6d332015-07-23 11:41:39 -070067 return false;
68}
aluebsdf6416a2016-03-16 18:26:35 -070069
peah2ace3f92016-09-10 04:42:27 -070070bool SampleRateSupportsMultiBand(int sample_rate_hz) {
aluebsdf6416a2016-03-16 18:26:35 -070071 return sample_rate_hz == AudioProcessing::kSampleRate32kHz ||
72 sample_rate_hz == AudioProcessing::kSampleRate48kHz;
73}
74
Per Åhgrenc0424252019-12-10 13:04:15 +010075// Checks whether the high-pass filter should be done in the full-band.
76bool EnforceSplitBandHpf() {
77 return field_trial::IsEnabled("WebRTC-FullBandHpfKillSwitch");
78}
79
Per Åhgrenb2b58d82019-12-02 14:59:40 +010080// Checks whether AEC3 should be allowed to decide what the default
81// configuration should be based on the render and capture channel configuration
82// at hand.
83bool UseSetupSpecificDefaultAec3Congfig() {
84 return !field_trial::IsEnabled(
85 "WebRTC-Aec3SetupSpecificDefaultConfigDefaultsKillSwitch");
86}
87
Per Åhgrenc8626b62019-08-23 15:49:51 +020088// Identify the native processing rate that best handles a sample rate.
Per Åhgrenfcbe4072019-09-15 00:27:58 +020089int SuitableProcessRate(int minimum_rate,
90 int max_splitting_rate,
91 bool band_splitting_required) {
Per Åhgrenc8626b62019-08-23 15:49:51 +020092 const int uppermost_native_rate =
Per Åhgrenfcbe4072019-09-15 00:27:58 +020093 band_splitting_required ? max_splitting_rate : 48000;
Per Åhgrenc8626b62019-08-23 15:49:51 +020094 for (auto rate : {16000, 32000, 48000}) {
peah2ace3f92016-09-10 04:42:27 -070095 if (rate >= uppermost_native_rate) {
96 return uppermost_native_rate;
97 }
98 if (rate >= minimum_rate) {
aluebsdf6416a2016-03-16 18:26:35 -070099 return rate;
100 }
101 }
peah2ace3f92016-09-10 04:42:27 -0700102 RTC_NOTREACHED();
103 return uppermost_native_rate;
aluebsdf6416a2016-03-16 18:26:35 -0700104}
105
Sam Zackrissonf0d1c032019-03-27 13:28:08 +0100106GainControl::Mode Agc1ConfigModeToInterfaceMode(
107 AudioProcessing::Config::GainController1::Mode mode) {
108 using Agc1Config = AudioProcessing::Config::GainController1;
109 switch (mode) {
110 case Agc1Config::kAdaptiveAnalog:
111 return GainControl::kAdaptiveAnalog;
112 case Agc1Config::kAdaptiveDigital:
113 return GainControl::kAdaptiveDigital;
114 case Agc1Config::kFixedDigital:
115 return GainControl::kFixedDigital;
116 }
117}
118
peah9e6a2902017-05-15 07:19:21 -0700119// Maximum lengths that frame of samples being passed from the render side to
120// the capture side can have (does not apply to AEC3).
121static const size_t kMaxAllowedValuesOfSamplesPerBand = 160;
122static const size_t kMaxAllowedValuesOfSamplesPerFrame = 480;
123
peah764e3642016-10-22 05:04:30 -0700124// Maximum number of frames to buffer in the render queue.
125// TODO(peah): Decrease this once we properly handle hugely unbalanced
126// reverse and forward call numbers.
127static const size_t kMaxNumFramesToBuffer = 100;
Michael Graczyk86c6d332015-07-23 11:41:39 -0700128} // namespace
andrew@webrtc.org60730cf2014-01-07 17:45:09 +0000129
130// Throughout webrtc, it's assumed that success is represented by zero.
kwiberg@webrtc.org2ebfac52015-01-14 10:51:54 +0000131static_assert(AudioProcessing::kNoError == 0, "kNoError must be zero");
andrew@webrtc.org60730cf2014-01-07 17:45:09 +0000132
saza1d600522019-10-18 13:29:43 +0200133AudioProcessingImpl::SubmoduleStates::SubmoduleStates(
Alex Loiko5825aa62017-12-18 16:02:40 +0100134 bool capture_post_processor_enabled,
Valeriia Nemychnikovaf06eb572018-08-29 10:37:09 +0200135 bool render_pre_processor_enabled,
136 bool capture_analyzer_enabled)
Alex Loiko5825aa62017-12-18 16:02:40 +0100137 : capture_post_processor_enabled_(capture_post_processor_enabled),
Valeriia Nemychnikovaf06eb572018-08-29 10:37:09 +0200138 render_pre_processor_enabled_(render_pre_processor_enabled),
139 capture_analyzer_enabled_(capture_analyzer_enabled) {}
peah2ace3f92016-09-10 04:42:27 -0700140
saza1d600522019-10-18 13:29:43 +0200141bool AudioProcessingImpl::SubmoduleStates::Update(
Sam Zackrissoncb1b5562018-09-28 14:15:09 +0200142 bool high_pass_filter_enabled,
peah2ace3f92016-09-10 04:42:27 -0700143 bool mobile_echo_controller_enabled,
ivoc9f4a4a02016-10-28 05:39:16 -0700144 bool residual_echo_detector_enabled,
peah2ace3f92016-09-10 04:42:27 -0700145 bool noise_suppressor_enabled,
peah2ace3f92016-09-10 04:42:27 -0700146 bool adaptive_gain_controller_enabled,
alessiob3ec96df2017-05-22 06:57:06 -0700147 bool gain_controller2_enabled,
Alex Loikob5c9a792018-04-16 16:31:22 +0200148 bool pre_amplifier_enabled,
Gustaf Ullbergce045ac2017-10-16 13:49:04 +0200149 bool echo_controller_enabled,
Sam Zackrisson0824c6f2019-10-07 14:03:56 +0200150 bool voice_detector_enabled,
peah2ace3f92016-09-10 04:42:27 -0700151 bool transient_suppressor_enabled) {
152 bool changed = false;
Sam Zackrissoncb1b5562018-09-28 14:15:09 +0200153 changed |= (high_pass_filter_enabled != high_pass_filter_enabled_);
peah2ace3f92016-09-10 04:42:27 -0700154 changed |=
155 (mobile_echo_controller_enabled != mobile_echo_controller_enabled_);
ivoc9f4a4a02016-10-28 05:39:16 -0700156 changed |=
157 (residual_echo_detector_enabled != residual_echo_detector_enabled_);
peah2ace3f92016-09-10 04:42:27 -0700158 changed |= (noise_suppressor_enabled != noise_suppressor_enabled_);
159 changed |=
peah2ace3f92016-09-10 04:42:27 -0700160 (adaptive_gain_controller_enabled != adaptive_gain_controller_enabled_);
Per Åhgren6ee75fd2019-04-26 11:33:37 +0200161 changed |= (gain_controller2_enabled != gain_controller2_enabled_);
Alex Loikob5c9a792018-04-16 16:31:22 +0200162 changed |= (pre_amplifier_enabled_ != pre_amplifier_enabled);
Gustaf Ullbergce045ac2017-10-16 13:49:04 +0200163 changed |= (echo_controller_enabled != echo_controller_enabled_);
Sam Zackrisson0824c6f2019-10-07 14:03:56 +0200164 changed |= (voice_detector_enabled != voice_detector_enabled_);
peah2ace3f92016-09-10 04:42:27 -0700165 changed |= (transient_suppressor_enabled != transient_suppressor_enabled_);
166 if (changed) {
Sam Zackrissoncb1b5562018-09-28 14:15:09 +0200167 high_pass_filter_enabled_ = high_pass_filter_enabled;
peah2ace3f92016-09-10 04:42:27 -0700168 mobile_echo_controller_enabled_ = mobile_echo_controller_enabled;
ivoc9f4a4a02016-10-28 05:39:16 -0700169 residual_echo_detector_enabled_ = residual_echo_detector_enabled;
peah2ace3f92016-09-10 04:42:27 -0700170 noise_suppressor_enabled_ = noise_suppressor_enabled;
peah2ace3f92016-09-10 04:42:27 -0700171 adaptive_gain_controller_enabled_ = adaptive_gain_controller_enabled;
alessiob3ec96df2017-05-22 06:57:06 -0700172 gain_controller2_enabled_ = gain_controller2_enabled;
Alex Loikob5c9a792018-04-16 16:31:22 +0200173 pre_amplifier_enabled_ = pre_amplifier_enabled;
Gustaf Ullbergce045ac2017-10-16 13:49:04 +0200174 echo_controller_enabled_ = echo_controller_enabled;
Sam Zackrisson0824c6f2019-10-07 14:03:56 +0200175 voice_detector_enabled_ = voice_detector_enabled;
peah2ace3f92016-09-10 04:42:27 -0700176 transient_suppressor_enabled_ = transient_suppressor_enabled;
177 }
178
179 changed |= first_update_;
180 first_update_ = false;
181 return changed;
182}
183
saza1d600522019-10-18 13:29:43 +0200184bool AudioProcessingImpl::SubmoduleStates::CaptureMultiBandSubModulesActive()
peah2ace3f92016-09-10 04:42:27 -0700185 const {
Gustaf Ullberg8675eee2019-10-09 13:34:36 +0200186 return CaptureMultiBandProcessingPresent() || voice_detector_enabled_;
peah2ace3f92016-09-10 04:42:27 -0700187}
188
saza1d600522019-10-18 13:29:43 +0200189bool AudioProcessingImpl::SubmoduleStates::CaptureMultiBandProcessingPresent()
190 const {
Gustaf Ullberg8675eee2019-10-09 13:34:36 +0200191 // If echo controller is present, assume it performs active processing.
192 return CaptureMultiBandProcessingActive(/*ec_processing_active=*/true);
193}
194
saza1d600522019-10-18 13:29:43 +0200195bool AudioProcessingImpl::SubmoduleStates::CaptureMultiBandProcessingActive(
Gustaf Ullberg8675eee2019-10-09 13:34:36 +0200196 bool ec_processing_active) const {
Per Åhgren62ea0aa2019-12-09 10:18:44 +0100197 return high_pass_filter_enabled_ || mobile_echo_controller_enabled_ ||
198 noise_suppressor_enabled_ || adaptive_gain_controller_enabled_ ||
Gustaf Ullberg8675eee2019-10-09 13:34:36 +0200199 (echo_controller_enabled_ && ec_processing_active);
peah2ace3f92016-09-10 04:42:27 -0700200}
201
saza1d600522019-10-18 13:29:43 +0200202bool AudioProcessingImpl::SubmoduleStates::CaptureFullBandProcessingActive()
peah23ac8b42017-05-23 05:33:56 -0700203 const {
Alex Loikob5c9a792018-04-16 16:31:22 +0200204 return gain_controller2_enabled_ || capture_post_processor_enabled_ ||
205 pre_amplifier_enabled_;
peah23ac8b42017-05-23 05:33:56 -0700206}
207
saza1d600522019-10-18 13:29:43 +0200208bool AudioProcessingImpl::SubmoduleStates::CaptureAnalyzerActive() const {
Valeriia Nemychnikovaf06eb572018-08-29 10:37:09 +0200209 return capture_analyzer_enabled_;
210}
211
saza1d600522019-10-18 13:29:43 +0200212bool AudioProcessingImpl::SubmoduleStates::RenderMultiBandSubModulesActive()
peah2ace3f92016-09-10 04:42:27 -0700213 const {
Per Åhgren62ea0aa2019-12-09 10:18:44 +0100214 return RenderMultiBandProcessingActive() || mobile_echo_controller_enabled_ ||
215 adaptive_gain_controller_enabled_ || echo_controller_enabled_;
peah2ace3f92016-09-10 04:42:27 -0700216}
217
saza1d600522019-10-18 13:29:43 +0200218bool AudioProcessingImpl::SubmoduleStates::RenderFullBandProcessingActive()
Alex Loiko5825aa62017-12-18 16:02:40 +0100219 const {
220 return render_pre_processor_enabled_;
221}
222
saza1d600522019-10-18 13:29:43 +0200223bool AudioProcessingImpl::SubmoduleStates::RenderMultiBandProcessingActive()
peah2ace3f92016-09-10 04:42:27 -0700224 const {
peah2ace3f92016-09-10 04:42:27 -0700225 return false;
peah2ace3f92016-09-10 04:42:27 -0700226}
227
saza1d600522019-10-18 13:29:43 +0200228bool AudioProcessingImpl::SubmoduleStates::HighPassFilteringRequired() const {
Per Åhgren62ea0aa2019-12-09 10:18:44 +0100229 return high_pass_filter_enabled_ || mobile_echo_controller_enabled_ ||
230 noise_suppressor_enabled_;
Sam Zackrissoncb1b5562018-09-28 14:15:09 +0200231}
232
peah88ac8532016-09-12 16:47:25 -0700233AudioProcessingImpl::AudioProcessingImpl(const webrtc::Config& config)
Sam Zackrissonfeee1e42019-09-20 07:50:35 +0200234 : AudioProcessingImpl(config,
235 /*capture_post_processor=*/nullptr,
236 /*render_pre_processor=*/nullptr,
237 /*echo_control_factory=*/nullptr,
238 /*echo_detector=*/nullptr,
239 /*capture_analyzer=*/nullptr) {}
aluebs@webrtc.orgd82f55d2015-01-15 18:07:21 +0000240
Per Åhgren13735822018-02-12 21:42:56 +0100241int AudioProcessingImpl::instance_count_ = 0;
242
Sam Zackrisson0beac582017-09-25 12:04:02 +0200243AudioProcessingImpl::AudioProcessingImpl(
244 const webrtc::Config& config,
Alex Loiko5825aa62017-12-18 16:02:40 +0100245 std::unique_ptr<CustomProcessing> capture_post_processor,
246 std::unique_ptr<CustomProcessing> render_pre_processor,
Gustaf Ullberg002ef282017-10-12 15:13:17 +0200247 std::unique_ptr<EchoControlFactory> echo_control_factory,
Valeriia Nemychnikovaf06eb572018-08-29 10:37:09 +0200248 rtc::scoped_refptr<EchoDetector> echo_detector,
249 std::unique_ptr<CustomAudioAnalyzer> capture_analyzer)
Per Åhgren13735822018-02-12 21:42:56 +0100250 : data_dumper_(
251 new ApmDataDumper(rtc::AtomicOps::Increment(&instance_count_))),
Per Åhgrenb2b58d82019-12-02 14:59:40 +0100252 use_setup_specific_default_aec3_config_(
253 UseSetupSpecificDefaultAec3Congfig()),
Alex Loiko73ec0192018-05-15 10:52:28 +0200254 capture_runtime_settings_(kRuntimeSettingQueueSize),
255 render_runtime_settings_(kRuntimeSettingQueueSize),
256 capture_runtime_settings_enqueuer_(&capture_runtime_settings_),
257 render_runtime_settings_enqueuer_(&render_runtime_settings_),
Gustaf Ullberg002ef282017-10-12 15:13:17 +0200258 echo_control_factory_(std::move(echo_control_factory)),
Valeriia Nemychnikovaf06eb572018-08-29 10:37:09 +0200259 submodule_states_(!!capture_post_processor,
260 !!render_pre_processor,
261 !!capture_analyzer),
saza1d600522019-10-18 13:29:43 +0200262 submodules_(std::move(capture_post_processor),
263 std::move(render_pre_processor),
264 std::move(echo_detector),
Per Åhgren3daedb62019-11-22 12:11:40 +0100265 std::move(capture_analyzer)),
Per Åhgren0695df12020-01-13 14:43:13 +0100266 constants_(!field_trial::IsEnabled(
Sam Zackrissonfeee1e42019-09-20 07:50:35 +0200267 "WebRTC-ApmExperimentalMultiChannelRenderKillSwitch"),
268 !field_trial::IsEnabled(
Per Åhgrenc0424252019-12-10 13:04:15 +0100269 "WebRTC-ApmExperimentalMultiChannelCaptureKillSwitch"),
270 EnforceSplitBandHpf()),
Alessio Bazzicacc22f512018-08-30 13:01:34 +0200271 capture_nonlocked_() {
Sam Zackrisson72cc71c2019-10-21 12:54:02 +0200272 RTC_LOG(LS_INFO) << "Injected APM submodules:"
Jonas Olssonb2b20312020-01-14 12:11:31 +0100273 "\nEcho control factory: "
274 << !!echo_control_factory_
Sam Zackrisson72cc71c2019-10-21 12:54:02 +0200275 << "\nEcho detector: " << !!submodules_.echo_detector
276 << "\nCapture analyzer: " << !!submodules_.capture_analyzer
277 << "\nCapture post processor: "
278 << !!submodules_.capture_post_processor
279 << "\nRender pre processor: "
280 << !!submodules_.render_pre_processor;
281
Sam Zackrisson421c8592019-02-11 13:39:46 +0100282 // Mark Echo Controller enabled if a factory is injected.
283 capture_nonlocked_.echo_controller_enabled =
284 static_cast<bool>(echo_control_factory_);
niklase@google.com470e71d2011-07-07 08:21:25 +0000285
Sam Zackrisson421c8592019-02-11 13:39:46 +0100286 // If no echo detector is injected, use the ResidualEchoDetector.
saza1d600522019-10-18 13:29:43 +0200287 if (!submodules_.echo_detector) {
288 submodules_.echo_detector =
Sam Zackrisson421c8592019-02-11 13:39:46 +0100289 new rtc::RefCountedObject<ResidualEchoDetector>();
peahdf3efa82015-11-28 12:35:15 -0800290 }
pbos@webrtc.org788acd12014-12-15 09:41:24 +0000291
Per Åhgren0695df12020-01-13 14:43:13 +0100292#if !(defined(WEBRTC_ANDROID) || defined(WEBRTC_IOS))
Per Åhgrenc0734712020-01-02 15:15:36 +0100293 // TODO(webrtc:5298): Remove once the use of ExperimentalNs has been
294 // deprecated.
Per Åhgrenc0734712020-01-02 15:15:36 +0100295 config_.transient_suppression.enabled = config.Get<ExperimentalNs>().enabled;
Per Åhgren0695df12020-01-13 14:43:13 +0100296
297 // TODO(webrtc:5298): Remove once the use of ExperimentalAgc has been
298 // deprecated.
299 config_.gain_controller1.analog_gain_controller.enabled =
300 config.Get<ExperimentalAgc>().enabled;
301 config_.gain_controller1.analog_gain_controller.startup_min_volume =
302 config.Get<ExperimentalAgc>().startup_min_volume;
303 config_.gain_controller1.analog_gain_controller.clipped_level_min =
304 config.Get<ExperimentalAgc>().clipped_level_min;
305 config_.gain_controller1.analog_gain_controller.enable_agc2_level_estimator =
306 config.Get<ExperimentalAgc>().enabled_agc2_level_estimator;
307 config_.gain_controller1.analog_gain_controller.enable_digital_adaptive =
308 !config.Get<ExperimentalAgc>().digital_adaptive_disabled;
Per Åhgrenc0734712020-01-02 15:15:36 +0100309#endif
Per Åhgren0ade9832020-09-01 23:57:20 +0200310
311 Initialize();
niklase@google.com470e71d2011-07-07 08:21:25 +0000312}
313
Per Åhgren0e3198e2019-11-18 08:52:22 +0100314AudioProcessingImpl::~AudioProcessingImpl() = default;
niklase@google.com470e71d2011-07-07 08:21:25 +0000315
niklase@google.com470e71d2011-07-07 08:21:25 +0000316int AudioProcessingImpl::Initialize() {
peahdf3efa82015-11-28 12:35:15 -0800317 // Run in a single-threaded manner during initialization.
Markus Handell0df0fae2020-07-07 15:53:34 +0200318 MutexLock lock_render(&mutex_render_);
319 MutexLock lock_capture(&mutex_capture_);
Per Åhgren0ade9832020-09-01 23:57:20 +0200320 InitializeLocked();
321 return kNoError;
niklase@google.com470e71d2011-07-07 08:21:25 +0000322}
323
peahde65ddc2016-09-16 15:02:15 -0700324int AudioProcessingImpl::Initialize(int capture_input_sample_rate_hz,
325 int capture_output_sample_rate_hz,
326 int render_input_sample_rate_hz,
327 ChannelLayout capture_input_layout,
328 ChannelLayout capture_output_layout,
329 ChannelLayout render_input_layout) {
Michael Graczyk86c6d332015-07-23 11:41:39 -0700330 const ProcessingConfig processing_config = {
peahde65ddc2016-09-16 15:02:15 -0700331 {{capture_input_sample_rate_hz, ChannelsFromLayout(capture_input_layout),
332 LayoutHasKeyboard(capture_input_layout)},
333 {capture_output_sample_rate_hz,
334 ChannelsFromLayout(capture_output_layout),
335 LayoutHasKeyboard(capture_output_layout)},
336 {render_input_sample_rate_hz, ChannelsFromLayout(render_input_layout),
337 LayoutHasKeyboard(render_input_layout)},
338 {render_input_sample_rate_hz, ChannelsFromLayout(render_input_layout),
339 LayoutHasKeyboard(render_input_layout)}}};
Michael Graczyk86c6d332015-07-23 11:41:39 -0700340
341 return Initialize(processing_config);
342}
343
344int AudioProcessingImpl::Initialize(const ProcessingConfig& processing_config) {
peahdf3efa82015-11-28 12:35:15 -0800345 // Run in a single-threaded manner during initialization.
Markus Handell0df0fae2020-07-07 15:53:34 +0200346 MutexLock lock_render(&mutex_render_);
347 MutexLock lock_capture(&mutex_capture_);
Michael Graczyk86c6d332015-07-23 11:41:39 -0700348 return InitializeLocked(processing_config);
andrew@webrtc.orga8b97372014-03-10 22:26:12 +0000349}
350
peahdf3efa82015-11-28 12:35:15 -0800351int AudioProcessingImpl::MaybeInitializeRender(
peah81b9bfe2015-11-27 02:47:28 -0800352 const ProcessingConfig& processing_config) {
peahdf3efa82015-11-28 12:35:15 -0800353 // Called from both threads. Thread check is therefore not possible.
Oskar Sundbom4b276482019-05-23 14:28:00 +0200354 if (processing_config == formats_.api_format) {
peah192164e2015-11-17 02:16:45 -0800355 return kNoError;
356 }
peahdf3efa82015-11-28 12:35:15 -0800357
Markus Handell0df0fae2020-07-07 15:53:34 +0200358 MutexLock lock_capture(&mutex_capture_);
peah192164e2015-11-17 02:16:45 -0800359 return InitializeLocked(processing_config);
360}
361
Per Åhgren0ade9832020-09-01 23:57:20 +0200362void AudioProcessingImpl::InitializeLocked() {
Per Åhgren4bdced52017-06-27 16:00:38 +0200363 UpdateActiveSubmoduleStates();
364
Per Åhgrend47941e2019-08-22 11:51:13 +0200365 const int render_audiobuffer_sample_rate_hz =
peahdf3efa82015-11-28 12:35:15 -0800366 formats_.api_format.reverse_output_stream().num_frames() == 0
Per Åhgrend47941e2019-08-22 11:51:13 +0200367 ? formats_.render_processing_format.sample_rate_hz()
368 : formats_.api_format.reverse_output_stream().sample_rate_hz();
peahdf3efa82015-11-28 12:35:15 -0800369 if (formats_.api_format.reverse_input_stream().num_channels() > 0) {
370 render_.render_audio.reset(new AudioBuffer(
Per Åhgrend47941e2019-08-22 11:51:13 +0200371 formats_.api_format.reverse_input_stream().sample_rate_hz(),
peahdf3efa82015-11-28 12:35:15 -0800372 formats_.api_format.reverse_input_stream().num_channels(),
Per Åhgrend47941e2019-08-22 11:51:13 +0200373 formats_.render_processing_format.sample_rate_hz(),
peahde65ddc2016-09-16 15:02:15 -0700374 formats_.render_processing_format.num_channels(),
Per Åhgrend47941e2019-08-22 11:51:13 +0200375 render_audiobuffer_sample_rate_hz,
376 formats_.render_processing_format.num_channels()));
peah2ace3f92016-09-10 04:42:27 -0700377 if (formats_.api_format.reverse_input_stream() !=
378 formats_.api_format.reverse_output_stream()) {
kwibergc2b785d2016-02-24 05:22:32 -0800379 render_.render_converter = AudioConverter::Create(
peahdf3efa82015-11-28 12:35:15 -0800380 formats_.api_format.reverse_input_stream().num_channels(),
381 formats_.api_format.reverse_input_stream().num_frames(),
382 formats_.api_format.reverse_output_stream().num_channels(),
kwibergc2b785d2016-02-24 05:22:32 -0800383 formats_.api_format.reverse_output_stream().num_frames());
ekmeyerson60d9b332015-08-14 10:35:55 -0700384 } else {
peahdf3efa82015-11-28 12:35:15 -0800385 render_.render_converter.reset(nullptr);
ekmeyerson60d9b332015-08-14 10:35:55 -0700386 }
Michael Graczyk86c6d332015-07-23 11:41:39 -0700387 } else {
peahdf3efa82015-11-28 12:35:15 -0800388 render_.render_audio.reset(nullptr);
389 render_.render_converter.reset(nullptr);
Michael Graczyk86c6d332015-07-23 11:41:39 -0700390 }
peahce4d9152017-05-19 01:28:05 -0700391
Per Åhgrend47941e2019-08-22 11:51:13 +0200392 capture_.capture_audio.reset(new AudioBuffer(
393 formats_.api_format.input_stream().sample_rate_hz(),
394 formats_.api_format.input_stream().num_channels(),
395 capture_nonlocked_.capture_processing_format.sample_rate_hz(),
396 formats_.api_format.output_stream().num_channels(),
397 formats_.api_format.output_stream().sample_rate_hz(),
398 formats_.api_format.output_stream().num_channels()));
niklase@google.com470e71d2011-07-07 08:21:25 +0000399
Gustaf Ullberg422b9e02019-10-09 13:02:14 +0200400 if (capture_nonlocked_.capture_processing_format.sample_rate_hz() <
401 formats_.api_format.output_stream().sample_rate_hz() &&
402 formats_.api_format.output_stream().sample_rate_hz() == 48000) {
403 capture_.capture_fullband_audio.reset(
404 new AudioBuffer(formats_.api_format.input_stream().sample_rate_hz(),
405 formats_.api_format.input_stream().num_channels(),
406 formats_.api_format.output_stream().sample_rate_hz(),
407 formats_.api_format.output_stream().num_channels(),
408 formats_.api_format.output_stream().sample_rate_hz(),
409 formats_.api_format.output_stream().num_channels()));
410 } else {
411 capture_.capture_fullband_audio.reset();
412 }
413
peah764e3642016-10-22 05:04:30 -0700414 AllocateRenderQueue();
415
Per Åhgren0695df12020-01-13 14:43:13 +0100416 InitializeGainController1();
Per Åhgrenc0734712020-01-02 15:15:36 +0100417 InitializeTransientSuppressor();
Per Åhgren0f14db22020-01-03 14:27:14 +0100418 InitializeHighPassFilter(true);
Sam Zackrisson0824c6f2019-10-07 14:03:56 +0200419 InitializeVoiceDetector();
ivoc9f4a4a02016-10-28 05:39:16 -0700420 InitializeResidualEchoDetector();
Gustaf Ullberg8eb9c7d2017-10-14 08:28:46 +0200421 InitializeEchoController();
alessiob3ec96df2017-05-22 06:57:06 -0700422 InitializeGainController2();
saza0bad15f2019-10-16 11:46:11 +0200423 InitializeNoiseSuppressor();
Valeriia Nemychnikovaf06eb572018-08-29 10:37:09 +0200424 InitializeAnalyzer();
Sam Zackrisson0beac582017-09-25 12:04:02 +0200425 InitializePostProcessor();
Alex Loiko5825aa62017-12-18 16:02:40 +0100426 InitializePreProcessor();
solenberg70f99032015-12-08 11:07:32 -0800427
aleloi868f32f2017-05-23 07:20:05 -0700428 if (aec_dump_) {
Minyue Li656d6092018-08-10 15:38:52 +0200429 aec_dump_->WriteInitMessage(formats_.api_format, rtc::TimeUTCMillis());
aleloi868f32f2017-05-23 07:20:05 -0700430 }
niklase@google.com470e71d2011-07-07 08:21:25 +0000431}
432
Michael Graczyk86c6d332015-07-23 11:41:39 -0700433int AudioProcessingImpl::InitializeLocked(const ProcessingConfig& config) {
Per Åhgren4bdced52017-06-27 16:00:38 +0200434 UpdateActiveSubmoduleStates();
435
Michael Graczyk86c6d332015-07-23 11:41:39 -0700436 for (const auto& stream : config.streams) {
Michael Graczyk86c6d332015-07-23 11:41:39 -0700437 if (stream.num_channels() > 0 && stream.sample_rate_hz() <= 0) {
438 return kBadSampleRateError;
439 }
andrew@webrtc.orga8b97372014-03-10 22:26:12 +0000440 }
Michael Graczyk86c6d332015-07-23 11:41:39 -0700441
Peter Kasting69558702016-01-12 16:26:35 -0800442 const size_t num_in_channels = config.input_stream().num_channels();
443 const size_t num_out_channels = config.output_stream().num_channels();
Michael Graczyk86c6d332015-07-23 11:41:39 -0700444
445 // Need at least one input channel.
446 // Need either one output channel or as many outputs as there are inputs.
447 if (num_in_channels == 0 ||
448 !(num_out_channels == 1 || num_out_channels == num_in_channels)) {
Michael Graczykc2047542015-07-22 21:06:11 -0700449 return kBadNumberChannelsError;
450 }
451
peahdf3efa82015-11-28 12:35:15 -0800452 formats_.api_format = config;
andrew@webrtc.orgddbb8a22014-04-22 21:00:04 +0000453
Per Åhgrenfcbe4072019-09-15 00:27:58 +0200454 // Choose maximum rate to use for the split filtering.
455 RTC_DCHECK(config_.pipeline.maximum_internal_processing_rate == 48000 ||
456 config_.pipeline.maximum_internal_processing_rate == 32000);
457 int max_splitting_rate = 48000;
458 if (config_.pipeline.maximum_internal_processing_rate == 32000) {
459 max_splitting_rate = config_.pipeline.maximum_internal_processing_rate;
460 }
461
Per Åhgrenc8626b62019-08-23 15:49:51 +0200462 int capture_processing_rate = SuitableProcessRate(
peah423d2362016-04-09 16:06:52 -0700463 std::min(formats_.api_format.input_stream().sample_rate_hz(),
peah2ace3f92016-09-10 04:42:27 -0700464 formats_.api_format.output_stream().sample_rate_hz()),
Per Åhgrenfcbe4072019-09-15 00:27:58 +0200465 max_splitting_rate,
peah2ace3f92016-09-10 04:42:27 -0700466 submodule_states_.CaptureMultiBandSubModulesActive() ||
467 submodule_states_.RenderMultiBandSubModulesActive());
Per Åhgrenc8626b62019-08-23 15:49:51 +0200468 RTC_DCHECK_NE(8000, capture_processing_rate);
andrew@webrtc.orga8b97372014-03-10 22:26:12 +0000469
peahde65ddc2016-09-16 15:02:15 -0700470 capture_nonlocked_.capture_processing_format =
471 StreamConfig(capture_processing_rate);
peah2ace3f92016-09-10 04:42:27 -0700472
peah2ce640f2017-04-07 03:57:48 -0700473 int render_processing_rate;
Gustaf Ullbergbd83b912017-10-18 12:32:42 +0200474 if (!capture_nonlocked_.echo_controller_enabled) {
Per Åhgrenc8626b62019-08-23 15:49:51 +0200475 render_processing_rate = SuitableProcessRate(
peah2ce640f2017-04-07 03:57:48 -0700476 std::min(formats_.api_format.reverse_input_stream().sample_rate_hz(),
477 formats_.api_format.reverse_output_stream().sample_rate_hz()),
Per Åhgrenfcbe4072019-09-15 00:27:58 +0200478 max_splitting_rate,
peah2ce640f2017-04-07 03:57:48 -0700479 submodule_states_.CaptureMultiBandSubModulesActive() ||
480 submodule_states_.RenderMultiBandSubModulesActive());
481 } else {
482 render_processing_rate = capture_processing_rate;
483 }
484
peahde65ddc2016-09-16 15:02:15 -0700485 // If the forward sample rate is 8 kHz, the render stream is also processed
aluebseb3603b2016-04-20 15:27:58 -0700486 // at this rate.
peahde65ddc2016-09-16 15:02:15 -0700487 if (capture_nonlocked_.capture_processing_format.sample_rate_hz() ==
488 kSampleRate8kHz) {
489 render_processing_rate = kSampleRate8kHz;
andrew@webrtc.orga8b97372014-03-10 22:26:12 +0000490 } else {
peahde65ddc2016-09-16 15:02:15 -0700491 render_processing_rate =
492 std::max(render_processing_rate, static_cast<int>(kSampleRate16kHz));
andrew@webrtc.orgddbb8a22014-04-22 21:00:04 +0000493 }
494
Per Åhgrenc8626b62019-08-23 15:49:51 +0200495 RTC_DCHECK_NE(8000, render_processing_rate);
496
peahce4d9152017-05-19 01:28:05 -0700497 if (submodule_states_.RenderMultiBandSubModulesActive()) {
Sam Zackrissonfeee1e42019-09-20 07:50:35 +0200498 // By default, downmix the render stream to mono for analysis. This has been
499 // demonstrated to work well for AEC in most practical scenarios.
Per Åhgrene14cb992019-11-27 09:34:22 +0100500 const bool multi_channel_render = config_.pipeline.multi_channel_render &&
501 constants_.multi_channel_render_support;
Sam Zackrissonfeee1e42019-09-20 07:50:35 +0200502 int render_processing_num_channels =
Per Åhgrene14cb992019-11-27 09:34:22 +0100503 multi_channel_render
Sam Zackrissonfeee1e42019-09-20 07:50:35 +0200504 ? formats_.api_format.reverse_input_stream().num_channels()
505 : 1;
506 formats_.render_processing_format =
507 StreamConfig(render_processing_rate, render_processing_num_channels);
peahce4d9152017-05-19 01:28:05 -0700508 } else {
509 formats_.render_processing_format = StreamConfig(
510 formats_.api_format.reverse_input_stream().sample_rate_hz(),
511 formats_.api_format.reverse_input_stream().num_channels());
512 }
andrew@webrtc.orgddbb8a22014-04-22 21:00:04 +0000513
peahde65ddc2016-09-16 15:02:15 -0700514 if (capture_nonlocked_.capture_processing_format.sample_rate_hz() ==
515 kSampleRate32kHz ||
516 capture_nonlocked_.capture_processing_format.sample_rate_hz() ==
517 kSampleRate48kHz) {
peahdf3efa82015-11-28 12:35:15 -0800518 capture_nonlocked_.split_rate = kSampleRate16kHz;
andrew@webrtc.orgddbb8a22014-04-22 21:00:04 +0000519 } else {
peahdf3efa82015-11-28 12:35:15 -0800520 capture_nonlocked_.split_rate =
peahde65ddc2016-09-16 15:02:15 -0700521 capture_nonlocked_.capture_processing_format.sample_rate_hz();
andrew@webrtc.orga8b97372014-03-10 22:26:12 +0000522 }
523
Per Åhgren0ade9832020-09-01 23:57:20 +0200524 InitializeLocked();
525 return kNoError;
andrew@webrtc.orga8b97372014-03-10 22:26:12 +0000526}
527
peah88ac8532016-09-12 16:47:25 -0700528void AudioProcessingImpl::ApplyConfig(const AudioProcessing::Config& config) {
Sam Zackrisson72cc71c2019-10-21 12:54:02 +0200529 RTC_LOG(LS_INFO) << "AudioProcessing::ApplyConfig: " << config.ToString();
530
peah88ac8532016-09-12 16:47:25 -0700531 // Run in a single-threaded manner when applying the settings.
Markus Handell0df0fae2020-07-07 15:53:34 +0200532 MutexLock lock_render(&mutex_render_);
533 MutexLock lock_capture(&mutex_capture_);
peah88ac8532016-09-12 16:47:25 -0700534
Sam Zackrissonfeee1e42019-09-20 07:50:35 +0200535 const bool pipeline_config_changed =
Per Åhgrene14cb992019-11-27 09:34:22 +0100536 config_.pipeline.multi_channel_render !=
537 config.pipeline.multi_channel_render ||
538 config_.pipeline.multi_channel_capture !=
Per Åhgrenc0424252019-12-10 13:04:15 +0100539 config.pipeline.multi_channel_capture ||
540 config_.pipeline.maximum_internal_processing_rate !=
541 config.pipeline.maximum_internal_processing_rate;
Sam Zackrissonfeee1e42019-09-20 07:50:35 +0200542
Per Åhgren200feba2019-03-06 04:16:46 +0100543 const bool aec_config_changed =
544 config_.echo_canceller.enabled != config.echo_canceller.enabled ||
Per Åhgren62ea0aa2019-12-09 10:18:44 +0100545 config_.echo_canceller.mobile_mode != config.echo_canceller.mobile_mode;
Per Åhgren200feba2019-03-06 04:16:46 +0100546
Sam Zackrissonf0d1c032019-03-27 13:28:08 +0100547 const bool agc1_config_changed =
548 config_.gain_controller1.enabled != config.gain_controller1.enabled ||
549 config_.gain_controller1.mode != config.gain_controller1.mode ||
550 config_.gain_controller1.target_level_dbfs !=
551 config.gain_controller1.target_level_dbfs ||
552 config_.gain_controller1.compression_gain_db !=
553 config.gain_controller1.compression_gain_db ||
554 config_.gain_controller1.enable_limiter !=
555 config.gain_controller1.enable_limiter ||
556 config_.gain_controller1.analog_level_minimum !=
557 config.gain_controller1.analog_level_minimum ||
558 config_.gain_controller1.analog_level_maximum !=
Per Åhgren0695df12020-01-13 14:43:13 +0100559 config.gain_controller1.analog_level_maximum ||
560 config_.gain_controller1.analog_gain_controller.enabled !=
561 config.gain_controller1.analog_gain_controller.enabled ||
562 config_.gain_controller1.analog_gain_controller.startup_min_volume !=
563 config.gain_controller1.analog_gain_controller.startup_min_volume ||
564 config_.gain_controller1.analog_gain_controller.clipped_level_min !=
565 config.gain_controller1.analog_gain_controller.clipped_level_min ||
566 config_.gain_controller1.analog_gain_controller
567 .enable_agc2_level_estimator !=
568 config.gain_controller1.analog_gain_controller
569 .enable_agc2_level_estimator ||
570 config_.gain_controller1.analog_gain_controller.enable_digital_adaptive !=
571 config.gain_controller1.analog_gain_controller
572 .enable_digital_adaptive;
Sam Zackrissonf0d1c032019-03-27 13:28:08 +0100573
Per Åhgren2bd85ab2020-01-03 10:36:34 +0100574 const bool agc2_config_changed =
575 config_.gain_controller2.enabled != config.gain_controller2.enabled;
576
Sam Zackrisson0824c6f2019-10-07 14:03:56 +0200577 const bool voice_detection_config_changed =
578 config_.voice_detection.enabled != config.voice_detection.enabled;
579
saza0bad15f2019-10-16 11:46:11 +0200580 const bool ns_config_changed =
581 config_.noise_suppression.enabled != config.noise_suppression.enabled ||
582 config_.noise_suppression.level != config.noise_suppression.level;
583
Per Åhgrenc0734712020-01-02 15:15:36 +0100584 const bool ts_config_changed = config_.transient_suppression.enabled !=
585 config.transient_suppression.enabled;
586
Per Åhgren0f14db22020-01-03 14:27:14 +0100587 const bool pre_amplifier_config_changed =
588 config_.pre_amplifier.enabled != config.pre_amplifier.enabled ||
589 config_.pre_amplifier.fixed_gain_factor !=
590 config.pre_amplifier.fixed_gain_factor;
591
Yves Gerey499bc6c2018-10-10 18:29:07 +0200592 config_ = config;
593
Per Åhgren200feba2019-03-06 04:16:46 +0100594 if (aec_config_changed) {
595 InitializeEchoController();
596 }
Sam Zackrissoncdf0e6d2018-09-17 11:05:17 +0200597
saza0bad15f2019-10-16 11:46:11 +0200598 if (ns_config_changed) {
599 InitializeNoiseSuppressor();
600 }
Sam Zackrisson23513132019-01-11 15:10:32 +0100601
Per Åhgrenc0734712020-01-02 15:15:36 +0100602 if (ts_config_changed) {
603 InitializeTransientSuppressor();
604 }
605
Per Åhgren0f14db22020-01-03 14:27:14 +0100606 InitializeHighPassFilter(false);
peah8271d042016-11-22 07:24:52 -0800607
Sam Zackrissonf0d1c032019-03-27 13:28:08 +0100608 if (agc1_config_changed) {
Per Åhgren0695df12020-01-13 14:43:13 +0100609 InitializeGainController1();
Sam Zackrissonf0d1c032019-03-27 13:28:08 +0100610 }
611
Sam Zackrissonab1aee02018-03-05 15:59:06 +0100612 const bool config_ok = GainController2::Validate(config_.gain_controller2);
alessiob3ec96df2017-05-22 06:57:06 -0700613 if (!config_ok) {
Alessio Bazzica0c83e152020-10-14 12:49:54 +0200614 RTC_LOG(LS_ERROR)
615 << "Invalid Gain Controller 2 config; using the default config.";
alessiob3ec96df2017-05-22 06:57:06 -0700616 config_.gain_controller2 = AudioProcessing::Config::GainController2();
617 }
Per Åhgren0f14db22020-01-03 14:27:14 +0100618
Per Åhgren2bd85ab2020-01-03 10:36:34 +0100619 if (agc2_config_changed) {
620 InitializeGainController2();
621 }
Per Åhgren0f14db22020-01-03 14:27:14 +0100622
623 if (pre_amplifier_config_changed) {
624 InitializePreAmplifier();
625 }
Sam Zackrissonb24c00f2018-11-26 16:18:25 +0100626
saza1d600522019-10-18 13:29:43 +0200627 if (config_.level_estimation.enabled && !submodules_.output_level_estimator) {
628 submodules_.output_level_estimator = std::make_unique<LevelEstimator>();
Sam Zackrissonb24c00f2018-11-26 16:18:25 +0100629 }
Sam Zackrisson4db667b2018-12-21 16:29:27 +0100630
Sam Zackrisson0824c6f2019-10-07 14:03:56 +0200631 if (voice_detection_config_changed) {
632 InitializeVoiceDetector();
Sam Zackrisson4db667b2018-12-21 16:29:27 +0100633 }
Sam Zackrissonfeee1e42019-09-20 07:50:35 +0200634
635 // Reinitialization must happen after all submodule configuration to avoid
636 // additional reinitializations on the next capture / render processing call.
637 if (pipeline_config_changed) {
638 InitializeLocked(formats_.api_format);
639 }
peah88ac8532016-09-12 16:47:25 -0700640}
641
Sam Zackrissonb37e59d2020-04-27 08:39:33 +0200642void AudioProcessingImpl::OverrideSubmoduleCreationForTesting(
643 const ApmSubmoduleCreationOverrides& overrides) {
Markus Handell0df0fae2020-07-07 15:53:34 +0200644 MutexLock lock(&mutex_capture_);
Sam Zackrissonb37e59d2020-04-27 08:39:33 +0200645 submodule_creation_overrides_ = overrides;
646}
647
andrew@webrtc.orgddbb8a22014-04-22 21:00:04 +0000648int AudioProcessingImpl::proc_sample_rate_hz() const {
peahdf3efa82015-11-28 12:35:15 -0800649 // Used as callback from submodules, hence locking is not allowed.
peahde65ddc2016-09-16 15:02:15 -0700650 return capture_nonlocked_.capture_processing_format.sample_rate_hz();
niklase@google.com470e71d2011-07-07 08:21:25 +0000651}
652
Gustaf Ullberg422b9e02019-10-09 13:02:14 +0200653int AudioProcessingImpl::proc_fullband_sample_rate_hz() const {
654 return capture_.capture_fullband_audio
655 ? capture_.capture_fullband_audio->num_frames() * 100
656 : capture_nonlocked_.capture_processing_format.sample_rate_hz();
657}
658
andrew@webrtc.orgddbb8a22014-04-22 21:00:04 +0000659int AudioProcessingImpl::proc_split_sample_rate_hz() const {
peahdf3efa82015-11-28 12:35:15 -0800660 // Used as callback from submodules, hence locking is not allowed.
661 return capture_nonlocked_.split_rate;
niklase@google.com470e71d2011-07-07 08:21:25 +0000662}
663
Peter Kasting69558702016-01-12 16:26:35 -0800664size_t AudioProcessingImpl::num_reverse_channels() const {
peahdf3efa82015-11-28 12:35:15 -0800665 // Used as callback from submodules, hence locking is not allowed.
peahde65ddc2016-09-16 15:02:15 -0700666 return formats_.render_processing_format.num_channels();
niklase@google.com470e71d2011-07-07 08:21:25 +0000667}
668
Peter Kasting69558702016-01-12 16:26:35 -0800669size_t AudioProcessingImpl::num_input_channels() const {
peahdf3efa82015-11-28 12:35:15 -0800670 // Used as callback from submodules, hence locking is not allowed.
671 return formats_.api_format.input_stream().num_channels();
niklase@google.com470e71d2011-07-07 08:21:25 +0000672}
673
Peter Kasting69558702016-01-12 16:26:35 -0800674size_t AudioProcessingImpl::num_proc_channels() const {
aluebsb2328d12016-01-11 20:32:29 -0800675 // Used as callback from submodules, hence locking is not allowed.
Per Åhgrene14cb992019-11-27 09:34:22 +0100676 const bool multi_channel_capture = config_.pipeline.multi_channel_capture &&
677 constants_.multi_channel_capture_support;
678 if (capture_nonlocked_.echo_controller_enabled && !multi_channel_capture) {
Sam Zackrissonfeee1e42019-09-20 07:50:35 +0200679 return 1;
680 }
681 return num_output_channels();
aluebsb2328d12016-01-11 20:32:29 -0800682}
683
Peter Kasting69558702016-01-12 16:26:35 -0800684size_t AudioProcessingImpl::num_output_channels() const {
peahdf3efa82015-11-28 12:35:15 -0800685 // Used as callback from submodules, hence locking is not allowed.
686 return formats_.api_format.output_stream().num_channels();
niklase@google.com470e71d2011-07-07 08:21:25 +0000687}
688
andrew@webrtc.org17342e52014-02-12 22:28:31 +0000689void AudioProcessingImpl::set_output_will_be_muted(bool muted) {
Markus Handell0df0fae2020-07-07 15:53:34 +0200690 MutexLock lock(&mutex_capture_);
peahdf3efa82015-11-28 12:35:15 -0800691 capture_.output_will_be_muted = muted;
saza1d600522019-10-18 13:29:43 +0200692 if (submodules_.agc_manager.get()) {
693 submodules_.agc_manager->SetCaptureMuted(capture_.output_will_be_muted);
pbos@webrtc.org788acd12014-12-15 09:41:24 +0000694 }
andrew@webrtc.org17342e52014-02-12 22:28:31 +0000695}
696
Alessio Bazzicac054e782018-04-16 12:10:09 +0200697void AudioProcessingImpl::SetRuntimeSetting(RuntimeSetting setting) {
Alex Loiko73ec0192018-05-15 10:52:28 +0200698 switch (setting.type()) {
699 case RuntimeSetting::Type::kCustomRenderProcessingRuntimeSetting:
Alessio Bazzica7c19a702019-11-07 13:22:00 +0100700 case RuntimeSetting::Type::kPlayoutAudioDeviceChange:
Alex Loiko73ec0192018-05-15 10:52:28 +0200701 render_runtime_settings_enqueuer_.Enqueue(setting);
702 return;
Alex Loiko73ec0192018-05-15 10:52:28 +0200703 case RuntimeSetting::Type::kCapturePreGain:
Sam Zackrissonf0d1c032019-03-27 13:28:08 +0100704 case RuntimeSetting::Type::kCaptureCompressionGain:
Per Åhgren6ee75fd2019-04-26 11:33:37 +0200705 case RuntimeSetting::Type::kCaptureFixedPostGain:
Per Åhgren552d3e32020-08-12 08:46:47 +0200706 case RuntimeSetting::Type::kCaptureOutputUsed:
Alessio Bazzica7c19a702019-11-07 13:22:00 +0100707 capture_runtime_settings_enqueuer_.Enqueue(setting);
708 return;
Fredrik Hernqvistca362852019-05-10 15:50:02 +0200709 case RuntimeSetting::Type::kPlayoutVolumeChange:
Alex Loiko73ec0192018-05-15 10:52:28 +0200710 capture_runtime_settings_enqueuer_.Enqueue(setting);
Alessio Bazzica7c19a702019-11-07 13:22:00 +0100711 render_runtime_settings_enqueuer_.Enqueue(setting);
712 return;
713 case RuntimeSetting::Type::kNotSpecified:
714 RTC_NOTREACHED();
Alex Loiko73ec0192018-05-15 10:52:28 +0200715 return;
716 }
717 // The language allows the enum to have a non-enumerator
718 // value. Check that this doesn't happen.
719 RTC_NOTREACHED();
Alessio Bazzicac054e782018-04-16 12:10:09 +0200720}
721
722AudioProcessingImpl::RuntimeSettingEnqueuer::RuntimeSettingEnqueuer(
723 SwapQueue<RuntimeSetting>* runtime_settings)
Alessio Bazzica33444dc2018-04-20 13:16:55 +0200724 : runtime_settings_(*runtime_settings) {
725 RTC_DCHECK(runtime_settings);
Alessio Bazzicac054e782018-04-16 12:10:09 +0200726}
727
728AudioProcessingImpl::RuntimeSettingEnqueuer::~RuntimeSettingEnqueuer() =
729 default;
730
731void AudioProcessingImpl::RuntimeSettingEnqueuer::Enqueue(
732 RuntimeSetting setting) {
733 size_t remaining_attempts = 10;
Alessio Bazzica33444dc2018-04-20 13:16:55 +0200734 while (!runtime_settings_.Insert(&setting) && remaining_attempts-- > 0) {
Alessio Bazzicac054e782018-04-16 12:10:09 +0200735 RuntimeSetting setting_to_discard;
Alessio Bazzica33444dc2018-04-20 13:16:55 +0200736 if (runtime_settings_.Remove(&setting_to_discard))
Alessio Bazzicac054e782018-04-16 12:10:09 +0200737 RTC_LOG(LS_ERROR)
738 << "The runtime settings queue is full. Oldest setting discarded.";
739 }
740 if (remaining_attempts == 0)
741 RTC_LOG(LS_ERROR) << "Cannot enqueue a new runtime setting.";
742}
andrew@webrtc.org17342e52014-02-12 22:28:31 +0000743
Sam Zackrisson12e319a2020-01-03 14:54:20 +0100744int AudioProcessingImpl::MaybeInitializeCapture(
745 const StreamConfig& input_config,
746 const StreamConfig& output_config) {
peahdf3efa82015-11-28 12:35:15 -0800747 ProcessingConfig processing_config;
peah2ace3f92016-09-10 04:42:27 -0700748 bool reinitialization_required = false;
peahdf3efa82015-11-28 12:35:15 -0800749 {
Sam Zackrisson12e319a2020-01-03 14:54:20 +0100750 // Acquire the capture lock in order to access api_format. The lock is
751 // released immediately, as we may need to acquire the render lock as part
752 // of the conditional reinitialization.
Markus Handell0df0fae2020-07-07 15:53:34 +0200753 MutexLock lock_capture(&mutex_capture_);
peahdf3efa82015-11-28 12:35:15 -0800754 processing_config = formats_.api_format;
peah2ace3f92016-09-10 04:42:27 -0700755 reinitialization_required = UpdateActiveSubmoduleStates();
niklase@google.com470e71d2011-07-07 08:21:25 +0000756 }
andrew@webrtc.org17e40642014-03-04 20:58:13 +0000757
Oskar Sundbom4b276482019-05-23 14:28:00 +0200758 if (processing_config.input_stream() != input_config) {
759 processing_config.input_stream() = input_config;
760 reinitialization_required = true;
peahdf3efa82015-11-28 12:35:15 -0800761 }
Oskar Sundbom4b276482019-05-23 14:28:00 +0200762
763 if (processing_config.output_stream() != output_config) {
764 processing_config.output_stream() = output_config;
765 reinitialization_required = true;
766 }
767
768 if (reinitialization_required) {
Markus Handell0df0fae2020-07-07 15:53:34 +0200769 MutexLock lock_render(&mutex_render_);
770 MutexLock lock_capture(&mutex_capture_);
Oskar Sundbom4b276482019-05-23 14:28:00 +0200771 RETURN_ON_ERR(InitializeLocked(processing_config));
772 }
Sam Zackrisson12e319a2020-01-03 14:54:20 +0100773 return kNoError;
774}
775
776int AudioProcessingImpl::ProcessStream(const float* const* src,
777 const StreamConfig& input_config,
778 const StreamConfig& output_config,
779 float* const* dest) {
780 TRACE_EVENT0("webrtc", "AudioProcessing::ProcessStream_StreamConfig");
781 if (!src || !dest) {
782 return kNullPointerError;
783 }
784
785 RETURN_ON_ERR(MaybeInitializeCapture(input_config, output_config));
Oskar Sundbom4b276482019-05-23 14:28:00 +0200786
Markus Handell0df0fae2020-07-07 15:53:34 +0200787 MutexLock lock_capture(&mutex_capture_);
andrew@webrtc.org17e40642014-03-04 20:58:13 +0000788
aleloi868f32f2017-05-23 07:20:05 -0700789 if (aec_dump_) {
790 RecordUnprocessedCaptureStream(src);
791 }
792
Per Åhgrena1351272019-08-15 12:15:46 +0200793 capture_.keyboard_info.Extract(src, formats_.api_format.input_stream());
peahdf3efa82015-11-28 12:35:15 -0800794 capture_.capture_audio->CopyFrom(src, formats_.api_format.input_stream());
Gustaf Ullberg8675eee2019-10-09 13:34:36 +0200795 if (capture_.capture_fullband_audio) {
796 capture_.capture_fullband_audio->CopyFrom(
797 src, formats_.api_format.input_stream());
798 }
peahde65ddc2016-09-16 15:02:15 -0700799 RETURN_ON_ERR(ProcessCaptureStreamLocked());
Gustaf Ullberg422b9e02019-10-09 13:02:14 +0200800 if (capture_.capture_fullband_audio) {
801 capture_.capture_fullband_audio->CopyTo(formats_.api_format.output_stream(),
802 dest);
803 } else {
804 capture_.capture_audio->CopyTo(formats_.api_format.output_stream(), dest);
805 }
andrew@webrtc.org17e40642014-03-04 20:58:13 +0000806
aleloi868f32f2017-05-23 07:20:05 -0700807 if (aec_dump_) {
808 RecordProcessedCaptureStream(dest);
809 }
andrew@webrtc.org17e40642014-03-04 20:58:13 +0000810 return kNoError;
811}
812
Alex Loiko73ec0192018-05-15 10:52:28 +0200813void AudioProcessingImpl::HandleCaptureRuntimeSettings() {
Alessio Bazzicac054e782018-04-16 12:10:09 +0200814 RuntimeSetting setting;
Alex Loiko73ec0192018-05-15 10:52:28 +0200815 while (capture_runtime_settings_.Remove(&setting)) {
Alex Loiko62347222018-09-10 10:18:07 +0200816 if (aec_dump_) {
817 aec_dump_->WriteRuntimeSetting(setting);
818 }
Alessio Bazzicac054e782018-04-16 12:10:09 +0200819 switch (setting.type()) {
820 case RuntimeSetting::Type::kCapturePreGain:
Alex Loikob5c9a792018-04-16 16:31:22 +0200821 if (config_.pre_amplifier.enabled) {
822 float value;
823 setting.GetFloat(&value);
Sam Zackrisson21bfa402019-10-23 09:43:01 +0200824 config_.pre_amplifier.fixed_gain_factor = value;
saza1d600522019-10-18 13:29:43 +0200825 submodules_.pre_amplifier->SetGainFactor(value);
Alex Loikob5c9a792018-04-16 16:31:22 +0200826 }
827 // TODO(bugs.chromium.org/9138): Log setting handling by Aec Dump.
Alessio Bazzicac054e782018-04-16 12:10:09 +0200828 break;
Sam Zackrissonf0d1c032019-03-27 13:28:08 +0100829 case RuntimeSetting::Type::kCaptureCompressionGain: {
Per Åhgren0e3198e2019-11-18 08:52:22 +0100830 if (!submodules_.agc_manager) {
831 float value;
832 setting.GetFloat(&value);
833 int int_value = static_cast<int>(value + .5f);
834 config_.gain_controller1.compression_gain_db = int_value;
Per Åhgren0695df12020-01-13 14:43:13 +0100835 if (submodules_.gain_control) {
836 int error =
837 submodules_.gain_control->set_compression_gain_db(int_value);
838 RTC_DCHECK_EQ(kNoError, error);
839 }
Per Åhgren0e3198e2019-11-18 08:52:22 +0100840 }
Sam Zackrissonf0d1c032019-03-27 13:28:08 +0100841 break;
842 }
Per Åhgren6ee75fd2019-04-26 11:33:37 +0200843 case RuntimeSetting::Type::kCaptureFixedPostGain: {
Per Åhgren2bd85ab2020-01-03 10:36:34 +0100844 if (submodules_.gain_controller2) {
Per Åhgren6ee75fd2019-04-26 11:33:37 +0200845 float value;
846 setting.GetFloat(&value);
847 config_.gain_controller2.fixed_digital.gain_db = value;
saza1d600522019-10-18 13:29:43 +0200848 submodules_.gain_controller2->ApplyConfig(config_.gain_controller2);
Per Åhgren6ee75fd2019-04-26 11:33:37 +0200849 }
850 break;
851 }
Fredrik Hernqvistca362852019-05-10 15:50:02 +0200852 case RuntimeSetting::Type::kPlayoutVolumeChange: {
853 int value;
854 setting.GetInt(&value);
855 capture_.playout_volume = value;
856 break;
857 }
Alessio Bazzica7c19a702019-11-07 13:22:00 +0100858 case RuntimeSetting::Type::kPlayoutAudioDeviceChange:
859 RTC_NOTREACHED();
860 break;
Alex Loiko73ec0192018-05-15 10:52:28 +0200861 case RuntimeSetting::Type::kCustomRenderProcessingRuntimeSetting:
862 RTC_NOTREACHED();
863 break;
864 case RuntimeSetting::Type::kNotSpecified:
865 RTC_NOTREACHED();
866 break;
Per Åhgren552d3e32020-08-12 08:46:47 +0200867 case RuntimeSetting::Type::kCaptureOutputUsed:
868 // TODO(b/154437967): Add support for reducing complexity when it is
869 // known that the capture output will not be used.
870 break;
Alex Loiko73ec0192018-05-15 10:52:28 +0200871 }
872 }
873}
874
875void AudioProcessingImpl::HandleRenderRuntimeSettings() {
876 RuntimeSetting setting;
877 while (render_runtime_settings_.Remove(&setting)) {
Alex Loiko62347222018-09-10 10:18:07 +0200878 if (aec_dump_) {
879 aec_dump_->WriteRuntimeSetting(setting);
880 }
Alex Loiko73ec0192018-05-15 10:52:28 +0200881 switch (setting.type()) {
Alessio Bazzica7c19a702019-11-07 13:22:00 +0100882 case RuntimeSetting::Type::kPlayoutAudioDeviceChange: // fall-through
Alessio Bazzica7587de42019-11-11 13:32:20 +0100883 case RuntimeSetting::Type::kPlayoutVolumeChange: // fall-through
Alex Loiko73ec0192018-05-15 10:52:28 +0200884 case RuntimeSetting::Type::kCustomRenderProcessingRuntimeSetting:
saza1d600522019-10-18 13:29:43 +0200885 if (submodules_.render_pre_processor) {
886 submodules_.render_pre_processor->SetRuntimeSetting(setting);
Alex Loiko73ec0192018-05-15 10:52:28 +0200887 }
888 break;
Sam Zackrissonf0d1c032019-03-27 13:28:08 +0100889 case RuntimeSetting::Type::kCapturePreGain: // fall-through
890 case RuntimeSetting::Type::kCaptureCompressionGain: // fall-through
Per Åhgren6ee75fd2019-04-26 11:33:37 +0200891 case RuntimeSetting::Type::kCaptureFixedPostGain: // fall-through
Per Åhgren552d3e32020-08-12 08:46:47 +0200892 case RuntimeSetting::Type::kCaptureOutputUsed: // fall-through
Alessio Bazzica33444dc2018-04-20 13:16:55 +0200893 case RuntimeSetting::Type::kNotSpecified:
Alessio Bazzicac054e782018-04-16 12:10:09 +0200894 RTC_NOTREACHED();
895 break;
896 }
897 }
898}
899
peah9e6a2902017-05-15 07:19:21 -0700900void AudioProcessingImpl::QueueBandedRenderAudio(AudioBuffer* audio) {
kwibergaf476c72016-11-28 15:21:39 -0800901 RTC_DCHECK_GE(160, audio->num_frames_per_band());
peah764e3642016-10-22 05:04:30 -0700902
saza1d600522019-10-18 13:29:43 +0200903 if (submodules_.echo_control_mobile) {
Per Åhgrenb6e24d72019-04-29 12:14:50 +0200904 EchoControlMobileImpl::PackRenderAudioBuffer(audio, num_output_channels(),
905 num_reverse_channels(),
906 &aecm_render_queue_buffer_);
907 RTC_DCHECK(aecm_render_signal_queue_);
908 // Insert the samples into the queue.
909 if (!aecm_render_signal_queue_->Insert(&aecm_render_queue_buffer_)) {
910 // The data queue is full and needs to be emptied.
911 EmptyQueuedRenderAudio();
peaha0624602016-10-25 04:45:24 -0700912
Per Åhgrenb6e24d72019-04-29 12:14:50 +0200913 // Retry the insert (should always work).
914 bool result =
915 aecm_render_signal_queue_->Insert(&aecm_render_queue_buffer_);
916 RTC_DCHECK(result);
917 }
peah764e3642016-10-22 05:04:30 -0700918 }
peah701d6282016-10-25 05:42:20 -0700919
Per Åhgren0695df12020-01-13 14:43:13 +0100920 if (!submodules_.agc_manager && submodules_.gain_control) {
Per Åhgrene35b32c2019-11-22 18:22:04 +0100921 GainControlImpl::PackRenderAudioBuffer(*audio, &agc_render_queue_buffer_);
peah701d6282016-10-25 05:42:20 -0700922 // Insert the samples into the queue.
923 if (!agc_render_signal_queue_->Insert(&agc_render_queue_buffer_)) {
924 // The data queue is full and needs to be emptied.
925 EmptyQueuedRenderAudio();
926
927 // Retry the insert (should always work).
928 bool result = agc_render_signal_queue_->Insert(&agc_render_queue_buffer_);
929 RTC_DCHECK(result);
930 }
931 }
peah9e6a2902017-05-15 07:19:21 -0700932}
ivoc9f4a4a02016-10-28 05:39:16 -0700933
peah9e6a2902017-05-15 07:19:21 -0700934void AudioProcessingImpl::QueueNonbandedRenderAudio(AudioBuffer* audio) {
ivoc9f4a4a02016-10-28 05:39:16 -0700935 ResidualEchoDetector::PackRenderAudioBuffer(audio, &red_render_queue_buffer_);
936
937 // Insert the samples into the queue.
938 if (!red_render_signal_queue_->Insert(&red_render_queue_buffer_)) {
939 // The data queue is full and needs to be emptied.
940 EmptyQueuedRenderAudio();
941
942 // Retry the insert (should always work).
943 bool result = red_render_signal_queue_->Insert(&red_render_queue_buffer_);
944 RTC_DCHECK(result);
945 }
peah764e3642016-10-22 05:04:30 -0700946}
947
948void AudioProcessingImpl::AllocateRenderQueue() {
peah701d6282016-10-25 05:42:20 -0700949 const size_t new_agc_render_queue_element_max_size =
peah9e6a2902017-05-15 07:19:21 -0700950 std::max(static_cast<size_t>(1), kMaxAllowedValuesOfSamplesPerBand);
peah701d6282016-10-25 05:42:20 -0700951
ivoc9f4a4a02016-10-28 05:39:16 -0700952 const size_t new_red_render_queue_element_max_size =
953 std::max(static_cast<size_t>(1), kMaxAllowedValuesOfSamplesPerFrame);
954
peaha0624602016-10-25 04:45:24 -0700955 // Reallocate the queues if the queue item sizes are too small to fit the
956 // data to put in the queues.
peah701d6282016-10-25 05:42:20 -0700957
958 if (agc_render_queue_element_max_size_ <
959 new_agc_render_queue_element_max_size) {
960 agc_render_queue_element_max_size_ = new_agc_render_queue_element_max_size;
961
962 std::vector<int16_t> template_queue_element(
963 agc_render_queue_element_max_size_);
964
965 agc_render_signal_queue_.reset(
966 new SwapQueue<std::vector<int16_t>, RenderQueueItemVerifier<int16_t>>(
967 kMaxNumFramesToBuffer, template_queue_element,
968 RenderQueueItemVerifier<int16_t>(
969 agc_render_queue_element_max_size_)));
970
971 agc_render_queue_buffer_.resize(agc_render_queue_element_max_size_);
972 agc_capture_queue_buffer_.resize(agc_render_queue_element_max_size_);
973 } else {
974 agc_render_signal_queue_->Clear();
peah764e3642016-10-22 05:04:30 -0700975 }
ivoc9f4a4a02016-10-28 05:39:16 -0700976
977 if (red_render_queue_element_max_size_ <
978 new_red_render_queue_element_max_size) {
979 red_render_queue_element_max_size_ = new_red_render_queue_element_max_size;
980
981 std::vector<float> template_queue_element(
982 red_render_queue_element_max_size_);
983
984 red_render_signal_queue_.reset(
985 new SwapQueue<std::vector<float>, RenderQueueItemVerifier<float>>(
986 kMaxNumFramesToBuffer, template_queue_element,
987 RenderQueueItemVerifier<float>(
988 red_render_queue_element_max_size_)));
989
990 red_render_queue_buffer_.resize(red_render_queue_element_max_size_);
991 red_capture_queue_buffer_.resize(red_render_queue_element_max_size_);
992 } else {
993 red_render_signal_queue_->Clear();
994 }
peah764e3642016-10-22 05:04:30 -0700995}
996
997void AudioProcessingImpl::EmptyQueuedRenderAudio() {
Markus Handell0df0fae2020-07-07 15:53:34 +0200998 MutexLock lock_capture(&mutex_capture_);
Markus Handell02ba1d22020-05-14 14:31:18 +0200999 EmptyQueuedRenderAudioLocked();
1000}
1001
1002void AudioProcessingImpl::EmptyQueuedRenderAudioLocked() {
saza1d600522019-10-18 13:29:43 +02001003 if (submodules_.echo_control_mobile) {
Per Åhgrenb6e24d72019-04-29 12:14:50 +02001004 RTC_DCHECK(aecm_render_signal_queue_);
1005 while (aecm_render_signal_queue_->Remove(&aecm_capture_queue_buffer_)) {
saza1d600522019-10-18 13:29:43 +02001006 submodules_.echo_control_mobile->ProcessRenderAudio(
Per Åhgrenb6e24d72019-04-29 12:14:50 +02001007 aecm_capture_queue_buffer_);
1008 }
peah701d6282016-10-25 05:42:20 -07001009 }
1010
Per Åhgren0695df12020-01-13 14:43:13 +01001011 if (submodules_.gain_control) {
1012 while (agc_render_signal_queue_->Remove(&agc_capture_queue_buffer_)) {
1013 submodules_.gain_control->ProcessRenderAudio(agc_capture_queue_buffer_);
1014 }
peah764e3642016-10-22 05:04:30 -07001015 }
ivoc9f4a4a02016-10-28 05:39:16 -07001016
1017 while (red_render_signal_queue_->Remove(&red_capture_queue_buffer_)) {
saza1d600522019-10-18 13:29:43 +02001018 RTC_DCHECK(submodules_.echo_detector);
1019 submodules_.echo_detector->AnalyzeRenderAudio(red_capture_queue_buffer_);
ivoc9f4a4a02016-10-28 05:39:16 -07001020 }
peah764e3642016-10-22 05:04:30 -07001021}
1022
Per Åhgren645f24c2020-03-16 12:06:02 +01001023int AudioProcessingImpl::ProcessStream(const int16_t* const src,
1024 const StreamConfig& input_config,
1025 const StreamConfig& output_config,
Per Åhgrendc5522b2020-03-19 14:55:58 +01001026 int16_t* const dest) {
Per Åhgren71652f42020-03-17 13:23:58 +01001027 TRACE_EVENT0("webrtc", "AudioProcessing::ProcessStream_AudioFrame");
Sam Zackrisson12e319a2020-01-03 14:54:20 +01001028 RETURN_ON_ERR(MaybeInitializeCapture(input_config, output_config));
Oskar Sundbom4b276482019-05-23 14:28:00 +02001029
Markus Handell0df0fae2020-07-07 15:53:34 +02001030 MutexLock lock_capture(&mutex_capture_);
niklase@google.com470e71d2011-07-07 08:21:25 +00001031
aleloi868f32f2017-05-23 07:20:05 -07001032 if (aec_dump_) {
Per Åhgren645f24c2020-03-16 12:06:02 +01001033 RecordUnprocessedCaptureStream(src, input_config);
aleloi868f32f2017-05-23 07:20:05 -07001034 }
1035
Per Åhgren645f24c2020-03-16 12:06:02 +01001036 capture_.capture_audio->CopyFrom(src, input_config);
Gustaf Ullberg3c918b12019-10-11 13:14:44 +02001037 if (capture_.capture_fullband_audio) {
Per Åhgren645f24c2020-03-16 12:06:02 +01001038 capture_.capture_fullband_audio->CopyFrom(src, input_config);
Gustaf Ullberg3c918b12019-10-11 13:14:44 +02001039 }
peahde65ddc2016-09-16 15:02:15 -07001040 RETURN_ON_ERR(ProcessCaptureStreamLocked());
Gustaf Ullberg8675eee2019-10-09 13:34:36 +02001041 if (submodule_states_.CaptureMultiBandProcessingPresent() ||
Per Åhgrena1351272019-08-15 12:15:46 +02001042 submodule_states_.CaptureFullBandProcessingActive()) {
Gustaf Ullberg422b9e02019-10-09 13:02:14 +02001043 if (capture_.capture_fullband_audio) {
Per Åhgren645f24c2020-03-16 12:06:02 +01001044 capture_.capture_fullband_audio->CopyTo(output_config, dest);
Gustaf Ullberg422b9e02019-10-09 13:02:14 +02001045 } else {
Per Åhgren645f24c2020-03-16 12:06:02 +01001046 capture_.capture_audio->CopyTo(output_config, dest);
Gustaf Ullberg422b9e02019-10-09 13:02:14 +02001047 }
Per Åhgrena1351272019-08-15 12:15:46 +02001048 }
Per Åhgren645f24c2020-03-16 12:06:02 +01001049
aleloi868f32f2017-05-23 07:20:05 -07001050 if (aec_dump_) {
Per Åhgren645f24c2020-03-16 12:06:02 +01001051 RecordProcessedCaptureStream(dest, output_config);
aleloi868f32f2017-05-23 07:20:05 -07001052 }
andrew@webrtc.org17e40642014-03-04 20:58:13 +00001053
1054 return kNoError;
1055}
1056
peahde65ddc2016-09-16 15:02:15 -07001057int AudioProcessingImpl::ProcessCaptureStreamLocked() {
Markus Handell02ba1d22020-05-14 14:31:18 +02001058 EmptyQueuedRenderAudioLocked();
Alex Loiko73ec0192018-05-15 10:52:28 +02001059 HandleCaptureRuntimeSettings();
Alessio Bazzicac054e782018-04-16 12:10:09 +02001060
peahb58a1582016-03-15 09:34:24 -07001061 // Ensure that not both the AEC and AECM are active at the same time.
Sam Zackrisson2a959d92018-07-23 14:48:07 +00001062 // TODO(peah): Simplify once the public API Enable functions for these
1063 // are moved to APM.
Sam Zackrisson308bc642019-12-23 10:22:08 +01001064 RTC_DCHECK_LE(
1065 !!submodules_.echo_controller + !!submodules_.echo_control_mobile, 1);
peahb58a1582016-03-15 09:34:24 -07001066
peahde65ddc2016-09-16 15:02:15 -07001067 AudioBuffer* capture_buffer = capture_.capture_audio.get(); // For brevity.
Per Åhgren2e8e1c62019-12-20 00:42:22 +01001068 AudioBuffer* linear_aec_buffer = capture_.linear_aec_output.get();
ekmeyerson60d9b332015-08-14 10:35:55 -07001069
Per Åhgrenc0424252019-12-10 13:04:15 +01001070 if (submodules_.high_pass_filter &&
1071 config_.high_pass_filter.apply_in_full_band &&
1072 !constants_.enforce_split_band_hpf) {
1073 submodules_.high_pass_filter->Process(capture_buffer,
1074 /*use_split_band_data=*/false);
1075 }
1076
saza1d600522019-10-18 13:29:43 +02001077 if (submodules_.pre_amplifier) {
1078 submodules_.pre_amplifier->ApplyGain(AudioFrameView<float>(
Per Åhgrend47941e2019-08-22 11:51:13 +02001079 capture_buffer->channels(), capture_buffer->num_channels(),
Alex Loikob5c9a792018-04-16 16:31:22 +02001080 capture_buffer->num_frames()));
1081 }
1082
Per Åhgren928146f2019-08-20 09:19:21 +02001083 capture_input_rms_.Analyze(rtc::ArrayView<const float>(
Per Åhgrend47941e2019-08-22 11:51:13 +02001084 capture_buffer->channels_const()[0],
henrik.lundin290d43a2016-11-29 08:09:09 -08001085 capture_nonlocked_.capture_processing_format.num_frames()));
peah1b08dc32016-12-20 13:45:58 -08001086 const bool log_rms = ++capture_rms_interval_counter_ >= 1000;
1087 if (log_rms) {
1088 capture_rms_interval_counter_ = 0;
1089 RmsLevel::Levels levels = capture_input_rms_.AverageAndPeak();
henrik.lundin45bb5132016-12-06 04:28:04 -08001090 RTC_HISTOGRAM_COUNTS_LINEAR("WebRTC.Audio.ApmCaptureInputLevelAverageRms",
1091 levels.average, 1, RmsLevel::kMinLevelDb, 64);
1092 RTC_HISTOGRAM_COUNTS_LINEAR("WebRTC.Audio.ApmCaptureInputLevelPeakRms",
1093 levels.peak, 1, RmsLevel::kMinLevelDb, 64);
henrik.lundin290d43a2016-11-29 08:09:09 -08001094 }
1095
saza1d600522019-10-18 13:29:43 +02001096 if (submodules_.echo_controller) {
Per Åhgren88cf0502018-07-16 17:08:41 +02001097 // Detect and flag any change in the analog gain.
Markus Handell02ba1d22020-05-14 14:31:18 +02001098 int analog_mic_level = recommended_stream_analog_level_locked();
Per Åhgren88cf0502018-07-16 17:08:41 +02001099 capture_.echo_path_gain_change =
1100 capture_.prev_analog_mic_level != analog_mic_level &&
1101 capture_.prev_analog_mic_level != -1;
1102 capture_.prev_analog_mic_level = analog_mic_level;
1103
Per Åhgrend2650d12018-10-02 17:00:59 +02001104 // Detect and flag any change in the pre-amplifier gain.
saza1d600522019-10-18 13:29:43 +02001105 if (submodules_.pre_amplifier) {
1106 float pre_amp_gain = submodules_.pre_amplifier->GetGainFactor();
Per Åhgrend2650d12018-10-02 17:00:59 +02001107 capture_.echo_path_gain_change =
1108 capture_.echo_path_gain_change ||
1109 (capture_.prev_pre_amp_gain != pre_amp_gain &&
Per Åhgrene8a55692018-10-02 23:10:38 +02001110 capture_.prev_pre_amp_gain >= 0.f);
Per Åhgrend2650d12018-10-02 17:00:59 +02001111 capture_.prev_pre_amp_gain = pre_amp_gain;
1112 }
Fredrik Hernqvistca362852019-05-10 15:50:02 +02001113
1114 // Detect volume change.
1115 capture_.echo_path_gain_change =
1116 capture_.echo_path_gain_change ||
1117 (capture_.prev_playout_volume != capture_.playout_volume &&
1118 capture_.prev_playout_volume >= 0);
1119 capture_.prev_playout_volume = capture_.playout_volume;
1120
saza1d600522019-10-18 13:29:43 +02001121 submodules_.echo_controller->AnalyzeCapture(capture_buffer);
peahe0eae3c2016-12-14 01:16:23 -08001122 }
1123
Per Åhgren0695df12020-01-13 14:43:13 +01001124 if (submodules_.agc_manager) {
Per Åhgren3daedb62019-11-22 12:11:40 +01001125 submodules_.agc_manager->AnalyzePreProcess(capture_buffer);
pbos@webrtc.org788acd12014-12-15 09:41:24 +00001126 }
1127
peah2ace3f92016-09-10 04:42:27 -07001128 if (submodule_states_.CaptureMultiBandSubModulesActive() &&
1129 SampleRateSupportsMultiBand(
peahde65ddc2016-09-16 15:02:15 -07001130 capture_nonlocked_.capture_processing_format.sample_rate_hz())) {
1131 capture_buffer->SplitIntoFrequencyBands();
niklase@google.com470e71d2011-07-07 08:21:25 +00001132 }
1133
Per Åhgrene14cb992019-11-27 09:34:22 +01001134 const bool multi_channel_capture = config_.pipeline.multi_channel_capture &&
1135 constants_.multi_channel_capture_support;
1136 if (submodules_.echo_controller && !multi_channel_capture) {
peah522d71b2017-02-23 05:16:26 -08001137 // Force down-mixing of the number of channels after the detection of
1138 // capture signal saturation.
1139 // TODO(peah): Look into ensuring that this kind of tampering with the
1140 // AudioBuffer functionality should not be needed.
1141 capture_buffer->set_num_channels(1);
1142 }
1143
Per Åhgrenc0424252019-12-10 13:04:15 +01001144 if (submodules_.high_pass_filter &&
1145 (!config_.high_pass_filter.apply_in_full_band ||
1146 constants_.enforce_split_band_hpf)) {
1147 submodules_.high_pass_filter->Process(capture_buffer,
1148 /*use_split_band_data=*/true);
peah8271d042016-11-22 07:24:52 -08001149 }
Per Åhgren0cbb58e2019-10-29 22:59:44 +01001150
Per Åhgren0695df12020-01-13 14:43:13 +01001151 if (submodules_.gain_control) {
1152 RETURN_ON_ERR(
1153 submodules_.gain_control->AnalyzeCaptureAudio(*capture_buffer));
1154 }
Per Åhgren2e8e1c62019-12-20 00:42:22 +01001155
Per Åhgren8ad9e742020-01-30 07:40:58 +01001156 if ((!config_.noise_suppression.analyze_linear_aec_output_when_available ||
1157 !linear_aec_buffer || submodules_.echo_control_mobile) &&
1158 submodules_.noise_suppressor) {
1159 submodules_.noise_suppressor->Analyze(*capture_buffer);
saza0bad15f2019-10-16 11:46:11 +02001160 }
peahb58a1582016-03-15 09:34:24 -07001161
saza1d600522019-10-18 13:29:43 +02001162 if (submodules_.echo_control_mobile) {
Per Åhgrenb6e24d72019-04-29 12:14:50 +02001163 // Ensure that the stream delay was set before the call to the
1164 // AECM ProcessCaptureAudio function.
Sam Zackrisson2d31aea2020-01-17 10:55:09 +01001165 if (!capture_.was_stream_delay_set) {
Per Åhgrenb6e24d72019-04-29 12:14:50 +02001166 return AudioProcessing::kStreamParameterNotSetError;
Per Åhgrend0fa8202018-04-18 09:35:13 +02001167 }
1168
saza1d600522019-10-18 13:29:43 +02001169 if (submodules_.noise_suppressor) {
Per Åhgren0cbb58e2019-10-29 22:59:44 +01001170 submodules_.noise_suppressor->Process(capture_buffer);
Per Åhgrenb6e24d72019-04-29 12:14:50 +02001171 }
peahe0eae3c2016-12-14 01:16:23 -08001172
saza1d600522019-10-18 13:29:43 +02001173 RETURN_ON_ERR(submodules_.echo_control_mobile->ProcessCaptureAudio(
Per Åhgren46537a32017-06-07 10:08:10 +02001174 capture_buffer, stream_delay_ms()));
Per Åhgrenb6e24d72019-04-29 12:14:50 +02001175 } else {
saza1d600522019-10-18 13:29:43 +02001176 if (submodules_.echo_controller) {
Per Åhgrenb6e24d72019-04-29 12:14:50 +02001177 data_dumper_->DumpRaw("stream_delay", stream_delay_ms());
1178
Sam Zackrisson2d31aea2020-01-17 10:55:09 +01001179 if (capture_.was_stream_delay_set) {
saza1d600522019-10-18 13:29:43 +02001180 submodules_.echo_controller->SetAudioBufferDelay(stream_delay_ms());
Per Åhgrenb6e24d72019-04-29 12:14:50 +02001181 }
1182
saza1d600522019-10-18 13:29:43 +02001183 submodules_.echo_controller->ProcessCapture(
Per Åhgrenc20a19c2019-11-13 11:12:29 +01001184 capture_buffer, linear_aec_buffer, capture_.echo_path_gain_change);
Per Åhgrenb6e24d72019-04-29 12:14:50 +02001185 }
1186
Per Åhgren2e8e1c62019-12-20 00:42:22 +01001187 if (config_.noise_suppression.analyze_linear_aec_output_when_available &&
Per Åhgren8ad9e742020-01-30 07:40:58 +01001188 linear_aec_buffer && submodules_.noise_suppressor) {
1189 submodules_.noise_suppressor->Analyze(*linear_aec_buffer);
Per Åhgren2e8e1c62019-12-20 00:42:22 +01001190 }
1191
saza1d600522019-10-18 13:29:43 +02001192 if (submodules_.noise_suppressor) {
Per Åhgren0cbb58e2019-10-29 22:59:44 +01001193 submodules_.noise_suppressor->Process(capture_buffer);
saza0bad15f2019-10-16 11:46:11 +02001194 }
Per Åhgren46537a32017-06-07 10:08:10 +02001195 }
ivoc9f4a4a02016-10-28 05:39:16 -07001196
Sam Zackrisson4db667b2018-12-21 16:29:27 +01001197 if (config_.voice_detection.enabled) {
Sam Zackrisson4db667b2018-12-21 16:29:27 +01001198 capture_.stats.voice_detected =
saza1d600522019-10-18 13:29:43 +02001199 submodules_.voice_detector->ProcessCaptureAudio(capture_buffer);
Sam Zackrisson4db667b2018-12-21 16:29:27 +01001200 } else {
1201 capture_.stats.voice_detected = absl::nullopt;
1202 }
pbos@webrtc.org788acd12014-12-15 09:41:24 +00001203
Per Åhgren0695df12020-01-13 14:43:13 +01001204 if (submodules_.agc_manager) {
Per Åhgren3daedb62019-11-22 12:11:40 +01001205 submodules_.agc_manager->Process(capture_buffer);
1206
1207 absl::optional<int> new_digital_gain =
1208 submodules_.agc_manager->GetDigitalComressionGain();
Per Åhgren0695df12020-01-13 14:43:13 +01001209 if (new_digital_gain && submodules_.gain_control) {
Per Åhgren3daedb62019-11-22 12:11:40 +01001210 submodules_.gain_control->set_compression_gain_db(*new_digital_gain);
1211 }
pbos@webrtc.org788acd12014-12-15 09:41:24 +00001212 }
Per Åhgren0695df12020-01-13 14:43:13 +01001213
1214 if (submodules_.gain_control) {
1215 // TODO(peah): Add reporting from AEC3 whether there is echo.
1216 RETURN_ON_ERR(submodules_.gain_control->ProcessCaptureAudio(
1217 capture_buffer, /*stream_has_echo*/ false));
1218 }
niklase@google.com470e71d2011-07-07 08:21:25 +00001219
Gustaf Ullberg8675eee2019-10-09 13:34:36 +02001220 if (submodule_states_.CaptureMultiBandProcessingPresent() &&
peah2ace3f92016-09-10 04:42:27 -07001221 SampleRateSupportsMultiBand(
peahde65ddc2016-09-16 15:02:15 -07001222 capture_nonlocked_.capture_processing_format.sample_rate_hz())) {
1223 capture_buffer->MergeFrequencyBands();
niklase@google.com470e71d2011-07-07 08:21:25 +00001224 }
1225
Gustaf Ullberg422b9e02019-10-09 13:02:14 +02001226 if (capture_.capture_fullband_audio) {
saza1d600522019-10-18 13:29:43 +02001227 const auto& ec = submodules_.echo_controller;
Gustaf Ullberg8675eee2019-10-09 13:34:36 +02001228 bool ec_active = ec ? ec->ActiveProcessing() : false;
1229 // Only update the fullband buffer if the multiband processing has changed
1230 // the signal. Keep the original signal otherwise.
1231 if (submodule_states_.CaptureMultiBandProcessingActive(ec_active)) {
1232 capture_buffer->CopyTo(capture_.capture_fullband_audio.get());
1233 }
Gustaf Ullberg422b9e02019-10-09 13:02:14 +02001234 capture_buffer = capture_.capture_fullband_audio.get();
1235 }
1236
peah9e6a2902017-05-15 07:19:21 -07001237 if (config_.residual_echo_detector.enabled) {
saza1d600522019-10-18 13:29:43 +02001238 RTC_DCHECK(submodules_.echo_detector);
1239 submodules_.echo_detector->AnalyzeCaptureAudio(rtc::ArrayView<const float>(
1240 capture_buffer->channels()[0], capture_buffer->num_frames()));
peah9e6a2902017-05-15 07:19:21 -07001241 }
1242
pbos@webrtc.org788acd12014-12-15 09:41:24 +00001243 // TODO(aluebs): Investigate if the transient suppression placement should be
1244 // before or after the AGC.
Per Åhgrenc0734712020-01-02 15:15:36 +01001245 if (submodules_.transient_suppressor) {
saza1d600522019-10-18 13:29:43 +02001246 float voice_probability = submodules_.agc_manager.get()
1247 ? submodules_.agc_manager->voice_probability()
1248 : 1.f;
pbos@webrtc.org788acd12014-12-15 09:41:24 +00001249
saza1d600522019-10-18 13:29:43 +02001250 submodules_.transient_suppressor->Suppress(
Per Åhgrend47941e2019-08-22 11:51:13 +02001251 capture_buffer->channels()[0], capture_buffer->num_frames(),
peahde65ddc2016-09-16 15:02:15 -07001252 capture_buffer->num_channels(),
Per Åhgrend47941e2019-08-22 11:51:13 +02001253 capture_buffer->split_bands_const(0)[kBand0To8kHz],
Per Åhgrena1351272019-08-15 12:15:46 +02001254 capture_buffer->num_frames_per_band(),
1255 capture_.keyboard_info.keyboard_data,
1256 capture_.keyboard_info.num_keyboard_frames, voice_probability,
peahdf3efa82015-11-28 12:35:15 -08001257 capture_.key_pressed);
pbos@webrtc.org788acd12014-12-15 09:41:24 +00001258 }
1259
Valeriia Nemychnikovaf06eb572018-08-29 10:37:09 +02001260 // Experimental APM sub-module that analyzes |capture_buffer|.
saza1d600522019-10-18 13:29:43 +02001261 if (submodules_.capture_analyzer) {
1262 submodules_.capture_analyzer->Analyze(capture_buffer);
Valeriia Nemychnikovaf06eb572018-08-29 10:37:09 +02001263 }
1264
Per Åhgren2bd85ab2020-01-03 10:36:34 +01001265 if (submodules_.gain_controller2) {
saza1d600522019-10-18 13:29:43 +02001266 submodules_.gain_controller2->NotifyAnalogLevel(
Markus Handell02ba1d22020-05-14 14:31:18 +02001267 recommended_stream_analog_level_locked());
saza1d600522019-10-18 13:29:43 +02001268 submodules_.gain_controller2->Process(capture_buffer);
alessiob3ec96df2017-05-22 06:57:06 -07001269 }
1270
saza1d600522019-10-18 13:29:43 +02001271 if (submodules_.capture_post_processor) {
1272 submodules_.capture_post_processor->Process(capture_buffer);
Sam Zackrisson0beac582017-09-25 12:04:02 +02001273 }
1274
andrew@webrtc.org755b04a2011-11-15 16:57:56 +00001275 // The level estimator operates on the recombined data.
Sam Zackrissonb24c00f2018-11-26 16:18:25 +01001276 if (config_.level_estimation.enabled) {
saza1d600522019-10-18 13:29:43 +02001277 submodules_.output_level_estimator->ProcessStream(*capture_buffer);
1278 capture_.stats.output_rms_dbfs = submodules_.output_level_estimator->RMS();
Sam Zackrissonb24c00f2018-11-26 16:18:25 +01001279 } else {
1280 capture_.stats.output_rms_dbfs = absl::nullopt;
1281 }
ajm@google.com808e0e02011-08-03 21:08:51 +00001282
Per Åhgren928146f2019-08-20 09:19:21 +02001283 capture_output_rms_.Analyze(rtc::ArrayView<const float>(
Per Åhgrend47941e2019-08-22 11:51:13 +02001284 capture_buffer->channels_const()[0],
peah1b08dc32016-12-20 13:45:58 -08001285 capture_nonlocked_.capture_processing_format.num_frames()));
1286 if (log_rms) {
1287 RmsLevel::Levels levels = capture_output_rms_.AverageAndPeak();
1288 RTC_HISTOGRAM_COUNTS_LINEAR("WebRTC.Audio.ApmCaptureOutputLevelAverageRms",
1289 levels.average, 1, RmsLevel::kMinLevelDb, 64);
1290 RTC_HISTOGRAM_COUNTS_LINEAR("WebRTC.Audio.ApmCaptureOutputLevelPeakRms",
1291 levels.peak, 1, RmsLevel::kMinLevelDb, 64);
1292 }
1293
Per Åhgren0e3198e2019-11-18 08:52:22 +01001294 if (submodules_.agc_manager) {
Markus Handell02ba1d22020-05-14 14:31:18 +02001295 int level = recommended_stream_analog_level_locked();
Per Åhgren0e3198e2019-11-18 08:52:22 +01001296 data_dumper_->DumpRaw("experimental_gain_control_stream_analog_level", 1,
1297 &level);
1298 }
1299
Per Åhgrencf4c8722019-12-30 14:32:14 +01001300 // Compute echo-related stats.
1301 if (submodules_.echo_controller) {
1302 auto ec_metrics = submodules_.echo_controller->GetMetrics();
1303 capture_.stats.echo_return_loss = ec_metrics.echo_return_loss;
1304 capture_.stats.echo_return_loss_enhancement =
1305 ec_metrics.echo_return_loss_enhancement;
1306 capture_.stats.delay_ms = ec_metrics.delay_ms;
1307 }
1308 if (config_.residual_echo_detector.enabled) {
1309 RTC_DCHECK(submodules_.echo_detector);
1310 auto ed_metrics = submodules_.echo_detector->GetMetrics();
1311 capture_.stats.residual_echo_likelihood = ed_metrics.echo_likelihood;
1312 capture_.stats.residual_echo_likelihood_recent_max =
1313 ed_metrics.echo_likelihood_recent_max;
1314 }
1315
1316 // Pass stats for reporting.
1317 stats_reporter_.UpdateStatistics(capture_.stats);
1318
peahdf3efa82015-11-28 12:35:15 -08001319 capture_.was_stream_delay_set = false;
niklase@google.com470e71d2011-07-07 08:21:25 +00001320 return kNoError;
1321}
1322
Gustaf Ullberg8c51f2e2019-10-22 15:21:31 +02001323int AudioProcessingImpl::AnalyzeReverseStream(
1324 const float* const* data,
1325 const StreamConfig& reverse_config) {
1326 TRACE_EVENT0("webrtc", "AudioProcessing::AnalyzeReverseStream_StreamConfig");
Markus Handell0df0fae2020-07-07 15:53:34 +02001327 MutexLock lock(&mutex_render_);
Gustaf Ullberg8c51f2e2019-10-22 15:21:31 +02001328 return AnalyzeReverseStreamLocked(data, reverse_config, reverse_config);
1329}
1330
peahde65ddc2016-09-16 15:02:15 -07001331int AudioProcessingImpl::ProcessReverseStream(const float* const* src,
1332 const StreamConfig& input_config,
1333 const StreamConfig& output_config,
1334 float* const* dest) {
peah369f8282015-12-17 06:42:29 -08001335 TRACE_EVENT0("webrtc", "AudioProcessing::ProcessReverseStream_StreamConfig");
Markus Handell0df0fae2020-07-07 15:53:34 +02001336 MutexLock lock(&mutex_render_);
peahde65ddc2016-09-16 15:02:15 -07001337 RETURN_ON_ERR(AnalyzeReverseStreamLocked(src, input_config, output_config));
Alex Loiko5825aa62017-12-18 16:02:40 +01001338 if (submodule_states_.RenderMultiBandProcessingActive() ||
1339 submodule_states_.RenderFullBandProcessingActive()) {
peahdf3efa82015-11-28 12:35:15 -08001340 render_.render_audio->CopyTo(formats_.api_format.reverse_output_stream(),
1341 dest);
peah2ace3f92016-09-10 04:42:27 -07001342 } else if (formats_.api_format.reverse_input_stream() !=
1343 formats_.api_format.reverse_output_stream()) {
peahde65ddc2016-09-16 15:02:15 -07001344 render_.render_converter->Convert(src, input_config.num_samples(), dest,
1345 output_config.num_samples());
ekmeyerson60d9b332015-08-14 10:35:55 -07001346 } else {
peahde65ddc2016-09-16 15:02:15 -07001347 CopyAudioIfNeeded(src, input_config.num_frames(),
1348 input_config.num_channels(), dest);
ekmeyerson60d9b332015-08-14 10:35:55 -07001349 }
1350
1351 return kNoError;
Michael Graczyk86c6d332015-07-23 11:41:39 -07001352}
1353
peahdf3efa82015-11-28 12:35:15 -08001354int AudioProcessingImpl::AnalyzeReverseStreamLocked(
ekmeyerson60d9b332015-08-14 10:35:55 -07001355 const float* const* src,
peahde65ddc2016-09-16 15:02:15 -07001356 const StreamConfig& input_config,
1357 const StreamConfig& output_config) {
peahdf3efa82015-11-28 12:35:15 -08001358 if (src == nullptr) {
andrew@webrtc.org17e40642014-03-04 20:58:13 +00001359 return kNullPointerError;
1360 }
andrew@webrtc.org17e40642014-03-04 20:58:13 +00001361
peahde65ddc2016-09-16 15:02:15 -07001362 if (input_config.num_channels() == 0) {
Michael Graczyk86c6d332015-07-23 11:41:39 -07001363 return kBadNumberChannelsError;
andrew@webrtc.org17e40642014-03-04 20:58:13 +00001364 }
1365
peahdf3efa82015-11-28 12:35:15 -08001366 ProcessingConfig processing_config = formats_.api_format;
peahde65ddc2016-09-16 15:02:15 -07001367 processing_config.reverse_input_stream() = input_config;
1368 processing_config.reverse_output_stream() = output_config;
Michael Graczyk86c6d332015-07-23 11:41:39 -07001369
peahdf3efa82015-11-28 12:35:15 -08001370 RETURN_ON_ERR(MaybeInitializeRender(processing_config));
Fredrik Solenbergbbf21a32018-04-12 22:44:09 +02001371 RTC_DCHECK_EQ(input_config.num_frames(),
1372 formats_.api_format.reverse_input_stream().num_frames());
Michael Graczyk86c6d332015-07-23 11:41:39 -07001373
aleloi868f32f2017-05-23 07:20:05 -07001374 if (aec_dump_) {
1375 const size_t channel_size =
1376 formats_.api_format.reverse_input_stream().num_frames();
1377 const size_t num_channels =
1378 formats_.api_format.reverse_input_stream().num_channels();
1379 aec_dump_->WriteRenderStreamMessage(
Alex Loikoe36e8bb2018-02-16 11:54:07 +01001380 AudioFrameView<const float>(src, num_channels, channel_size));
aleloi868f32f2017-05-23 07:20:05 -07001381 }
peahdf3efa82015-11-28 12:35:15 -08001382 render_.render_audio->CopyFrom(src,
1383 formats_.api_format.reverse_input_stream());
peahde65ddc2016-09-16 15:02:15 -07001384 return ProcessRenderStreamLocked();
ekmeyerson60d9b332015-08-14 10:35:55 -07001385}
1386
Per Åhgren645f24c2020-03-16 12:06:02 +01001387int AudioProcessingImpl::ProcessReverseStream(const int16_t* const src,
1388 const StreamConfig& input_config,
1389 const StreamConfig& output_config,
1390 int16_t* const dest) {
Per Åhgren71652f42020-03-17 13:23:58 +01001391 TRACE_EVENT0("webrtc", "AudioProcessing::ProcessReverseStream_AudioFrame");
Per Åhgren2507f8c2020-03-19 12:33:29 +01001392
1393 if (input_config.num_channels() <= 0) {
1394 return AudioProcessing::Error::kBadNumberChannelsError;
1395 }
1396
Markus Handell0df0fae2020-07-07 15:53:34 +02001397 MutexLock lock(&mutex_render_);
peahdf3efa82015-11-28 12:35:15 -08001398 ProcessingConfig processing_config = formats_.api_format;
ekmeyerson60d9b332015-08-14 10:35:55 -07001399 processing_config.reverse_input_stream().set_sample_rate_hz(
Per Åhgren645f24c2020-03-16 12:06:02 +01001400 input_config.sample_rate_hz());
ekmeyerson60d9b332015-08-14 10:35:55 -07001401 processing_config.reverse_input_stream().set_num_channels(
Per Åhgren645f24c2020-03-16 12:06:02 +01001402 input_config.num_channels());
ekmeyerson60d9b332015-08-14 10:35:55 -07001403 processing_config.reverse_output_stream().set_sample_rate_hz(
Per Åhgren645f24c2020-03-16 12:06:02 +01001404 output_config.sample_rate_hz());
ekmeyerson60d9b332015-08-14 10:35:55 -07001405 processing_config.reverse_output_stream().set_num_channels(
Per Åhgren645f24c2020-03-16 12:06:02 +01001406 output_config.num_channels());
Michael Graczyk86c6d332015-07-23 11:41:39 -07001407
peahdf3efa82015-11-28 12:35:15 -08001408 RETURN_ON_ERR(MaybeInitializeRender(processing_config));
Per Åhgren645f24c2020-03-16 12:06:02 +01001409 if (input_config.num_frames() !=
peahdf3efa82015-11-28 12:35:15 -08001410 formats_.api_format.reverse_input_stream().num_frames()) {
andrew@webrtc.org17e40642014-03-04 20:58:13 +00001411 return kBadDataLengthError;
1412 }
niklase@google.com470e71d2011-07-07 08:21:25 +00001413
aleloi868f32f2017-05-23 07:20:05 -07001414 if (aec_dump_) {
Per Åhgren645f24c2020-03-16 12:06:02 +01001415 aec_dump_->WriteRenderStreamMessage(src, input_config.num_frames(),
1416 input_config.num_channels());
aleloi868f32f2017-05-23 07:20:05 -07001417 }
1418
Per Åhgren645f24c2020-03-16 12:06:02 +01001419 render_.render_audio->CopyFrom(src, input_config);
peahde65ddc2016-09-16 15:02:15 -07001420 RETURN_ON_ERR(ProcessRenderStreamLocked());
Per Åhgrena1351272019-08-15 12:15:46 +02001421 if (submodule_states_.RenderMultiBandProcessingActive() ||
1422 submodule_states_.RenderFullBandProcessingActive()) {
Per Åhgren645f24c2020-03-16 12:06:02 +01001423 render_.render_audio->CopyTo(output_config, dest);
Per Åhgrena1351272019-08-15 12:15:46 +02001424 }
aluebsb0319552016-03-17 20:39:53 -07001425 return kNoError;
andrew@webrtc.org17e40642014-03-04 20:58:13 +00001426}
niklase@google.com470e71d2011-07-07 08:21:25 +00001427
peahde65ddc2016-09-16 15:02:15 -07001428int AudioProcessingImpl::ProcessRenderStreamLocked() {
1429 AudioBuffer* render_buffer = render_.render_audio.get(); // For brevity.
peah9e6a2902017-05-15 07:19:21 -07001430
Alex Loiko73ec0192018-05-15 10:52:28 +02001431 HandleRenderRuntimeSettings();
1432
saza1d600522019-10-18 13:29:43 +02001433 if (submodules_.render_pre_processor) {
1434 submodules_.render_pre_processor->Process(render_buffer);
Alex Loiko5825aa62017-12-18 16:02:40 +01001435 }
1436
Alessio Bazzicad2b97402018-08-09 14:23:11 +02001437 QueueNonbandedRenderAudio(render_buffer);
1438
peah2ace3f92016-09-10 04:42:27 -07001439 if (submodule_states_.RenderMultiBandSubModulesActive() &&
peahde65ddc2016-09-16 15:02:15 -07001440 SampleRateSupportsMultiBand(
1441 formats_.render_processing_format.sample_rate_hz())) {
1442 render_buffer->SplitIntoFrequencyBands();
niklase@google.com470e71d2011-07-07 08:21:25 +00001443 }
1444
peahce4d9152017-05-19 01:28:05 -07001445 if (submodule_states_.RenderMultiBandSubModulesActive()) {
1446 QueueBandedRenderAudio(render_buffer);
1447 }
1448
Alessio Bazzicad2b97402018-08-09 14:23:11 +02001449 // TODO(peah): Perform the queuing inside QueueRenderAudiuo().
saza1d600522019-10-18 13:29:43 +02001450 if (submodules_.echo_controller) {
1451 submodules_.echo_controller->AnalyzeRender(render_buffer);
peahe0eae3c2016-12-14 01:16:23 -08001452 }
niklase@google.com470e71d2011-07-07 08:21:25 +00001453
peah2ace3f92016-09-10 04:42:27 -07001454 if (submodule_states_.RenderMultiBandProcessingActive() &&
peahde65ddc2016-09-16 15:02:15 -07001455 SampleRateSupportsMultiBand(
1456 formats_.render_processing_format.sample_rate_hz())) {
1457 render_buffer->MergeFrequencyBands();
ekmeyerson60d9b332015-08-14 10:35:55 -07001458 }
1459
andrew@webrtc.org17e40642014-03-04 20:58:13 +00001460 return kNoError;
niklase@google.com470e71d2011-07-07 08:21:25 +00001461}
1462
1463int AudioProcessingImpl::set_stream_delay_ms(int delay) {
Markus Handell0df0fae2020-07-07 15:53:34 +02001464 MutexLock lock(&mutex_capture_);
andrew@webrtc.org5f23d642012-05-29 21:14:06 +00001465 Error retval = kNoError;
peahdf3efa82015-11-28 12:35:15 -08001466 capture_.was_stream_delay_set = true;
andrew@webrtc.org6f9f8172012-03-06 19:03:39 +00001467
niklase@google.com470e71d2011-07-07 08:21:25 +00001468 if (delay < 0) {
andrew@webrtc.org5f23d642012-05-29 21:14:06 +00001469 delay = 0;
1470 retval = kBadStreamParameterWarning;
niklase@google.com470e71d2011-07-07 08:21:25 +00001471 }
1472
1473 // TODO(ajm): the max is rather arbitrarily chosen; investigate.
1474 if (delay > 500) {
andrew@webrtc.org5f23d642012-05-29 21:14:06 +00001475 delay = 500;
1476 retval = kBadStreamParameterWarning;
niklase@google.com470e71d2011-07-07 08:21:25 +00001477 }
1478
peahdf3efa82015-11-28 12:35:15 -08001479 capture_nonlocked_.stream_delay_ms = delay;
andrew@webrtc.org5f23d642012-05-29 21:14:06 +00001480 return retval;
niklase@google.com470e71d2011-07-07 08:21:25 +00001481}
1482
Per Åhgrenc20a19c2019-11-13 11:12:29 +01001483bool AudioProcessingImpl::GetLinearAecOutput(
1484 rtc::ArrayView<std::array<float, 160>> linear_output) const {
Markus Handell0df0fae2020-07-07 15:53:34 +02001485 MutexLock lock(&mutex_capture_);
Per Åhgrenc20a19c2019-11-13 11:12:29 +01001486 AudioBuffer* linear_aec_buffer = capture_.linear_aec_output.get();
1487
1488 RTC_DCHECK(linear_aec_buffer);
1489 if (linear_aec_buffer) {
1490 RTC_DCHECK_EQ(1, linear_aec_buffer->num_bands());
1491 RTC_DCHECK_EQ(linear_output.size(), linear_aec_buffer->num_channels());
1492
1493 for (size_t ch = 0; ch < linear_aec_buffer->num_channels(); ++ch) {
1494 RTC_DCHECK_EQ(linear_output[ch].size(), linear_aec_buffer->num_frames());
1495 rtc::ArrayView<const float> channel_view =
1496 rtc::ArrayView<const float>(linear_aec_buffer->channels_const()[ch],
1497 linear_aec_buffer->num_frames());
1498 std::copy(channel_view.begin(), channel_view.end(),
1499 linear_output[ch].begin());
1500 }
1501 return true;
1502 }
1503 RTC_LOG(LS_ERROR) << "No linear AEC output available";
1504 RTC_NOTREACHED();
1505 return false;
1506}
1507
niklase@google.com470e71d2011-07-07 08:21:25 +00001508int AudioProcessingImpl::stream_delay_ms() const {
peahdf3efa82015-11-28 12:35:15 -08001509 // Used as callback from submodules, hence locking is not allowed.
1510 return capture_nonlocked_.stream_delay_ms;
niklase@google.com470e71d2011-07-07 08:21:25 +00001511}
1512
andrew@webrtc.org17e40642014-03-04 20:58:13 +00001513void AudioProcessingImpl::set_stream_key_pressed(bool key_pressed) {
Markus Handell0df0fae2020-07-07 15:53:34 +02001514 MutexLock lock(&mutex_capture_);
peahdf3efa82015-11-28 12:35:15 -08001515 capture_.key_pressed = key_pressed;
andrew@webrtc.org17e40642014-03-04 20:58:13 +00001516}
1517
Sam Zackrissonf0d1c032019-03-27 13:28:08 +01001518void AudioProcessingImpl::set_stream_analog_level(int level) {
Markus Handell0df0fae2020-07-07 15:53:34 +02001519 MutexLock lock_capture(&mutex_capture_);
Per Åhgren0e3198e2019-11-18 08:52:22 +01001520
1521 if (submodules_.agc_manager) {
1522 submodules_.agc_manager->set_stream_analog_level(level);
1523 data_dumper_->DumpRaw("experimental_gain_control_set_stream_analog_level",
1524 1, &level);
Per Åhgren0695df12020-01-13 14:43:13 +01001525 } else if (submodules_.gain_control) {
Per Åhgren0e3198e2019-11-18 08:52:22 +01001526 int error = submodules_.gain_control->set_stream_analog_level(level);
1527 RTC_DCHECK_EQ(kNoError, error);
Per Åhgren0695df12020-01-13 14:43:13 +01001528 } else {
1529 capture_.cached_stream_analog_level_ = level;
Per Åhgren0e3198e2019-11-18 08:52:22 +01001530 }
Sam Zackrissonf0d1c032019-03-27 13:28:08 +01001531}
1532
1533int AudioProcessingImpl::recommended_stream_analog_level() const {
Markus Handell0df0fae2020-07-07 15:53:34 +02001534 MutexLock lock_capture(&mutex_capture_);
Markus Handell02ba1d22020-05-14 14:31:18 +02001535 return recommended_stream_analog_level_locked();
1536}
1537
1538int AudioProcessingImpl::recommended_stream_analog_level_locked() const {
Per Åhgren0e3198e2019-11-18 08:52:22 +01001539 if (submodules_.agc_manager) {
1540 return submodules_.agc_manager->stream_analog_level();
Per Åhgren0695df12020-01-13 14:43:13 +01001541 } else if (submodules_.gain_control) {
1542 return submodules_.gain_control->stream_analog_level();
1543 } else {
1544 return capture_.cached_stream_analog_level_;
Per Åhgren0e3198e2019-11-18 08:52:22 +01001545 }
Sam Zackrissonf0d1c032019-03-27 13:28:08 +01001546}
1547
Per Åhgren09e9a832020-05-11 11:03:47 +02001548bool AudioProcessingImpl::CreateAndAttachAecDump(const std::string& file_name,
1549 int64_t max_log_size_bytes,
1550 rtc::TaskQueue* worker_queue) {
1551 std::unique_ptr<AecDump> aec_dump =
1552 AecDumpFactory::Create(file_name, max_log_size_bytes, worker_queue);
1553 if (!aec_dump) {
1554 return false;
1555 }
1556
1557 AttachAecDump(std::move(aec_dump));
1558 return true;
1559}
1560
1561bool AudioProcessingImpl::CreateAndAttachAecDump(FILE* handle,
1562 int64_t max_log_size_bytes,
1563 rtc::TaskQueue* worker_queue) {
1564 std::unique_ptr<AecDump> aec_dump =
1565 AecDumpFactory::Create(handle, max_log_size_bytes, worker_queue);
1566 if (!aec_dump) {
1567 return false;
1568 }
1569
1570 AttachAecDump(std::move(aec_dump));
1571 return true;
1572}
1573
aleloi868f32f2017-05-23 07:20:05 -07001574void AudioProcessingImpl::AttachAecDump(std::unique_ptr<AecDump> aec_dump) {
1575 RTC_DCHECK(aec_dump);
Markus Handell0df0fae2020-07-07 15:53:34 +02001576 MutexLock lock_render(&mutex_render_);
1577 MutexLock lock_capture(&mutex_capture_);
aleloi868f32f2017-05-23 07:20:05 -07001578
1579 // The previously attached AecDump will be destroyed with the
1580 // 'aec_dump' parameter, which is after locks are released.
1581 aec_dump_.swap(aec_dump);
1582 WriteAecDumpConfigMessage(true);
Minyue Li656d6092018-08-10 15:38:52 +02001583 aec_dump_->WriteInitMessage(formats_.api_format, rtc::TimeUTCMillis());
aleloi868f32f2017-05-23 07:20:05 -07001584}
1585
1586void AudioProcessingImpl::DetachAecDump() {
1587 // The d-tor of a task-queue based AecDump blocks until all pending
1588 // tasks are done. This construction avoids blocking while holding
1589 // the render and capture locks.
1590 std::unique_ptr<AecDump> aec_dump = nullptr;
1591 {
Markus Handell0df0fae2020-07-07 15:53:34 +02001592 MutexLock lock_render(&mutex_render_);
1593 MutexLock lock_capture(&mutex_capture_);
aleloi868f32f2017-05-23 07:20:05 -07001594 aec_dump = std::move(aec_dump_);
1595 }
1596}
1597
peah8271d042016-11-22 07:24:52 -08001598void AudioProcessingImpl::MutateConfig(
1599 rtc::FunctionView<void(AudioProcessing::Config*)> mutator) {
Markus Handell0df0fae2020-07-07 15:53:34 +02001600 MutexLock lock_render(&mutex_render_);
1601 MutexLock lock_capture(&mutex_capture_);
peah8271d042016-11-22 07:24:52 -08001602 mutator(&config_);
1603 ApplyConfig(config_);
1604}
1605
1606AudioProcessing::Config AudioProcessingImpl::GetConfig() const {
Markus Handell0df0fae2020-07-07 15:53:34 +02001607 MutexLock lock_render(&mutex_render_);
1608 MutexLock lock_capture(&mutex_capture_);
peah8271d042016-11-22 07:24:52 -08001609 return config_;
1610}
1611
peah2ace3f92016-09-10 04:42:27 -07001612bool AudioProcessingImpl::UpdateActiveSubmoduleStates() {
1613 return submodule_states_.Update(
Per Åhgren62ea0aa2019-12-09 10:18:44 +01001614 config_.high_pass_filter.enabled, !!submodules_.echo_control_mobile,
Per Åhgren8ad9e742020-01-30 07:40:58 +01001615 config_.residual_echo_detector.enabled, !!submodules_.noise_suppressor,
Per Åhgren0695df12020-01-13 14:43:13 +01001616 !!submodules_.gain_control, !!submodules_.gain_controller2,
Per Åhgren0cbb58e2019-10-29 22:59:44 +01001617 config_.pre_amplifier.enabled, capture_nonlocked_.echo_controller_enabled,
Per Åhgrenc0734712020-01-02 15:15:36 +01001618 config_.voice_detection.enabled, !!submodules_.transient_suppressor);
ekmeyerson60d9b332015-08-14 10:35:55 -07001619}
1620
Per Åhgrenc0734712020-01-02 15:15:36 +01001621void AudioProcessingImpl::InitializeTransientSuppressor() {
1622 if (config_.transient_suppression.enabled) {
sazaaa42ecd2020-04-01 15:24:40 +02001623 // Attempt to create a transient suppressor, if one is not already created.
Per Åhgrenc0734712020-01-02 15:15:36 +01001624 if (!submodules_.transient_suppressor) {
Sam Zackrissonb37e59d2020-04-27 08:39:33 +02001625 submodules_.transient_suppressor =
1626 CreateTransientSuppressor(submodule_creation_overrides_);
pbos@webrtc.org788acd12014-12-15 09:41:24 +00001627 }
sazaaa42ecd2020-04-01 15:24:40 +02001628 if (submodules_.transient_suppressor) {
1629 submodules_.transient_suppressor->Initialize(
1630 proc_fullband_sample_rate_hz(), capture_nonlocked_.split_rate,
1631 num_proc_channels());
1632 } else {
1633 RTC_LOG(LS_WARNING)
1634 << "No transient suppressor created (probably disabled)";
1635 }
Per Åhgrenc0734712020-01-02 15:15:36 +01001636 } else {
1637 submodules_.transient_suppressor.reset();
pbos@webrtc.org788acd12014-12-15 09:41:24 +00001638 }
pbos@webrtc.org788acd12014-12-15 09:41:24 +00001639}
1640
Per Åhgren0f14db22020-01-03 14:27:14 +01001641void AudioProcessingImpl::InitializeHighPassFilter(bool forced_reset) {
Per Åhgrenb8106462019-12-04 08:34:12 +01001642 bool high_pass_filter_needed_by_aec =
1643 config_.echo_canceller.enabled &&
1644 config_.echo_canceller.enforce_high_pass_filtering &&
1645 !config_.echo_canceller.mobile_mode;
1646 if (submodule_states_.HighPassFilteringRequired() ||
1647 high_pass_filter_needed_by_aec) {
Per Åhgrenc0424252019-12-10 13:04:15 +01001648 bool use_full_band = config_.high_pass_filter.apply_in_full_band &&
1649 !constants_.enforce_split_band_hpf;
1650 int rate = use_full_band ? proc_fullband_sample_rate_hz()
1651 : proc_split_sample_rate_hz();
1652 size_t num_channels =
1653 use_full_band ? num_output_channels() : num_proc_channels();
1654
Per Åhgren0f14db22020-01-03 14:27:14 +01001655 if (!submodules_.high_pass_filter ||
1656 rate != submodules_.high_pass_filter->sample_rate_hz() ||
1657 forced_reset ||
1658 num_channels != submodules_.high_pass_filter->num_channels()) {
1659 submodules_.high_pass_filter.reset(
1660 new HighPassFilter(rate, num_channels));
1661 }
peah8271d042016-11-22 07:24:52 -08001662 } else {
saza1d600522019-10-18 13:29:43 +02001663 submodules_.high_pass_filter.reset();
peah8271d042016-11-22 07:24:52 -08001664 }
1665}
alessiob3ec96df2017-05-22 06:57:06 -07001666
Sam Zackrisson0824c6f2019-10-07 14:03:56 +02001667void AudioProcessingImpl::InitializeVoiceDetector() {
1668 if (config_.voice_detection.enabled) {
saza1d600522019-10-18 13:29:43 +02001669 submodules_.voice_detector = std::make_unique<VoiceDetection>(
Sam Zackrisson0824c6f2019-10-07 14:03:56 +02001670 proc_split_sample_rate_hz(), VoiceDetection::kVeryLowLikelihood);
1671 } else {
saza1d600522019-10-18 13:29:43 +02001672 submodules_.voice_detector.reset();
Sam Zackrisson0824c6f2019-10-07 14:03:56 +02001673 }
1674}
Gustaf Ullberg8eb9c7d2017-10-14 08:28:46 +02001675void AudioProcessingImpl::InitializeEchoController() {
Per Åhgrenb6e24d72019-04-29 12:14:50 +02001676 bool use_echo_controller =
1677 echo_control_factory_ ||
Per Åhgren62ea0aa2019-12-09 10:18:44 +01001678 (config_.echo_canceller.enabled && !config_.echo_canceller.mobile_mode);
Per Åhgrenb6e24d72019-04-29 12:14:50 +02001679
1680 if (use_echo_controller) {
1681 // Create and activate the echo controller.
Per Åhgren200feba2019-03-06 04:16:46 +01001682 if (echo_control_factory_) {
Per Åhgren4e5c7092019-11-01 20:44:11 +01001683 submodules_.echo_controller = echo_control_factory_->Create(
1684 proc_sample_rate_hz(), num_reverse_channels(), num_proc_channels());
Gustaf Ullberg2c6f3732019-11-07 17:15:12 +01001685 RTC_DCHECK(submodules_.echo_controller);
Per Åhgren200feba2019-03-06 04:16:46 +01001686 } else {
Per Åhgrenb2b58d82019-12-02 14:59:40 +01001687 EchoCanceller3Config config =
1688 use_setup_specific_default_aec3_config_
1689 ? EchoCanceller3::CreateDefaultConfig(num_reverse_channels(),
1690 num_proc_channels())
1691 : EchoCanceller3Config();
saza1d600522019-10-18 13:29:43 +02001692 submodules_.echo_controller = std::make_unique<EchoCanceller3>(
Per Åhgrenb2b58d82019-12-02 14:59:40 +01001693 config, proc_sample_rate_hz(), num_reverse_channels(),
Sam Zackrissonfeee1e42019-09-20 07:50:35 +02001694 num_proc_channels());
Per Åhgren200feba2019-03-06 04:16:46 +01001695 }
1696
Per Åhgrenc20a19c2019-11-13 11:12:29 +01001697 // Setup the storage for returning the linear AEC output.
1698 if (config_.echo_canceller.export_linear_aec_output) {
1699 constexpr int kLinearOutputRateHz = 16000;
1700 capture_.linear_aec_output = std::make_unique<AudioBuffer>(
1701 kLinearOutputRateHz, num_proc_channels(), kLinearOutputRateHz,
1702 num_proc_channels(), kLinearOutputRateHz, num_proc_channels());
1703 } else {
1704 capture_.linear_aec_output.reset();
1705 }
1706
Per Åhgren200feba2019-03-06 04:16:46 +01001707 capture_nonlocked_.echo_controller_enabled = true;
Per Åhgren200feba2019-03-06 04:16:46 +01001708
saza1d600522019-10-18 13:29:43 +02001709 submodules_.echo_control_mobile.reset();
Per Åhgrenb6e24d72019-04-29 12:14:50 +02001710 aecm_render_signal_queue_.reset();
Per Åhgrenf204faf2019-04-25 15:18:06 +02001711 return;
peahe0eae3c2016-12-14 01:16:23 -08001712 }
Per Åhgrenf204faf2019-04-25 15:18:06 +02001713
saza1d600522019-10-18 13:29:43 +02001714 submodules_.echo_controller.reset();
Per Åhgrenf204faf2019-04-25 15:18:06 +02001715 capture_nonlocked_.echo_controller_enabled = false;
Per Åhgrenc20a19c2019-11-13 11:12:29 +01001716 capture_.linear_aec_output.reset();
Per Åhgrenf204faf2019-04-25 15:18:06 +02001717
1718 if (!config_.echo_canceller.enabled) {
saza1d600522019-10-18 13:29:43 +02001719 submodules_.echo_control_mobile.reset();
Per Åhgrenb6e24d72019-04-29 12:14:50 +02001720 aecm_render_signal_queue_.reset();
Per Åhgrenf204faf2019-04-25 15:18:06 +02001721 return;
1722 }
1723
1724 if (config_.echo_canceller.mobile_mode) {
1725 // Create and activate AECM.
Per Åhgrenb6e24d72019-04-29 12:14:50 +02001726 size_t max_element_size =
1727 std::max(static_cast<size_t>(1),
1728 kMaxAllowedValuesOfSamplesPerBand *
1729 EchoControlMobileImpl::NumCancellersRequired(
1730 num_output_channels(), num_reverse_channels()));
1731
1732 std::vector<int16_t> template_queue_element(max_element_size);
1733
1734 aecm_render_signal_queue_.reset(
1735 new SwapQueue<std::vector<int16_t>, RenderQueueItemVerifier<int16_t>>(
1736 kMaxNumFramesToBuffer, template_queue_element,
1737 RenderQueueItemVerifier<int16_t>(max_element_size)));
1738
1739 aecm_render_queue_buffer_.resize(max_element_size);
1740 aecm_capture_queue_buffer_.resize(max_element_size);
1741
saza1d600522019-10-18 13:29:43 +02001742 submodules_.echo_control_mobile.reset(new EchoControlMobileImpl());
Per Åhgrenb6e24d72019-04-29 12:14:50 +02001743
saza1d600522019-10-18 13:29:43 +02001744 submodules_.echo_control_mobile->Initialize(proc_split_sample_rate_hz(),
1745 num_reverse_channels(),
1746 num_output_channels());
Per Åhgrenf204faf2019-04-25 15:18:06 +02001747 return;
1748 }
1749
saza1d600522019-10-18 13:29:43 +02001750 submodules_.echo_control_mobile.reset();
Per Åhgrenb6e24d72019-04-29 12:14:50 +02001751 aecm_render_signal_queue_.reset();
peahe0eae3c2016-12-14 01:16:23 -08001752}
peah8271d042016-11-22 07:24:52 -08001753
Per Åhgren0695df12020-01-13 14:43:13 +01001754void AudioProcessingImpl::InitializeGainController1() {
1755 if (!config_.gain_controller1.enabled) {
1756 submodules_.agc_manager.reset();
1757 submodules_.gain_control.reset();
1758 return;
1759 }
1760
1761 if (!submodules_.gain_control) {
1762 submodules_.gain_control.reset(new GainControlImpl());
1763 }
1764
1765 submodules_.gain_control->Initialize(num_proc_channels(),
1766 proc_sample_rate_hz());
1767
1768 if (!config_.gain_controller1.analog_gain_controller.enabled) {
1769 int error = submodules_.gain_control->set_mode(
1770 Agc1ConfigModeToInterfaceMode(config_.gain_controller1.mode));
1771 RTC_DCHECK_EQ(kNoError, error);
1772 error = submodules_.gain_control->set_target_level_dbfs(
1773 config_.gain_controller1.target_level_dbfs);
1774 RTC_DCHECK_EQ(kNoError, error);
1775 error = submodules_.gain_control->set_compression_gain_db(
1776 config_.gain_controller1.compression_gain_db);
1777 RTC_DCHECK_EQ(kNoError, error);
1778 error = submodules_.gain_control->enable_limiter(
1779 config_.gain_controller1.enable_limiter);
1780 RTC_DCHECK_EQ(kNoError, error);
1781 error = submodules_.gain_control->set_analog_level_limits(
1782 config_.gain_controller1.analog_level_minimum,
1783 config_.gain_controller1.analog_level_maximum);
1784 RTC_DCHECK_EQ(kNoError, error);
1785
1786 submodules_.agc_manager.reset();
1787 return;
1788 }
1789
1790 if (!submodules_.agc_manager.get() ||
1791 submodules_.agc_manager->num_channels() !=
1792 static_cast<int>(num_proc_channels()) ||
1793 submodules_.agc_manager->sample_rate_hz() !=
1794 capture_nonlocked_.split_rate) {
1795 int stream_analog_level = -1;
1796 const bool re_creation = !!submodules_.agc_manager;
1797 if (re_creation) {
1798 stream_analog_level = submodules_.agc_manager->stream_analog_level();
1799 }
1800 submodules_.agc_manager.reset(new AgcManagerDirect(
1801 num_proc_channels(),
1802 config_.gain_controller1.analog_gain_controller.startup_min_volume,
1803 config_.gain_controller1.analog_gain_controller.clipped_level_min,
1804 config_.gain_controller1.analog_gain_controller
1805 .enable_agc2_level_estimator,
1806 !config_.gain_controller1.analog_gain_controller
1807 .enable_digital_adaptive,
1808 capture_nonlocked_.split_rate));
1809 if (re_creation) {
1810 submodules_.agc_manager->set_stream_analog_level(stream_analog_level);
1811 }
1812 }
1813 submodules_.agc_manager->Initialize();
1814 submodules_.agc_manager->SetupDigitalGainControl(
1815 submodules_.gain_control.get());
1816 submodules_.agc_manager->SetCaptureMuted(capture_.output_will_be_muted);
1817}
1818
alessiob3ec96df2017-05-22 06:57:06 -07001819void AudioProcessingImpl::InitializeGainController2() {
Alessio Bazzica270f7b52017-10-13 11:05:17 +02001820 if (config_.gain_controller2.enabled) {
Per Åhgren2bd85ab2020-01-03 10:36:34 +01001821 if (!submodules_.gain_controller2) {
1822 // TODO(alessiob): Move the injected gain controller once injection is
1823 // implemented.
1824 submodules_.gain_controller2.reset(new GainController2());
1825 }
1826
saza1d600522019-10-18 13:29:43 +02001827 submodules_.gain_controller2->Initialize(proc_fullband_sample_rate_hz());
Per Åhgren2bd85ab2020-01-03 10:36:34 +01001828 submodules_.gain_controller2->ApplyConfig(config_.gain_controller2);
1829 } else {
1830 submodules_.gain_controller2.reset();
alessiob3ec96df2017-05-22 06:57:06 -07001831 }
1832}
1833
saza0bad15f2019-10-16 11:46:11 +02001834void AudioProcessingImpl::InitializeNoiseSuppressor() {
Per Åhgren0cbb58e2019-10-29 22:59:44 +01001835 submodules_.noise_suppressor.reset();
1836
saza0bad15f2019-10-16 11:46:11 +02001837 if (config_.noise_suppression.enabled) {
sazaaa42ecd2020-04-01 15:24:40 +02001838 auto map_level =
1839 [](AudioProcessing::Config::NoiseSuppression::Level level) {
1840 using NoiseSuppresionConfig =
1841 AudioProcessing::Config::NoiseSuppression;
1842 switch (level) {
1843 case NoiseSuppresionConfig::kLow:
1844 return NsConfig::SuppressionLevel::k6dB;
1845 case NoiseSuppresionConfig::kModerate:
1846 return NsConfig::SuppressionLevel::k12dB;
1847 case NoiseSuppresionConfig::kHigh:
1848 return NsConfig::SuppressionLevel::k18dB;
1849 case NoiseSuppresionConfig::kVeryHigh:
1850 return NsConfig::SuppressionLevel::k21dB;
1851 default:
1852 RTC_NOTREACHED();
1853 }
1854 };
Per Åhgren0cbb58e2019-10-29 22:59:44 +01001855
sazaaa42ecd2020-04-01 15:24:40 +02001856 NsConfig cfg;
1857 cfg.target_level = map_level(config_.noise_suppression.level);
1858 submodules_.noise_suppressor = std::make_unique<NoiseSuppressor>(
1859 cfg, proc_sample_rate_hz(), num_proc_channels());
saza0bad15f2019-10-16 11:46:11 +02001860 }
1861}
1862
Alex Loikob5c9a792018-04-16 16:31:22 +02001863void AudioProcessingImpl::InitializePreAmplifier() {
1864 if (config_.pre_amplifier.enabled) {
saza1d600522019-10-18 13:29:43 +02001865 submodules_.pre_amplifier.reset(
Alex Loikob5c9a792018-04-16 16:31:22 +02001866 new GainApplier(true, config_.pre_amplifier.fixed_gain_factor));
1867 } else {
saza1d600522019-10-18 13:29:43 +02001868 submodules_.pre_amplifier.reset();
Alex Loikob5c9a792018-04-16 16:31:22 +02001869 }
1870}
1871
ivoc9f4a4a02016-10-28 05:39:16 -07001872void AudioProcessingImpl::InitializeResidualEchoDetector() {
saza1d600522019-10-18 13:29:43 +02001873 RTC_DCHECK(submodules_.echo_detector);
1874 submodules_.echo_detector->Initialize(
Gustaf Ullberg422b9e02019-10-09 13:02:14 +02001875 proc_fullband_sample_rate_hz(), 1,
Ivo Creusenb1facc12018-04-12 16:15:58 +02001876 formats_.render_processing_format.sample_rate_hz(), 1);
ivoc9f4a4a02016-10-28 05:39:16 -07001877}
1878
Valeriia Nemychnikovaf06eb572018-08-29 10:37:09 +02001879void AudioProcessingImpl::InitializeAnalyzer() {
saza1d600522019-10-18 13:29:43 +02001880 if (submodules_.capture_analyzer) {
1881 submodules_.capture_analyzer->Initialize(proc_fullband_sample_rate_hz(),
1882 num_proc_channels());
Valeriia Nemychnikovaf06eb572018-08-29 10:37:09 +02001883 }
1884}
1885
Sam Zackrisson0beac582017-09-25 12:04:02 +02001886void AudioProcessingImpl::InitializePostProcessor() {
saza1d600522019-10-18 13:29:43 +02001887 if (submodules_.capture_post_processor) {
1888 submodules_.capture_post_processor->Initialize(
Gustaf Ullberg422b9e02019-10-09 13:02:14 +02001889 proc_fullband_sample_rate_hz(), num_proc_channels());
Sam Zackrisson0beac582017-09-25 12:04:02 +02001890 }
1891}
1892
Alex Loiko5825aa62017-12-18 16:02:40 +01001893void AudioProcessingImpl::InitializePreProcessor() {
saza1d600522019-10-18 13:29:43 +02001894 if (submodules_.render_pre_processor) {
1895 submodules_.render_pre_processor->Initialize(
Alex Loiko5825aa62017-12-18 16:02:40 +01001896 formats_.render_processing_format.sample_rate_hz(),
1897 formats_.render_processing_format.num_channels());
1898 }
1899}
1900
aleloi868f32f2017-05-23 07:20:05 -07001901void AudioProcessingImpl::WriteAecDumpConfigMessage(bool forced) {
1902 if (!aec_dump_) {
1903 return;
1904 }
Per Åhgrenf204faf2019-04-25 15:18:06 +02001905
1906 std::string experiments_description = "";
aleloi868f32f2017-05-23 07:20:05 -07001907 // TODO(peah): Add semicolon-separated concatenations of experiment
1908 // descriptions for other submodules.
Per Åhgren0695df12020-01-13 14:43:13 +01001909 if (config_.gain_controller1.analog_gain_controller.clipped_level_min !=
1910 kClippedLevelMin) {
aleloi868f32f2017-05-23 07:20:05 -07001911 experiments_description += "AgcClippingLevelExperiment;";
1912 }
Sam Zackrisson701bd172020-02-18 14:50:28 +01001913 if (!!submodules_.capture_post_processor) {
1914 experiments_description += "CapturePostProcessor;";
1915 }
1916 if (!!submodules_.render_pre_processor) {
1917 experiments_description += "RenderPreProcessor;";
1918 }
Gustaf Ullbergce045ac2017-10-16 13:49:04 +02001919 if (capture_nonlocked_.echo_controller_enabled) {
1920 experiments_description += "EchoController;";
aleloi868f32f2017-05-23 07:20:05 -07001921 }
Alessio Bazzica270f7b52017-10-13 11:05:17 +02001922 if (config_.gain_controller2.enabled) {
1923 experiments_description += "GainController2;";
1924 }
aleloi868f32f2017-05-23 07:20:05 -07001925
1926 InternalAPMConfig apm_config;
1927
Per Åhgren200feba2019-03-06 04:16:46 +01001928 apm_config.aec_enabled = config_.echo_canceller.enabled;
Per Åhgren62ea0aa2019-12-09 10:18:44 +01001929 apm_config.aec_delay_agnostic_enabled = false;
1930 apm_config.aec_extended_filter_enabled = false;
1931 apm_config.aec_suppression_level = 0;
aleloi868f32f2017-05-23 07:20:05 -07001932
saza1d600522019-10-18 13:29:43 +02001933 apm_config.aecm_enabled = !!submodules_.echo_control_mobile;
aleloi868f32f2017-05-23 07:20:05 -07001934 apm_config.aecm_comfort_noise_enabled =
saza1d600522019-10-18 13:29:43 +02001935 submodules_.echo_control_mobile &&
1936 submodules_.echo_control_mobile->is_comfort_noise_enabled();
Per Åhgrenb6e24d72019-04-29 12:14:50 +02001937 apm_config.aecm_routing_mode =
saza1d600522019-10-18 13:29:43 +02001938 submodules_.echo_control_mobile
1939 ? static_cast<int>(submodules_.echo_control_mobile->routing_mode())
Per Åhgrenb6e24d72019-04-29 12:14:50 +02001940 : 0;
aleloi868f32f2017-05-23 07:20:05 -07001941
Per Åhgren0695df12020-01-13 14:43:13 +01001942 apm_config.agc_enabled = !!submodules_.gain_control;
1943
1944 apm_config.agc_mode = submodules_.gain_control
1945 ? static_cast<int>(submodules_.gain_control->mode())
1946 : GainControl::kAdaptiveAnalog;
aleloi868f32f2017-05-23 07:20:05 -07001947 apm_config.agc_limiter_enabled =
Per Åhgren0695df12020-01-13 14:43:13 +01001948 submodules_.gain_control ? submodules_.gain_control->is_limiter_enabled()
1949 : false;
Per Åhgren0e3198e2019-11-18 08:52:22 +01001950 apm_config.noise_robust_agc_enabled = !!submodules_.agc_manager;
aleloi868f32f2017-05-23 07:20:05 -07001951
1952 apm_config.hpf_enabled = config_.high_pass_filter.enabled;
1953
saza0bad15f2019-10-16 11:46:11 +02001954 apm_config.ns_enabled = config_.noise_suppression.enabled;
1955 apm_config.ns_level = static_cast<int>(config_.noise_suppression.level);
aleloi868f32f2017-05-23 07:20:05 -07001956
1957 apm_config.transient_suppression_enabled =
Per Åhgrenc0734712020-01-02 15:15:36 +01001958 config_.transient_suppression.enabled;
aleloi868f32f2017-05-23 07:20:05 -07001959 apm_config.experiments_description = experiments_description;
Alex Loiko5feb30e2018-04-16 13:52:32 +02001960 apm_config.pre_amplifier_enabled = config_.pre_amplifier.enabled;
1961 apm_config.pre_amplifier_fixed_gain_factor =
1962 config_.pre_amplifier.fixed_gain_factor;
aleloi868f32f2017-05-23 07:20:05 -07001963
1964 if (!forced && apm_config == apm_config_for_aec_dump_) {
1965 return;
1966 }
1967 aec_dump_->WriteConfig(apm_config);
1968 apm_config_for_aec_dump_ = apm_config;
1969}
1970
1971void AudioProcessingImpl::RecordUnprocessedCaptureStream(
1972 const float* const* src) {
1973 RTC_DCHECK(aec_dump_);
1974 WriteAecDumpConfigMessage(false);
1975
1976 const size_t channel_size = formats_.api_format.input_stream().num_frames();
1977 const size_t num_channels = formats_.api_format.input_stream().num_channels();
1978 aec_dump_->AddCaptureStreamInput(
Alex Loikoe36e8bb2018-02-16 11:54:07 +01001979 AudioFrameView<const float>(src, num_channels, channel_size));
aleloi868f32f2017-05-23 07:20:05 -07001980 RecordAudioProcessingState();
1981}
1982
1983void AudioProcessingImpl::RecordUnprocessedCaptureStream(
Per Åhgren645f24c2020-03-16 12:06:02 +01001984 const int16_t* const data,
1985 const StreamConfig& config) {
aleloi868f32f2017-05-23 07:20:05 -07001986 RTC_DCHECK(aec_dump_);
1987 WriteAecDumpConfigMessage(false);
1988
Per Åhgren645f24c2020-03-16 12:06:02 +01001989 aec_dump_->AddCaptureStreamInput(data, config.num_channels(),
1990 config.num_frames());
aleloi868f32f2017-05-23 07:20:05 -07001991 RecordAudioProcessingState();
1992}
1993
1994void AudioProcessingImpl::RecordProcessedCaptureStream(
1995 const float* const* processed_capture_stream) {
1996 RTC_DCHECK(aec_dump_);
1997
1998 const size_t channel_size = formats_.api_format.output_stream().num_frames();
1999 const size_t num_channels =
2000 formats_.api_format.output_stream().num_channels();
Alex Loikoe36e8bb2018-02-16 11:54:07 +01002001 aec_dump_->AddCaptureStreamOutput(AudioFrameView<const float>(
2002 processed_capture_stream, num_channels, channel_size));
aleloi868f32f2017-05-23 07:20:05 -07002003 aec_dump_->WriteCaptureStreamMessage();
2004}
2005
2006void AudioProcessingImpl::RecordProcessedCaptureStream(
Per Åhgren645f24c2020-03-16 12:06:02 +01002007 const int16_t* const data,
2008 const StreamConfig& config) {
aleloi868f32f2017-05-23 07:20:05 -07002009 RTC_DCHECK(aec_dump_);
2010
Per Åhgren645f24c2020-03-16 12:06:02 +01002011 aec_dump_->AddCaptureStreamOutput(data, config.num_channels(),
Per Åhgren088329f2020-03-18 21:59:52 +01002012 config.num_frames());
aleloi868f32f2017-05-23 07:20:05 -07002013 aec_dump_->WriteCaptureStreamMessage();
2014}
2015
2016void AudioProcessingImpl::RecordAudioProcessingState() {
2017 RTC_DCHECK(aec_dump_);
2018 AecDump::AudioProcessingState audio_proc_state;
2019 audio_proc_state.delay = capture_nonlocked_.stream_delay_ms;
Per Åhgren62ea0aa2019-12-09 10:18:44 +01002020 audio_proc_state.drift = 0;
Markus Handell02ba1d22020-05-14 14:31:18 +02002021 audio_proc_state.level = recommended_stream_analog_level_locked();
aleloi868f32f2017-05-23 07:20:05 -07002022 audio_proc_state.keypress = capture_.key_pressed;
2023 aec_dump_->AddAudioProcessingState(audio_proc_state);
2024}
2025
Per Åhgrenc0734712020-01-02 15:15:36 +01002026AudioProcessingImpl::ApmCaptureState::ApmCaptureState()
Sam Zackrisson2d31aea2020-01-17 10:55:09 +01002027 : was_stream_delay_set(false),
kwiberg83ffe452016-08-29 14:46:07 -07002028 output_will_be_muted(false),
2029 key_pressed(false),
peahde65ddc2016-09-16 15:02:15 -07002030 capture_processing_format(kSampleRate16kHz),
peah67995532017-04-10 14:12:41 -07002031 split_rate(kSampleRate16kHz),
Per Åhgren88cf0502018-07-16 17:08:41 +02002032 echo_path_gain_change(false),
Per Åhgrend2650d12018-10-02 17:00:59 +02002033 prev_analog_mic_level(-1),
Fredrik Hernqvistca362852019-05-10 15:50:02 +02002034 prev_pre_amp_gain(-1.f),
2035 playout_volume(-1),
2036 prev_playout_volume(-1) {}
kwiberg83ffe452016-08-29 14:46:07 -07002037
2038AudioProcessingImpl::ApmCaptureState::~ApmCaptureState() = default;
2039
Per Åhgrena1351272019-08-15 12:15:46 +02002040void AudioProcessingImpl::ApmCaptureState::KeyboardInfo::Extract(
2041 const float* const* data,
2042 const StreamConfig& stream_config) {
2043 if (stream_config.has_keyboard()) {
2044 keyboard_data = data[stream_config.num_channels()];
2045 } else {
2046 keyboard_data = NULL;
2047 }
2048 num_keyboard_frames = stream_config.num_frames();
2049}
2050
kwiberg83ffe452016-08-29 14:46:07 -07002051AudioProcessingImpl::ApmRenderState::ApmRenderState() = default;
2052
2053AudioProcessingImpl::ApmRenderState::~ApmRenderState() = default;
2054
Per Åhgrencf4c8722019-12-30 14:32:14 +01002055AudioProcessingImpl::ApmStatsReporter::ApmStatsReporter()
2056 : stats_message_queue_(1) {}
2057
2058AudioProcessingImpl::ApmStatsReporter::~ApmStatsReporter() = default;
2059
2060AudioProcessingStats AudioProcessingImpl::ApmStatsReporter::GetStatistics() {
Markus Handell0df0fae2020-07-07 15:53:34 +02002061 MutexLock lock_stats(&mutex_stats_);
Per Åhgrencf4c8722019-12-30 14:32:14 +01002062 bool new_stats_available = stats_message_queue_.Remove(&cached_stats_);
2063 // If the message queue is full, return the cached stats.
2064 static_cast<void>(new_stats_available);
2065
2066 return cached_stats_;
2067}
2068
2069void AudioProcessingImpl::ApmStatsReporter::UpdateStatistics(
2070 const AudioProcessingStats& new_stats) {
2071 AudioProcessingStats stats_to_queue = new_stats;
2072 bool stats_message_passed = stats_message_queue_.Insert(&stats_to_queue);
2073 // If the message queue is full, discard the new stats.
2074 static_cast<void>(stats_message_passed);
2075}
2076
niklase@google.com470e71d2011-07-07 08:21:25 +00002077} // namespace webrtc