blob: 166853916d1bb1ee391caa69fd4708e3f79596a9 [file] [log] [blame]
niklase@google.com470e71d2011-07-07 08:21:25 +00001/*
andrew@webrtc.org40654032012-01-30 20:51:15 +00002 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
niklase@google.com470e71d2011-07-07 08:21:25 +00003 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020011#include "modules/audio_processing/audio_processing_impl.h"
niklase@google.com470e71d2011-07-07 08:21:25 +000012
Michael Graczyk86c6d332015-07-23 11:41:39 -070013#include <algorithm>
Yves Gerey988cc082018-10-23 12:03:01 +020014#include <cstdint>
Mirko Bonadei317a1f02019-09-17 17:06:18 +020015#include <memory>
alessiob3ec96df2017-05-22 06:57:06 -070016#include <string>
Yves Gerey988cc082018-10-23 12:03:01 +020017#include <type_traits>
18#include <utility>
niklase@google.com470e71d2011-07-07 08:21:25 +000019
Yves Gerey988cc082018-10-23 12:03:01 +020020#include "absl/types/optional.h"
21#include "api/array_view.h"
Per Åhgren645f24c2020-03-16 12:06:02 +010022#include "api/audio/audio_frame.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020023#include "common_audio/audio_converter.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020024#include "common_audio/include/audio_util.h"
Per Åhgren09e9a832020-05-11 11:03:47 +020025#include "modules/audio_processing/aec_dump/aec_dump_factory.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020026#include "modules/audio_processing/audio_buffer.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020027#include "modules/audio_processing/common.h"
Yves Gerey988cc082018-10-23 12:03:01 +020028#include "modules/audio_processing/include/audio_frame_view.h"
Per Åhgren13735822018-02-12 21:42:56 +010029#include "modules/audio_processing/logging/apm_data_dumper.h"
Sam Zackrissonb37e59d2020-04-27 08:39:33 +020030#include "modules/audio_processing/optionally_built_submodule_creators.h"
Steve Anton10542f22019-01-11 09:11:00 -080031#include "rtc_base/atomic_ops.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020032#include "rtc_base/checks.h"
Steve Anton10542f22019-01-11 09:11:00 -080033#include "rtc_base/constructor_magic.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020034#include "rtc_base/logging.h"
Steve Anton10542f22019-01-11 09:11:00 -080035#include "rtc_base/ref_counted_object.h"
36#include "rtc_base/time_utils.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020037#include "rtc_base/trace_event.h"
Alessio Bazzica0441bb62021-08-10 15:23:23 +020038#include "system_wrappers/include/denormal_disabler.h"
Sam Zackrissonfeee1e42019-09-20 07:50:35 +020039#include "system_wrappers/include/field_trial.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020040#include "system_wrappers/include/metrics.h"
andrew@webrtc.org7bf26462011-12-03 00:03:31 +000041
Michael Graczyk86c6d332015-07-23 11:41:39 -070042#define RETURN_ON_ERR(expr) \
43 do { \
44 int err = (expr); \
45 if (err != kNoError) { \
46 return err; \
47 } \
andrew@webrtc.org60730cf2014-01-07 17:45:09 +000048 } while (0)
49
niklase@google.com470e71d2011-07-07 08:21:25 +000050namespace webrtc {
aluebsdf6416a2016-03-16 18:26:35 -070051
Michael Graczyk86c6d332015-07-23 11:41:39 -070052namespace {
53
54static bool LayoutHasKeyboard(AudioProcessing::ChannelLayout layout) {
55 switch (layout) {
56 case AudioProcessing::kMono:
57 case AudioProcessing::kStereo:
58 return false;
59 case AudioProcessing::kMonoAndKeyboard:
60 case AudioProcessing::kStereoAndKeyboard:
61 return true;
62 }
63
kwiberg9e2be5f2016-09-14 05:23:22 -070064 RTC_NOTREACHED();
Michael Graczyk86c6d332015-07-23 11:41:39 -070065 return false;
66}
aluebsdf6416a2016-03-16 18:26:35 -070067
peah2ace3f92016-09-10 04:42:27 -070068bool SampleRateSupportsMultiBand(int sample_rate_hz) {
aluebsdf6416a2016-03-16 18:26:35 -070069 return sample_rate_hz == AudioProcessing::kSampleRate32kHz ||
70 sample_rate_hz == AudioProcessing::kSampleRate48kHz;
71}
72
Per Åhgrenc0424252019-12-10 13:04:15 +010073// Checks whether the high-pass filter should be done in the full-band.
74bool EnforceSplitBandHpf() {
75 return field_trial::IsEnabled("WebRTC-FullBandHpfKillSwitch");
76}
77
Per Åhgrenb2b58d82019-12-02 14:59:40 +010078// Checks whether AEC3 should be allowed to decide what the default
79// configuration should be based on the render and capture channel configuration
80// at hand.
81bool UseSetupSpecificDefaultAec3Congfig() {
82 return !field_trial::IsEnabled(
83 "WebRTC-Aec3SetupSpecificDefaultConfigDefaultsKillSwitch");
84}
85
Per Åhgrenc8626b62019-08-23 15:49:51 +020086// Identify the native processing rate that best handles a sample rate.
Per Åhgrenfcbe4072019-09-15 00:27:58 +020087int SuitableProcessRate(int minimum_rate,
88 int max_splitting_rate,
89 bool band_splitting_required) {
Per Åhgrenc8626b62019-08-23 15:49:51 +020090 const int uppermost_native_rate =
Per Åhgrenfcbe4072019-09-15 00:27:58 +020091 band_splitting_required ? max_splitting_rate : 48000;
Per Åhgrenc8626b62019-08-23 15:49:51 +020092 for (auto rate : {16000, 32000, 48000}) {
peah2ace3f92016-09-10 04:42:27 -070093 if (rate >= uppermost_native_rate) {
94 return uppermost_native_rate;
95 }
96 if (rate >= minimum_rate) {
aluebsdf6416a2016-03-16 18:26:35 -070097 return rate;
98 }
99 }
peah2ace3f92016-09-10 04:42:27 -0700100 RTC_NOTREACHED();
101 return uppermost_native_rate;
aluebsdf6416a2016-03-16 18:26:35 -0700102}
103
Sam Zackrissonf0d1c032019-03-27 13:28:08 +0100104GainControl::Mode Agc1ConfigModeToInterfaceMode(
105 AudioProcessing::Config::GainController1::Mode mode) {
106 using Agc1Config = AudioProcessing::Config::GainController1;
107 switch (mode) {
108 case Agc1Config::kAdaptiveAnalog:
109 return GainControl::kAdaptiveAnalog;
110 case Agc1Config::kAdaptiveDigital:
111 return GainControl::kAdaptiveDigital;
112 case Agc1Config::kFixedDigital:
113 return GainControl::kFixedDigital;
114 }
Karl Wibergc95b9392020-11-08 00:49:37 +0100115 RTC_CHECK_NOTREACHED();
Sam Zackrissonf0d1c032019-03-27 13:28:08 +0100116}
117
Per Åhgren19775cb2021-03-12 23:08:09 +0000118bool MinimizeProcessingForUnusedOutput() {
119 return !field_trial::IsEnabled("WebRTC-MutedStateKillSwitch");
120}
121
peah9e6a2902017-05-15 07:19:21 -0700122// Maximum lengths that frame of samples being passed from the render side to
123// the capture side can have (does not apply to AEC3).
124static const size_t kMaxAllowedValuesOfSamplesPerBand = 160;
125static const size_t kMaxAllowedValuesOfSamplesPerFrame = 480;
126
peah764e3642016-10-22 05:04:30 -0700127// Maximum number of frames to buffer in the render queue.
128// TODO(peah): Decrease this once we properly handle hugely unbalanced
129// reverse and forward call numbers.
130static const size_t kMaxNumFramesToBuffer = 100;
Alessio Bazzica3438a932020-10-14 12:47:50 +0200131
Michael Graczyk86c6d332015-07-23 11:41:39 -0700132} // namespace
andrew@webrtc.org60730cf2014-01-07 17:45:09 +0000133
134// Throughout webrtc, it's assumed that success is represented by zero.
kwiberg@webrtc.org2ebfac52015-01-14 10:51:54 +0000135static_assert(AudioProcessing::kNoError == 0, "kNoError must be zero");
andrew@webrtc.org60730cf2014-01-07 17:45:09 +0000136
saza1d600522019-10-18 13:29:43 +0200137AudioProcessingImpl::SubmoduleStates::SubmoduleStates(
Alex Loiko5825aa62017-12-18 16:02:40 +0100138 bool capture_post_processor_enabled,
Valeriia Nemychnikovaf06eb572018-08-29 10:37:09 +0200139 bool render_pre_processor_enabled,
140 bool capture_analyzer_enabled)
Alex Loiko5825aa62017-12-18 16:02:40 +0100141 : capture_post_processor_enabled_(capture_post_processor_enabled),
Valeriia Nemychnikovaf06eb572018-08-29 10:37:09 +0200142 render_pre_processor_enabled_(render_pre_processor_enabled),
143 capture_analyzer_enabled_(capture_analyzer_enabled) {}
peah2ace3f92016-09-10 04:42:27 -0700144
saza1d600522019-10-18 13:29:43 +0200145bool AudioProcessingImpl::SubmoduleStates::Update(
Sam Zackrissoncb1b5562018-09-28 14:15:09 +0200146 bool high_pass_filter_enabled,
peah2ace3f92016-09-10 04:42:27 -0700147 bool mobile_echo_controller_enabled,
ivoc9f4a4a02016-10-28 05:39:16 -0700148 bool residual_echo_detector_enabled,
peah2ace3f92016-09-10 04:42:27 -0700149 bool noise_suppressor_enabled,
peah2ace3f92016-09-10 04:42:27 -0700150 bool adaptive_gain_controller_enabled,
alessiob3ec96df2017-05-22 06:57:06 -0700151 bool gain_controller2_enabled,
Per Åhgrendb5d7282021-03-15 16:31:04 +0000152 bool gain_adjustment_enabled,
Gustaf Ullbergce045ac2017-10-16 13:49:04 +0200153 bool echo_controller_enabled,
Sam Zackrisson0824c6f2019-10-07 14:03:56 +0200154 bool voice_detector_enabled,
peah2ace3f92016-09-10 04:42:27 -0700155 bool transient_suppressor_enabled) {
156 bool changed = false;
Sam Zackrissoncb1b5562018-09-28 14:15:09 +0200157 changed |= (high_pass_filter_enabled != high_pass_filter_enabled_);
peah2ace3f92016-09-10 04:42:27 -0700158 changed |=
159 (mobile_echo_controller_enabled != mobile_echo_controller_enabled_);
ivoc9f4a4a02016-10-28 05:39:16 -0700160 changed |=
161 (residual_echo_detector_enabled != residual_echo_detector_enabled_);
peah2ace3f92016-09-10 04:42:27 -0700162 changed |= (noise_suppressor_enabled != noise_suppressor_enabled_);
163 changed |=
peah2ace3f92016-09-10 04:42:27 -0700164 (adaptive_gain_controller_enabled != adaptive_gain_controller_enabled_);
Per Åhgren6ee75fd2019-04-26 11:33:37 +0200165 changed |= (gain_controller2_enabled != gain_controller2_enabled_);
Per Åhgrendb5d7282021-03-15 16:31:04 +0000166 changed |= (gain_adjustment_enabled != gain_adjustment_enabled_);
Gustaf Ullbergce045ac2017-10-16 13:49:04 +0200167 changed |= (echo_controller_enabled != echo_controller_enabled_);
Sam Zackrisson0824c6f2019-10-07 14:03:56 +0200168 changed |= (voice_detector_enabled != voice_detector_enabled_);
peah2ace3f92016-09-10 04:42:27 -0700169 changed |= (transient_suppressor_enabled != transient_suppressor_enabled_);
170 if (changed) {
Sam Zackrissoncb1b5562018-09-28 14:15:09 +0200171 high_pass_filter_enabled_ = high_pass_filter_enabled;
peah2ace3f92016-09-10 04:42:27 -0700172 mobile_echo_controller_enabled_ = mobile_echo_controller_enabled;
ivoc9f4a4a02016-10-28 05:39:16 -0700173 residual_echo_detector_enabled_ = residual_echo_detector_enabled;
peah2ace3f92016-09-10 04:42:27 -0700174 noise_suppressor_enabled_ = noise_suppressor_enabled;
peah2ace3f92016-09-10 04:42:27 -0700175 adaptive_gain_controller_enabled_ = adaptive_gain_controller_enabled;
alessiob3ec96df2017-05-22 06:57:06 -0700176 gain_controller2_enabled_ = gain_controller2_enabled;
Per Åhgrendb5d7282021-03-15 16:31:04 +0000177 gain_adjustment_enabled_ = gain_adjustment_enabled;
Gustaf Ullbergce045ac2017-10-16 13:49:04 +0200178 echo_controller_enabled_ = echo_controller_enabled;
Sam Zackrisson0824c6f2019-10-07 14:03:56 +0200179 voice_detector_enabled_ = voice_detector_enabled;
peah2ace3f92016-09-10 04:42:27 -0700180 transient_suppressor_enabled_ = transient_suppressor_enabled;
181 }
182
183 changed |= first_update_;
184 first_update_ = false;
185 return changed;
186}
187
saza1d600522019-10-18 13:29:43 +0200188bool AudioProcessingImpl::SubmoduleStates::CaptureMultiBandSubModulesActive()
peah2ace3f92016-09-10 04:42:27 -0700189 const {
Gustaf Ullberg8675eee2019-10-09 13:34:36 +0200190 return CaptureMultiBandProcessingPresent() || voice_detector_enabled_;
peah2ace3f92016-09-10 04:42:27 -0700191}
192
saza1d600522019-10-18 13:29:43 +0200193bool AudioProcessingImpl::SubmoduleStates::CaptureMultiBandProcessingPresent()
194 const {
Gustaf Ullberg8675eee2019-10-09 13:34:36 +0200195 // If echo controller is present, assume it performs active processing.
196 return CaptureMultiBandProcessingActive(/*ec_processing_active=*/true);
197}
198
saza1d600522019-10-18 13:29:43 +0200199bool AudioProcessingImpl::SubmoduleStates::CaptureMultiBandProcessingActive(
Gustaf Ullberg8675eee2019-10-09 13:34:36 +0200200 bool ec_processing_active) const {
Per Åhgren62ea0aa2019-12-09 10:18:44 +0100201 return high_pass_filter_enabled_ || mobile_echo_controller_enabled_ ||
202 noise_suppressor_enabled_ || adaptive_gain_controller_enabled_ ||
Gustaf Ullberg8675eee2019-10-09 13:34:36 +0200203 (echo_controller_enabled_ && ec_processing_active);
peah2ace3f92016-09-10 04:42:27 -0700204}
205
saza1d600522019-10-18 13:29:43 +0200206bool AudioProcessingImpl::SubmoduleStates::CaptureFullBandProcessingActive()
peah23ac8b42017-05-23 05:33:56 -0700207 const {
Alex Loikob5c9a792018-04-16 16:31:22 +0200208 return gain_controller2_enabled_ || capture_post_processor_enabled_ ||
Per Åhgrendb5d7282021-03-15 16:31:04 +0000209 gain_adjustment_enabled_;
peah23ac8b42017-05-23 05:33:56 -0700210}
211
saza1d600522019-10-18 13:29:43 +0200212bool AudioProcessingImpl::SubmoduleStates::CaptureAnalyzerActive() const {
Valeriia Nemychnikovaf06eb572018-08-29 10:37:09 +0200213 return capture_analyzer_enabled_;
214}
215
saza1d600522019-10-18 13:29:43 +0200216bool AudioProcessingImpl::SubmoduleStates::RenderMultiBandSubModulesActive()
peah2ace3f92016-09-10 04:42:27 -0700217 const {
Per Åhgren62ea0aa2019-12-09 10:18:44 +0100218 return RenderMultiBandProcessingActive() || mobile_echo_controller_enabled_ ||
219 adaptive_gain_controller_enabled_ || echo_controller_enabled_;
peah2ace3f92016-09-10 04:42:27 -0700220}
221
saza1d600522019-10-18 13:29:43 +0200222bool AudioProcessingImpl::SubmoduleStates::RenderFullBandProcessingActive()
Alex Loiko5825aa62017-12-18 16:02:40 +0100223 const {
224 return render_pre_processor_enabled_;
225}
226
saza1d600522019-10-18 13:29:43 +0200227bool AudioProcessingImpl::SubmoduleStates::RenderMultiBandProcessingActive()
peah2ace3f92016-09-10 04:42:27 -0700228 const {
peah2ace3f92016-09-10 04:42:27 -0700229 return false;
peah2ace3f92016-09-10 04:42:27 -0700230}
231
saza1d600522019-10-18 13:29:43 +0200232bool AudioProcessingImpl::SubmoduleStates::HighPassFilteringRequired() const {
Per Åhgren62ea0aa2019-12-09 10:18:44 +0100233 return high_pass_filter_enabled_ || mobile_echo_controller_enabled_ ||
234 noise_suppressor_enabled_;
Sam Zackrissoncb1b5562018-09-28 14:15:09 +0200235}
236
Alessio Bazzicabe1b8982021-09-17 08:26:10 +0200237AudioProcessingImpl::AudioProcessingImpl()
238 : AudioProcessingImpl(/*capture_post_processor=*/nullptr,
Sam Zackrissonfeee1e42019-09-20 07:50:35 +0200239 /*render_pre_processor=*/nullptr,
240 /*echo_control_factory=*/nullptr,
241 /*echo_detector=*/nullptr,
242 /*capture_analyzer=*/nullptr) {}
aluebs@webrtc.orgd82f55d2015-01-15 18:07:21 +0000243
Per Åhgren13735822018-02-12 21:42:56 +0100244int AudioProcessingImpl::instance_count_ = 0;
245
Sam Zackrisson0beac582017-09-25 12:04:02 +0200246AudioProcessingImpl::AudioProcessingImpl(
Alex Loiko5825aa62017-12-18 16:02:40 +0100247 std::unique_ptr<CustomProcessing> capture_post_processor,
248 std::unique_ptr<CustomProcessing> render_pre_processor,
Gustaf Ullberg002ef282017-10-12 15:13:17 +0200249 std::unique_ptr<EchoControlFactory> echo_control_factory,
Valeriia Nemychnikovaf06eb572018-08-29 10:37:09 +0200250 rtc::scoped_refptr<EchoDetector> echo_detector,
251 std::unique_ptr<CustomAudioAnalyzer> capture_analyzer)
Per Åhgren13735822018-02-12 21:42:56 +0100252 : data_dumper_(
253 new ApmDataDumper(rtc::AtomicOps::Increment(&instance_count_))),
Per Åhgrenb2b58d82019-12-02 14:59:40 +0100254 use_setup_specific_default_aec3_config_(
255 UseSetupSpecificDefaultAec3Congfig()),
Alessio Bazzica0441bb62021-08-10 15:23:23 +0200256 use_denormal_disabler_(
257 !field_trial::IsEnabled("WebRTC-ApmDenormalDisablerKillSwitch")),
Per Åhgren652ada52021-03-03 10:52:44 +0000258 capture_runtime_settings_(RuntimeSettingQueueSize()),
259 render_runtime_settings_(RuntimeSettingQueueSize()),
Alex Loiko73ec0192018-05-15 10:52:28 +0200260 capture_runtime_settings_enqueuer_(&capture_runtime_settings_),
261 render_runtime_settings_enqueuer_(&render_runtime_settings_),
Gustaf Ullberg002ef282017-10-12 15:13:17 +0200262 echo_control_factory_(std::move(echo_control_factory)),
Valeriia Nemychnikovaf06eb572018-08-29 10:37:09 +0200263 submodule_states_(!!capture_post_processor,
264 !!render_pre_processor,
265 !!capture_analyzer),
saza1d600522019-10-18 13:29:43 +0200266 submodules_(std::move(capture_post_processor),
267 std::move(render_pre_processor),
268 std::move(echo_detector),
Per Åhgren3daedb62019-11-22 12:11:40 +0100269 std::move(capture_analyzer)),
Per Åhgren0695df12020-01-13 14:43:13 +0100270 constants_(!field_trial::IsEnabled(
Sam Zackrissonfeee1e42019-09-20 07:50:35 +0200271 "WebRTC-ApmExperimentalMultiChannelRenderKillSwitch"),
272 !field_trial::IsEnabled(
Per Åhgrenc0424252019-12-10 13:04:15 +0100273 "WebRTC-ApmExperimentalMultiChannelCaptureKillSwitch"),
Per Åhgren19775cb2021-03-12 23:08:09 +0000274 EnforceSplitBandHpf(),
Gustaf Ullberga399c822021-05-18 12:17:56 +0200275 MinimizeProcessingForUnusedOutput(),
276 field_trial::IsEnabled("WebRTC-TransientSuppressorForcedOff")),
Per Åhgren19775cb2021-03-12 23:08:09 +0000277 capture_(),
Alessio Bazzicacc22f512018-08-30 13:01:34 +0200278 capture_nonlocked_() {
Sam Zackrisson72cc71c2019-10-21 12:54:02 +0200279 RTC_LOG(LS_INFO) << "Injected APM submodules:"
Jonas Olssonb2b20312020-01-14 12:11:31 +0100280 "\nEcho control factory: "
281 << !!echo_control_factory_
Sam Zackrisson72cc71c2019-10-21 12:54:02 +0200282 << "\nEcho detector: " << !!submodules_.echo_detector
283 << "\nCapture analyzer: " << !!submodules_.capture_analyzer
284 << "\nCapture post processor: "
285 << !!submodules_.capture_post_processor
286 << "\nRender pre processor: "
287 << !!submodules_.render_pre_processor;
Alessio Bazzica0441bb62021-08-10 15:23:23 +0200288 RTC_LOG(LS_INFO) << "Denormal disabler: "
289 << (DenormalDisabler::IsSupported() ? "supported"
290 : "unsupported");
Sam Zackrisson72cc71c2019-10-21 12:54:02 +0200291
Sam Zackrisson421c8592019-02-11 13:39:46 +0100292 // Mark Echo Controller enabled if a factory is injected.
293 capture_nonlocked_.echo_controller_enabled =
294 static_cast<bool>(echo_control_factory_);
niklase@google.com470e71d2011-07-07 08:21:25 +0000295
Sam Zackrisson421c8592019-02-11 13:39:46 +0100296 // If no echo detector is injected, use the ResidualEchoDetector.
saza1d600522019-10-18 13:29:43 +0200297 if (!submodules_.echo_detector) {
Tommi87f70902021-04-27 14:43:08 +0200298 submodules_.echo_detector = rtc::make_ref_counted<ResidualEchoDetector>();
peahdf3efa82015-11-28 12:35:15 -0800299 }
pbos@webrtc.org788acd12014-12-15 09:41:24 +0000300
Per Åhgren0ade9832020-09-01 23:57:20 +0200301 Initialize();
niklase@google.com470e71d2011-07-07 08:21:25 +0000302}
303
Per Åhgren0e3198e2019-11-18 08:52:22 +0100304AudioProcessingImpl::~AudioProcessingImpl() = default;
niklase@google.com470e71d2011-07-07 08:21:25 +0000305
niklase@google.com470e71d2011-07-07 08:21:25 +0000306int AudioProcessingImpl::Initialize() {
peahdf3efa82015-11-28 12:35:15 -0800307 // Run in a single-threaded manner during initialization.
Markus Handell0df0fae2020-07-07 15:53:34 +0200308 MutexLock lock_render(&mutex_render_);
309 MutexLock lock_capture(&mutex_capture_);
Per Åhgren0ade9832020-09-01 23:57:20 +0200310 InitializeLocked();
311 return kNoError;
niklase@google.com470e71d2011-07-07 08:21:25 +0000312}
313
peahde65ddc2016-09-16 15:02:15 -0700314int AudioProcessingImpl::Initialize(int capture_input_sample_rate_hz,
315 int capture_output_sample_rate_hz,
316 int render_input_sample_rate_hz,
317 ChannelLayout capture_input_layout,
318 ChannelLayout capture_output_layout,
319 ChannelLayout render_input_layout) {
Michael Graczyk86c6d332015-07-23 11:41:39 -0700320 const ProcessingConfig processing_config = {
peahde65ddc2016-09-16 15:02:15 -0700321 {{capture_input_sample_rate_hz, ChannelsFromLayout(capture_input_layout),
322 LayoutHasKeyboard(capture_input_layout)},
323 {capture_output_sample_rate_hz,
324 ChannelsFromLayout(capture_output_layout),
325 LayoutHasKeyboard(capture_output_layout)},
326 {render_input_sample_rate_hz, ChannelsFromLayout(render_input_layout),
327 LayoutHasKeyboard(render_input_layout)},
328 {render_input_sample_rate_hz, ChannelsFromLayout(render_input_layout),
329 LayoutHasKeyboard(render_input_layout)}}};
Michael Graczyk86c6d332015-07-23 11:41:39 -0700330
331 return Initialize(processing_config);
332}
333
334int AudioProcessingImpl::Initialize(const ProcessingConfig& processing_config) {
peahdf3efa82015-11-28 12:35:15 -0800335 // Run in a single-threaded manner during initialization.
Markus Handell0df0fae2020-07-07 15:53:34 +0200336 MutexLock lock_render(&mutex_render_);
337 MutexLock lock_capture(&mutex_capture_);
Michael Graczyk86c6d332015-07-23 11:41:39 -0700338 return InitializeLocked(processing_config);
andrew@webrtc.orga8b97372014-03-10 22:26:12 +0000339}
340
peahdf3efa82015-11-28 12:35:15 -0800341int AudioProcessingImpl::MaybeInitializeRender(
peah81b9bfe2015-11-27 02:47:28 -0800342 const ProcessingConfig& processing_config) {
peahdf3efa82015-11-28 12:35:15 -0800343 // Called from both threads. Thread check is therefore not possible.
Oskar Sundbom4b276482019-05-23 14:28:00 +0200344 if (processing_config == formats_.api_format) {
peah192164e2015-11-17 02:16:45 -0800345 return kNoError;
346 }
peahdf3efa82015-11-28 12:35:15 -0800347
Markus Handell0df0fae2020-07-07 15:53:34 +0200348 MutexLock lock_capture(&mutex_capture_);
peah192164e2015-11-17 02:16:45 -0800349 return InitializeLocked(processing_config);
350}
351
Per Åhgren0ade9832020-09-01 23:57:20 +0200352void AudioProcessingImpl::InitializeLocked() {
Per Åhgren4bdced52017-06-27 16:00:38 +0200353 UpdateActiveSubmoduleStates();
354
Per Åhgrend47941e2019-08-22 11:51:13 +0200355 const int render_audiobuffer_sample_rate_hz =
peahdf3efa82015-11-28 12:35:15 -0800356 formats_.api_format.reverse_output_stream().num_frames() == 0
Per Åhgrend47941e2019-08-22 11:51:13 +0200357 ? formats_.render_processing_format.sample_rate_hz()
358 : formats_.api_format.reverse_output_stream().sample_rate_hz();
peahdf3efa82015-11-28 12:35:15 -0800359 if (formats_.api_format.reverse_input_stream().num_channels() > 0) {
360 render_.render_audio.reset(new AudioBuffer(
Per Åhgrend47941e2019-08-22 11:51:13 +0200361 formats_.api_format.reverse_input_stream().sample_rate_hz(),
peahdf3efa82015-11-28 12:35:15 -0800362 formats_.api_format.reverse_input_stream().num_channels(),
Per Åhgrend47941e2019-08-22 11:51:13 +0200363 formats_.render_processing_format.sample_rate_hz(),
peahde65ddc2016-09-16 15:02:15 -0700364 formats_.render_processing_format.num_channels(),
Per Åhgrend47941e2019-08-22 11:51:13 +0200365 render_audiobuffer_sample_rate_hz,
366 formats_.render_processing_format.num_channels()));
peah2ace3f92016-09-10 04:42:27 -0700367 if (formats_.api_format.reverse_input_stream() !=
368 formats_.api_format.reverse_output_stream()) {
kwibergc2b785d2016-02-24 05:22:32 -0800369 render_.render_converter = AudioConverter::Create(
peahdf3efa82015-11-28 12:35:15 -0800370 formats_.api_format.reverse_input_stream().num_channels(),
371 formats_.api_format.reverse_input_stream().num_frames(),
372 formats_.api_format.reverse_output_stream().num_channels(),
kwibergc2b785d2016-02-24 05:22:32 -0800373 formats_.api_format.reverse_output_stream().num_frames());
ekmeyerson60d9b332015-08-14 10:35:55 -0700374 } else {
peahdf3efa82015-11-28 12:35:15 -0800375 render_.render_converter.reset(nullptr);
ekmeyerson60d9b332015-08-14 10:35:55 -0700376 }
Michael Graczyk86c6d332015-07-23 11:41:39 -0700377 } else {
peahdf3efa82015-11-28 12:35:15 -0800378 render_.render_audio.reset(nullptr);
379 render_.render_converter.reset(nullptr);
Michael Graczyk86c6d332015-07-23 11:41:39 -0700380 }
peahce4d9152017-05-19 01:28:05 -0700381
Per Åhgrend47941e2019-08-22 11:51:13 +0200382 capture_.capture_audio.reset(new AudioBuffer(
383 formats_.api_format.input_stream().sample_rate_hz(),
384 formats_.api_format.input_stream().num_channels(),
385 capture_nonlocked_.capture_processing_format.sample_rate_hz(),
386 formats_.api_format.output_stream().num_channels(),
387 formats_.api_format.output_stream().sample_rate_hz(),
388 formats_.api_format.output_stream().num_channels()));
niklase@google.com470e71d2011-07-07 08:21:25 +0000389
Gustaf Ullberg422b9e02019-10-09 13:02:14 +0200390 if (capture_nonlocked_.capture_processing_format.sample_rate_hz() <
391 formats_.api_format.output_stream().sample_rate_hz() &&
392 formats_.api_format.output_stream().sample_rate_hz() == 48000) {
393 capture_.capture_fullband_audio.reset(
394 new AudioBuffer(formats_.api_format.input_stream().sample_rate_hz(),
395 formats_.api_format.input_stream().num_channels(),
396 formats_.api_format.output_stream().sample_rate_hz(),
397 formats_.api_format.output_stream().num_channels(),
398 formats_.api_format.output_stream().sample_rate_hz(),
399 formats_.api_format.output_stream().num_channels()));
400 } else {
401 capture_.capture_fullband_audio.reset();
402 }
403
peah764e3642016-10-22 05:04:30 -0700404 AllocateRenderQueue();
405
Per Åhgren0695df12020-01-13 14:43:13 +0100406 InitializeGainController1();
Per Åhgrenc0734712020-01-02 15:15:36 +0100407 InitializeTransientSuppressor();
Per Åhgren0f14db22020-01-03 14:27:14 +0100408 InitializeHighPassFilter(true);
Sam Zackrisson0824c6f2019-10-07 14:03:56 +0200409 InitializeVoiceDetector();
ivoc9f4a4a02016-10-28 05:39:16 -0700410 InitializeResidualEchoDetector();
Gustaf Ullberg8eb9c7d2017-10-14 08:28:46 +0200411 InitializeEchoController();
alessiob3ec96df2017-05-22 06:57:06 -0700412 InitializeGainController2();
saza0bad15f2019-10-16 11:46:11 +0200413 InitializeNoiseSuppressor();
Valeriia Nemychnikovaf06eb572018-08-29 10:37:09 +0200414 InitializeAnalyzer();
Sam Zackrisson0beac582017-09-25 12:04:02 +0200415 InitializePostProcessor();
Alex Loiko5825aa62017-12-18 16:02:40 +0100416 InitializePreProcessor();
Per Åhgrendb5d7282021-03-15 16:31:04 +0000417 InitializeCaptureLevelsAdjuster();
solenberg70f99032015-12-08 11:07:32 -0800418
aleloi868f32f2017-05-23 07:20:05 -0700419 if (aec_dump_) {
Minyue Li656d6092018-08-10 15:38:52 +0200420 aec_dump_->WriteInitMessage(formats_.api_format, rtc::TimeUTCMillis());
aleloi868f32f2017-05-23 07:20:05 -0700421 }
niklase@google.com470e71d2011-07-07 08:21:25 +0000422}
423
Michael Graczyk86c6d332015-07-23 11:41:39 -0700424int AudioProcessingImpl::InitializeLocked(const ProcessingConfig& config) {
Per Åhgren4bdced52017-06-27 16:00:38 +0200425 UpdateActiveSubmoduleStates();
426
Michael Graczyk86c6d332015-07-23 11:41:39 -0700427 for (const auto& stream : config.streams) {
Michael Graczyk86c6d332015-07-23 11:41:39 -0700428 if (stream.num_channels() > 0 && stream.sample_rate_hz() <= 0) {
429 return kBadSampleRateError;
430 }
andrew@webrtc.orga8b97372014-03-10 22:26:12 +0000431 }
Michael Graczyk86c6d332015-07-23 11:41:39 -0700432
Peter Kasting69558702016-01-12 16:26:35 -0800433 const size_t num_in_channels = config.input_stream().num_channels();
434 const size_t num_out_channels = config.output_stream().num_channels();
Michael Graczyk86c6d332015-07-23 11:41:39 -0700435
436 // Need at least one input channel.
437 // Need either one output channel or as many outputs as there are inputs.
438 if (num_in_channels == 0 ||
439 !(num_out_channels == 1 || num_out_channels == num_in_channels)) {
Michael Graczykc2047542015-07-22 21:06:11 -0700440 return kBadNumberChannelsError;
441 }
442
peahdf3efa82015-11-28 12:35:15 -0800443 formats_.api_format = config;
andrew@webrtc.orgddbb8a22014-04-22 21:00:04 +0000444
Per Åhgrenfcbe4072019-09-15 00:27:58 +0200445 // Choose maximum rate to use for the split filtering.
446 RTC_DCHECK(config_.pipeline.maximum_internal_processing_rate == 48000 ||
447 config_.pipeline.maximum_internal_processing_rate == 32000);
448 int max_splitting_rate = 48000;
449 if (config_.pipeline.maximum_internal_processing_rate == 32000) {
450 max_splitting_rate = config_.pipeline.maximum_internal_processing_rate;
451 }
452
Per Åhgrenc8626b62019-08-23 15:49:51 +0200453 int capture_processing_rate = SuitableProcessRate(
peah423d2362016-04-09 16:06:52 -0700454 std::min(formats_.api_format.input_stream().sample_rate_hz(),
peah2ace3f92016-09-10 04:42:27 -0700455 formats_.api_format.output_stream().sample_rate_hz()),
Per Åhgrenfcbe4072019-09-15 00:27:58 +0200456 max_splitting_rate,
peah2ace3f92016-09-10 04:42:27 -0700457 submodule_states_.CaptureMultiBandSubModulesActive() ||
458 submodule_states_.RenderMultiBandSubModulesActive());
Per Åhgrenc8626b62019-08-23 15:49:51 +0200459 RTC_DCHECK_NE(8000, capture_processing_rate);
andrew@webrtc.orga8b97372014-03-10 22:26:12 +0000460
peahde65ddc2016-09-16 15:02:15 -0700461 capture_nonlocked_.capture_processing_format =
462 StreamConfig(capture_processing_rate);
peah2ace3f92016-09-10 04:42:27 -0700463
peah2ce640f2017-04-07 03:57:48 -0700464 int render_processing_rate;
Gustaf Ullbergbd83b912017-10-18 12:32:42 +0200465 if (!capture_nonlocked_.echo_controller_enabled) {
Per Åhgrenc8626b62019-08-23 15:49:51 +0200466 render_processing_rate = SuitableProcessRate(
peah2ce640f2017-04-07 03:57:48 -0700467 std::min(formats_.api_format.reverse_input_stream().sample_rate_hz(),
468 formats_.api_format.reverse_output_stream().sample_rate_hz()),
Per Åhgrenfcbe4072019-09-15 00:27:58 +0200469 max_splitting_rate,
peah2ce640f2017-04-07 03:57:48 -0700470 submodule_states_.CaptureMultiBandSubModulesActive() ||
471 submodule_states_.RenderMultiBandSubModulesActive());
472 } else {
473 render_processing_rate = capture_processing_rate;
474 }
475
peahde65ddc2016-09-16 15:02:15 -0700476 // If the forward sample rate is 8 kHz, the render stream is also processed
aluebseb3603b2016-04-20 15:27:58 -0700477 // at this rate.
peahde65ddc2016-09-16 15:02:15 -0700478 if (capture_nonlocked_.capture_processing_format.sample_rate_hz() ==
479 kSampleRate8kHz) {
480 render_processing_rate = kSampleRate8kHz;
andrew@webrtc.orga8b97372014-03-10 22:26:12 +0000481 } else {
peahde65ddc2016-09-16 15:02:15 -0700482 render_processing_rate =
483 std::max(render_processing_rate, static_cast<int>(kSampleRate16kHz));
andrew@webrtc.orgddbb8a22014-04-22 21:00:04 +0000484 }
485
Per Åhgrenc8626b62019-08-23 15:49:51 +0200486 RTC_DCHECK_NE(8000, render_processing_rate);
487
peahce4d9152017-05-19 01:28:05 -0700488 if (submodule_states_.RenderMultiBandSubModulesActive()) {
Sam Zackrissonfeee1e42019-09-20 07:50:35 +0200489 // By default, downmix the render stream to mono for analysis. This has been
490 // demonstrated to work well for AEC in most practical scenarios.
Per Åhgrene14cb992019-11-27 09:34:22 +0100491 const bool multi_channel_render = config_.pipeline.multi_channel_render &&
492 constants_.multi_channel_render_support;
Sam Zackrissonfeee1e42019-09-20 07:50:35 +0200493 int render_processing_num_channels =
Per Åhgrene14cb992019-11-27 09:34:22 +0100494 multi_channel_render
Sam Zackrissonfeee1e42019-09-20 07:50:35 +0200495 ? formats_.api_format.reverse_input_stream().num_channels()
496 : 1;
497 formats_.render_processing_format =
498 StreamConfig(render_processing_rate, render_processing_num_channels);
peahce4d9152017-05-19 01:28:05 -0700499 } else {
500 formats_.render_processing_format = StreamConfig(
501 formats_.api_format.reverse_input_stream().sample_rate_hz(),
502 formats_.api_format.reverse_input_stream().num_channels());
503 }
andrew@webrtc.orgddbb8a22014-04-22 21:00:04 +0000504
peahde65ddc2016-09-16 15:02:15 -0700505 if (capture_nonlocked_.capture_processing_format.sample_rate_hz() ==
506 kSampleRate32kHz ||
507 capture_nonlocked_.capture_processing_format.sample_rate_hz() ==
508 kSampleRate48kHz) {
peahdf3efa82015-11-28 12:35:15 -0800509 capture_nonlocked_.split_rate = kSampleRate16kHz;
andrew@webrtc.orgddbb8a22014-04-22 21:00:04 +0000510 } else {
peahdf3efa82015-11-28 12:35:15 -0800511 capture_nonlocked_.split_rate =
peahde65ddc2016-09-16 15:02:15 -0700512 capture_nonlocked_.capture_processing_format.sample_rate_hz();
andrew@webrtc.orga8b97372014-03-10 22:26:12 +0000513 }
514
Per Åhgren0ade9832020-09-01 23:57:20 +0200515 InitializeLocked();
516 return kNoError;
andrew@webrtc.orga8b97372014-03-10 22:26:12 +0000517}
518
peah88ac8532016-09-12 16:47:25 -0700519void AudioProcessingImpl::ApplyConfig(const AudioProcessing::Config& config) {
Sam Zackrisson72cc71c2019-10-21 12:54:02 +0200520 RTC_LOG(LS_INFO) << "AudioProcessing::ApplyConfig: " << config.ToString();
521
peah88ac8532016-09-12 16:47:25 -0700522 // Run in a single-threaded manner when applying the settings.
Markus Handell0df0fae2020-07-07 15:53:34 +0200523 MutexLock lock_render(&mutex_render_);
524 MutexLock lock_capture(&mutex_capture_);
peah88ac8532016-09-12 16:47:25 -0700525
Sam Zackrissonfeee1e42019-09-20 07:50:35 +0200526 const bool pipeline_config_changed =
Per Åhgrene14cb992019-11-27 09:34:22 +0100527 config_.pipeline.multi_channel_render !=
528 config.pipeline.multi_channel_render ||
529 config_.pipeline.multi_channel_capture !=
Per Åhgrenc0424252019-12-10 13:04:15 +0100530 config.pipeline.multi_channel_capture ||
531 config_.pipeline.maximum_internal_processing_rate !=
532 config.pipeline.maximum_internal_processing_rate;
Sam Zackrissonfeee1e42019-09-20 07:50:35 +0200533
Per Åhgren200feba2019-03-06 04:16:46 +0100534 const bool aec_config_changed =
535 config_.echo_canceller.enabled != config.echo_canceller.enabled ||
Per Åhgren62ea0aa2019-12-09 10:18:44 +0100536 config_.echo_canceller.mobile_mode != config.echo_canceller.mobile_mode;
Per Åhgren200feba2019-03-06 04:16:46 +0100537
Sam Zackrissonf0d1c032019-03-27 13:28:08 +0100538 const bool agc1_config_changed =
Alessio Bazzica3438a932020-10-14 12:47:50 +0200539 config_.gain_controller1 != config.gain_controller1;
Sam Zackrissonf0d1c032019-03-27 13:28:08 +0100540
Per Åhgren2bd85ab2020-01-03 10:36:34 +0100541 const bool agc2_config_changed =
Alessio Bazzica3438a932020-10-14 12:47:50 +0200542 config_.gain_controller2 != config.gain_controller2;
Per Åhgren2bd85ab2020-01-03 10:36:34 +0100543
Sam Zackrisson0824c6f2019-10-07 14:03:56 +0200544 const bool voice_detection_config_changed =
545 config_.voice_detection.enabled != config.voice_detection.enabled;
546
saza0bad15f2019-10-16 11:46:11 +0200547 const bool ns_config_changed =
548 config_.noise_suppression.enabled != config.noise_suppression.enabled ||
549 config_.noise_suppression.level != config.noise_suppression.level;
550
Per Åhgrenc0734712020-01-02 15:15:36 +0100551 const bool ts_config_changed = config_.transient_suppression.enabled !=
552 config.transient_suppression.enabled;
553
Per Åhgren0f14db22020-01-03 14:27:14 +0100554 const bool pre_amplifier_config_changed =
555 config_.pre_amplifier.enabled != config.pre_amplifier.enabled ||
556 config_.pre_amplifier.fixed_gain_factor !=
557 config.pre_amplifier.fixed_gain_factor;
558
Per Åhgrendb5d7282021-03-15 16:31:04 +0000559 const bool gain_adjustment_config_changed =
560 config_.capture_level_adjustment != config.capture_level_adjustment;
561
Yves Gerey499bc6c2018-10-10 18:29:07 +0200562 config_ = config;
563
Per Åhgren200feba2019-03-06 04:16:46 +0100564 if (aec_config_changed) {
565 InitializeEchoController();
566 }
Sam Zackrissoncdf0e6d2018-09-17 11:05:17 +0200567
saza0bad15f2019-10-16 11:46:11 +0200568 if (ns_config_changed) {
569 InitializeNoiseSuppressor();
570 }
Sam Zackrisson23513132019-01-11 15:10:32 +0100571
Per Åhgrenc0734712020-01-02 15:15:36 +0100572 if (ts_config_changed) {
573 InitializeTransientSuppressor();
574 }
575
Per Åhgren0f14db22020-01-03 14:27:14 +0100576 InitializeHighPassFilter(false);
peah8271d042016-11-22 07:24:52 -0800577
Sam Zackrissonf0d1c032019-03-27 13:28:08 +0100578 if (agc1_config_changed) {
Per Åhgren0695df12020-01-13 14:43:13 +0100579 InitializeGainController1();
Sam Zackrissonf0d1c032019-03-27 13:28:08 +0100580 }
581
Sam Zackrissonab1aee02018-03-05 15:59:06 +0100582 const bool config_ok = GainController2::Validate(config_.gain_controller2);
alessiob3ec96df2017-05-22 06:57:06 -0700583 if (!config_ok) {
Alessio Bazzica0c83e152020-10-14 12:49:54 +0200584 RTC_LOG(LS_ERROR)
585 << "Invalid Gain Controller 2 config; using the default config.";
alessiob3ec96df2017-05-22 06:57:06 -0700586 config_.gain_controller2 = AudioProcessing::Config::GainController2();
587 }
Per Åhgren0f14db22020-01-03 14:27:14 +0100588
Per Åhgren2bd85ab2020-01-03 10:36:34 +0100589 if (agc2_config_changed) {
590 InitializeGainController2();
591 }
Per Åhgren0f14db22020-01-03 14:27:14 +0100592
Per Åhgrendb5d7282021-03-15 16:31:04 +0000593 if (pre_amplifier_config_changed || gain_adjustment_config_changed) {
594 InitializeCaptureLevelsAdjuster();
Per Åhgren0f14db22020-01-03 14:27:14 +0100595 }
Sam Zackrissonb24c00f2018-11-26 16:18:25 +0100596
saza1d600522019-10-18 13:29:43 +0200597 if (config_.level_estimation.enabled && !submodules_.output_level_estimator) {
598 submodules_.output_level_estimator = std::make_unique<LevelEstimator>();
Sam Zackrissonb24c00f2018-11-26 16:18:25 +0100599 }
Sam Zackrisson4db667b2018-12-21 16:29:27 +0100600
Sam Zackrisson0824c6f2019-10-07 14:03:56 +0200601 if (voice_detection_config_changed) {
602 InitializeVoiceDetector();
Sam Zackrisson4db667b2018-12-21 16:29:27 +0100603 }
Sam Zackrissonfeee1e42019-09-20 07:50:35 +0200604
605 // Reinitialization must happen after all submodule configuration to avoid
606 // additional reinitializations on the next capture / render processing call.
607 if (pipeline_config_changed) {
608 InitializeLocked(formats_.api_format);
609 }
peah88ac8532016-09-12 16:47:25 -0700610}
611
Sam Zackrissonb37e59d2020-04-27 08:39:33 +0200612void AudioProcessingImpl::OverrideSubmoduleCreationForTesting(
613 const ApmSubmoduleCreationOverrides& overrides) {
Markus Handell0df0fae2020-07-07 15:53:34 +0200614 MutexLock lock(&mutex_capture_);
Sam Zackrissonb37e59d2020-04-27 08:39:33 +0200615 submodule_creation_overrides_ = overrides;
616}
617
andrew@webrtc.orgddbb8a22014-04-22 21:00:04 +0000618int AudioProcessingImpl::proc_sample_rate_hz() const {
peahdf3efa82015-11-28 12:35:15 -0800619 // Used as callback from submodules, hence locking is not allowed.
peahde65ddc2016-09-16 15:02:15 -0700620 return capture_nonlocked_.capture_processing_format.sample_rate_hz();
niklase@google.com470e71d2011-07-07 08:21:25 +0000621}
622
Gustaf Ullberg422b9e02019-10-09 13:02:14 +0200623int AudioProcessingImpl::proc_fullband_sample_rate_hz() const {
624 return capture_.capture_fullband_audio
625 ? capture_.capture_fullband_audio->num_frames() * 100
626 : capture_nonlocked_.capture_processing_format.sample_rate_hz();
627}
628
andrew@webrtc.orgddbb8a22014-04-22 21:00:04 +0000629int AudioProcessingImpl::proc_split_sample_rate_hz() const {
peahdf3efa82015-11-28 12:35:15 -0800630 // Used as callback from submodules, hence locking is not allowed.
631 return capture_nonlocked_.split_rate;
niklase@google.com470e71d2011-07-07 08:21:25 +0000632}
633
Peter Kasting69558702016-01-12 16:26:35 -0800634size_t AudioProcessingImpl::num_reverse_channels() const {
peahdf3efa82015-11-28 12:35:15 -0800635 // Used as callback from submodules, hence locking is not allowed.
peahde65ddc2016-09-16 15:02:15 -0700636 return formats_.render_processing_format.num_channels();
niklase@google.com470e71d2011-07-07 08:21:25 +0000637}
638
Peter Kasting69558702016-01-12 16:26:35 -0800639size_t AudioProcessingImpl::num_input_channels() const {
peahdf3efa82015-11-28 12:35:15 -0800640 // Used as callback from submodules, hence locking is not allowed.
641 return formats_.api_format.input_stream().num_channels();
niklase@google.com470e71d2011-07-07 08:21:25 +0000642}
643
Peter Kasting69558702016-01-12 16:26:35 -0800644size_t AudioProcessingImpl::num_proc_channels() const {
aluebsb2328d12016-01-11 20:32:29 -0800645 // Used as callback from submodules, hence locking is not allowed.
Per Åhgrene14cb992019-11-27 09:34:22 +0100646 const bool multi_channel_capture = config_.pipeline.multi_channel_capture &&
647 constants_.multi_channel_capture_support;
648 if (capture_nonlocked_.echo_controller_enabled && !multi_channel_capture) {
Sam Zackrissonfeee1e42019-09-20 07:50:35 +0200649 return 1;
650 }
651 return num_output_channels();
aluebsb2328d12016-01-11 20:32:29 -0800652}
653
Peter Kasting69558702016-01-12 16:26:35 -0800654size_t AudioProcessingImpl::num_output_channels() const {
peahdf3efa82015-11-28 12:35:15 -0800655 // Used as callback from submodules, hence locking is not allowed.
656 return formats_.api_format.output_stream().num_channels();
niklase@google.com470e71d2011-07-07 08:21:25 +0000657}
658
andrew@webrtc.org17342e52014-02-12 22:28:31 +0000659void AudioProcessingImpl::set_output_will_be_muted(bool muted) {
Markus Handell0df0fae2020-07-07 15:53:34 +0200660 MutexLock lock(&mutex_capture_);
Per Åhgren0a144a72021-02-09 08:47:51 +0100661 HandleCaptureOutputUsedSetting(!muted);
662}
663
664void AudioProcessingImpl::HandleCaptureOutputUsedSetting(
665 bool capture_output_used) {
Per Åhgren19775cb2021-03-12 23:08:09 +0000666 capture_.capture_output_used =
667 capture_output_used || !constants_.minimize_processing_for_unused_output;
668
saza1d600522019-10-18 13:29:43 +0200669 if (submodules_.agc_manager.get()) {
Per Åhgren0a144a72021-02-09 08:47:51 +0100670 submodules_.agc_manager->HandleCaptureOutputUsedChange(
671 capture_.capture_output_used);
pbos@webrtc.org788acd12014-12-15 09:41:24 +0000672 }
Per Åhgren652ada52021-03-03 10:52:44 +0000673 if (submodules_.echo_controller) {
674 submodules_.echo_controller->SetCaptureOutputUsage(
675 capture_.capture_output_used);
676 }
Per Åhgren15179a92021-03-12 14:12:44 +0000677 if (submodules_.noise_suppressor) {
678 submodules_.noise_suppressor->SetCaptureOutputUsage(
679 capture_.capture_output_used);
680 }
andrew@webrtc.org17342e52014-02-12 22:28:31 +0000681}
682
Alessio Bazzicac054e782018-04-16 12:10:09 +0200683void AudioProcessingImpl::SetRuntimeSetting(RuntimeSetting setting) {
Per Åhgren0a144a72021-02-09 08:47:51 +0100684 PostRuntimeSetting(setting);
685}
686
687bool AudioProcessingImpl::PostRuntimeSetting(RuntimeSetting setting) {
Alex Loiko73ec0192018-05-15 10:52:28 +0200688 switch (setting.type()) {
689 case RuntimeSetting::Type::kCustomRenderProcessingRuntimeSetting:
Alessio Bazzica7c19a702019-11-07 13:22:00 +0100690 case RuntimeSetting::Type::kPlayoutAudioDeviceChange:
Per Åhgren0a144a72021-02-09 08:47:51 +0100691 return render_runtime_settings_enqueuer_.Enqueue(setting);
Alex Loiko73ec0192018-05-15 10:52:28 +0200692 case RuntimeSetting::Type::kCapturePreGain:
Per Åhgrendb5d7282021-03-15 16:31:04 +0000693 case RuntimeSetting::Type::kCapturePostGain:
Sam Zackrissonf0d1c032019-03-27 13:28:08 +0100694 case RuntimeSetting::Type::kCaptureCompressionGain:
Per Åhgren6ee75fd2019-04-26 11:33:37 +0200695 case RuntimeSetting::Type::kCaptureFixedPostGain:
Per Åhgren552d3e32020-08-12 08:46:47 +0200696 case RuntimeSetting::Type::kCaptureOutputUsed:
Per Åhgren0a144a72021-02-09 08:47:51 +0100697 return capture_runtime_settings_enqueuer_.Enqueue(setting);
698 case RuntimeSetting::Type::kPlayoutVolumeChange: {
699 bool enqueueing_successful;
700 enqueueing_successful =
701 capture_runtime_settings_enqueuer_.Enqueue(setting);
702 enqueueing_successful =
703 render_runtime_settings_enqueuer_.Enqueue(setting) &&
704 enqueueing_successful;
705 return enqueueing_successful;
706 }
Alessio Bazzica7c19a702019-11-07 13:22:00 +0100707 case RuntimeSetting::Type::kNotSpecified:
708 RTC_NOTREACHED();
Per Åhgren0a144a72021-02-09 08:47:51 +0100709 return true;
Alex Loiko73ec0192018-05-15 10:52:28 +0200710 }
711 // The language allows the enum to have a non-enumerator
712 // value. Check that this doesn't happen.
713 RTC_NOTREACHED();
Per Åhgren0a144a72021-02-09 08:47:51 +0100714 return true;
Alessio Bazzicac054e782018-04-16 12:10:09 +0200715}
716
717AudioProcessingImpl::RuntimeSettingEnqueuer::RuntimeSettingEnqueuer(
718 SwapQueue<RuntimeSetting>* runtime_settings)
Alessio Bazzica33444dc2018-04-20 13:16:55 +0200719 : runtime_settings_(*runtime_settings) {
720 RTC_DCHECK(runtime_settings);
Alessio Bazzicac054e782018-04-16 12:10:09 +0200721}
722
723AudioProcessingImpl::RuntimeSettingEnqueuer::~RuntimeSettingEnqueuer() =
724 default;
725
Per Åhgren0a144a72021-02-09 08:47:51 +0100726bool AudioProcessingImpl::RuntimeSettingEnqueuer::Enqueue(
Alessio Bazzicac054e782018-04-16 12:10:09 +0200727 RuntimeSetting setting) {
Per Åhgren652ada52021-03-03 10:52:44 +0000728 const bool successful_insert = runtime_settings_.Insert(&setting);
729
730 if (!successful_insert) {
Alessio Bazzica0b10f462020-10-23 12:40:30 +0200731 RTC_HISTOGRAM_BOOLEAN("WebRTC.Audio.ApmRuntimeSettingCannotEnqueue", 1);
Alessio Bazzicac054e782018-04-16 12:10:09 +0200732 RTC_LOG(LS_ERROR) << "Cannot enqueue a new runtime setting.";
Alessio Bazzica0b10f462020-10-23 12:40:30 +0200733 }
Per Åhgren652ada52021-03-03 10:52:44 +0000734 return successful_insert;
Alessio Bazzicac054e782018-04-16 12:10:09 +0200735}
andrew@webrtc.org17342e52014-02-12 22:28:31 +0000736
Sam Zackrisson12e319a2020-01-03 14:54:20 +0100737int AudioProcessingImpl::MaybeInitializeCapture(
738 const StreamConfig& input_config,
739 const StreamConfig& output_config) {
peahdf3efa82015-11-28 12:35:15 -0800740 ProcessingConfig processing_config;
peah2ace3f92016-09-10 04:42:27 -0700741 bool reinitialization_required = false;
peahdf3efa82015-11-28 12:35:15 -0800742 {
Sam Zackrisson12e319a2020-01-03 14:54:20 +0100743 // Acquire the capture lock in order to access api_format. The lock is
744 // released immediately, as we may need to acquire the render lock as part
745 // of the conditional reinitialization.
Markus Handell0df0fae2020-07-07 15:53:34 +0200746 MutexLock lock_capture(&mutex_capture_);
peahdf3efa82015-11-28 12:35:15 -0800747 processing_config = formats_.api_format;
peah2ace3f92016-09-10 04:42:27 -0700748 reinitialization_required = UpdateActiveSubmoduleStates();
niklase@google.com470e71d2011-07-07 08:21:25 +0000749 }
andrew@webrtc.org17e40642014-03-04 20:58:13 +0000750
Oskar Sundbom4b276482019-05-23 14:28:00 +0200751 if (processing_config.input_stream() != input_config) {
752 processing_config.input_stream() = input_config;
753 reinitialization_required = true;
peahdf3efa82015-11-28 12:35:15 -0800754 }
Oskar Sundbom4b276482019-05-23 14:28:00 +0200755
756 if (processing_config.output_stream() != output_config) {
757 processing_config.output_stream() = output_config;
758 reinitialization_required = true;
759 }
760
761 if (reinitialization_required) {
Markus Handell0df0fae2020-07-07 15:53:34 +0200762 MutexLock lock_render(&mutex_render_);
763 MutexLock lock_capture(&mutex_capture_);
Oskar Sundbom4b276482019-05-23 14:28:00 +0200764 RETURN_ON_ERR(InitializeLocked(processing_config));
765 }
Sam Zackrisson12e319a2020-01-03 14:54:20 +0100766 return kNoError;
767}
768
769int AudioProcessingImpl::ProcessStream(const float* const* src,
770 const StreamConfig& input_config,
771 const StreamConfig& output_config,
772 float* const* dest) {
773 TRACE_EVENT0("webrtc", "AudioProcessing::ProcessStream_StreamConfig");
774 if (!src || !dest) {
775 return kNullPointerError;
776 }
777
778 RETURN_ON_ERR(MaybeInitializeCapture(input_config, output_config));
Oskar Sundbom4b276482019-05-23 14:28:00 +0200779
Markus Handell0df0fae2020-07-07 15:53:34 +0200780 MutexLock lock_capture(&mutex_capture_);
Alessio Bazzica0441bb62021-08-10 15:23:23 +0200781 DenormalDisabler denormal_disabler(use_denormal_disabler_);
andrew@webrtc.org17e40642014-03-04 20:58:13 +0000782
aleloi868f32f2017-05-23 07:20:05 -0700783 if (aec_dump_) {
784 RecordUnprocessedCaptureStream(src);
785 }
786
Per Åhgrena1351272019-08-15 12:15:46 +0200787 capture_.keyboard_info.Extract(src, formats_.api_format.input_stream());
peahdf3efa82015-11-28 12:35:15 -0800788 capture_.capture_audio->CopyFrom(src, formats_.api_format.input_stream());
Gustaf Ullberg8675eee2019-10-09 13:34:36 +0200789 if (capture_.capture_fullband_audio) {
790 capture_.capture_fullband_audio->CopyFrom(
791 src, formats_.api_format.input_stream());
792 }
peahde65ddc2016-09-16 15:02:15 -0700793 RETURN_ON_ERR(ProcessCaptureStreamLocked());
Gustaf Ullberg422b9e02019-10-09 13:02:14 +0200794 if (capture_.capture_fullband_audio) {
795 capture_.capture_fullband_audio->CopyTo(formats_.api_format.output_stream(),
796 dest);
797 } else {
798 capture_.capture_audio->CopyTo(formats_.api_format.output_stream(), dest);
799 }
andrew@webrtc.org17e40642014-03-04 20:58:13 +0000800
aleloi868f32f2017-05-23 07:20:05 -0700801 if (aec_dump_) {
802 RecordProcessedCaptureStream(dest);
803 }
andrew@webrtc.org17e40642014-03-04 20:58:13 +0000804 return kNoError;
805}
806
Alex Loiko73ec0192018-05-15 10:52:28 +0200807void AudioProcessingImpl::HandleCaptureRuntimeSettings() {
Alessio Bazzicac054e782018-04-16 12:10:09 +0200808 RuntimeSetting setting;
Per Åhgren652ada52021-03-03 10:52:44 +0000809 int num_settings_processed = 0;
Alex Loiko73ec0192018-05-15 10:52:28 +0200810 while (capture_runtime_settings_.Remove(&setting)) {
Alex Loiko62347222018-09-10 10:18:07 +0200811 if (aec_dump_) {
812 aec_dump_->WriteRuntimeSetting(setting);
813 }
Alessio Bazzicac054e782018-04-16 12:10:09 +0200814 switch (setting.type()) {
815 case RuntimeSetting::Type::kCapturePreGain:
Per Åhgrendb5d7282021-03-15 16:31:04 +0000816 if (config_.pre_amplifier.enabled ||
817 config_.capture_level_adjustment.enabled) {
Alex Loikob5c9a792018-04-16 16:31:22 +0200818 float value;
819 setting.GetFloat(&value);
Per Åhgrendb5d7282021-03-15 16:31:04 +0000820 // If the pre-amplifier is used, apply the new gain to the
821 // pre-amplifier regardless if the capture level adjustment is
822 // activated. This approach allows both functionalities to coexist
823 // until they have been properly merged.
824 if (config_.pre_amplifier.enabled) {
825 config_.pre_amplifier.fixed_gain_factor = value;
826 } else {
827 config_.capture_level_adjustment.pre_gain_factor = value;
828 }
829
830 // Use both the pre-amplifier and the capture level adjustment gains
831 // as pre-gains.
832 float gain = 1.f;
833 if (config_.pre_amplifier.enabled) {
834 gain *= config_.pre_amplifier.fixed_gain_factor;
835 }
836 if (config_.capture_level_adjustment.enabled) {
837 gain *= config_.capture_level_adjustment.pre_gain_factor;
838 }
839
840 submodules_.capture_levels_adjuster->SetPreGain(gain);
841 }
842 // TODO(bugs.chromium.org/9138): Log setting handling by Aec Dump.
843 break;
844 case RuntimeSetting::Type::kCapturePostGain:
845 if (config_.capture_level_adjustment.enabled) {
846 float value;
847 setting.GetFloat(&value);
848 config_.capture_level_adjustment.post_gain_factor = value;
849 submodules_.capture_levels_adjuster->SetPostGain(
850 config_.capture_level_adjustment.post_gain_factor);
Alex Loikob5c9a792018-04-16 16:31:22 +0200851 }
852 // TODO(bugs.chromium.org/9138): Log setting handling by Aec Dump.
Alessio Bazzicac054e782018-04-16 12:10:09 +0200853 break;
Sam Zackrissonf0d1c032019-03-27 13:28:08 +0100854 case RuntimeSetting::Type::kCaptureCompressionGain: {
Per Åhgren0e3198e2019-11-18 08:52:22 +0100855 if (!submodules_.agc_manager) {
856 float value;
857 setting.GetFloat(&value);
858 int int_value = static_cast<int>(value + .5f);
859 config_.gain_controller1.compression_gain_db = int_value;
Per Åhgren0695df12020-01-13 14:43:13 +0100860 if (submodules_.gain_control) {
861 int error =
862 submodules_.gain_control->set_compression_gain_db(int_value);
863 RTC_DCHECK_EQ(kNoError, error);
864 }
Per Åhgren0e3198e2019-11-18 08:52:22 +0100865 }
Sam Zackrissonf0d1c032019-03-27 13:28:08 +0100866 break;
867 }
Per Åhgren6ee75fd2019-04-26 11:33:37 +0200868 case RuntimeSetting::Type::kCaptureFixedPostGain: {
Per Åhgren2bd85ab2020-01-03 10:36:34 +0100869 if (submodules_.gain_controller2) {
Per Åhgren6ee75fd2019-04-26 11:33:37 +0200870 float value;
871 setting.GetFloat(&value);
872 config_.gain_controller2.fixed_digital.gain_db = value;
saza1d600522019-10-18 13:29:43 +0200873 submodules_.gain_controller2->ApplyConfig(config_.gain_controller2);
Per Åhgren6ee75fd2019-04-26 11:33:37 +0200874 }
875 break;
876 }
Fredrik Hernqvistca362852019-05-10 15:50:02 +0200877 case RuntimeSetting::Type::kPlayoutVolumeChange: {
878 int value;
879 setting.GetInt(&value);
880 capture_.playout_volume = value;
881 break;
882 }
Alessio Bazzica7c19a702019-11-07 13:22:00 +0100883 case RuntimeSetting::Type::kPlayoutAudioDeviceChange:
884 RTC_NOTREACHED();
885 break;
Alex Loiko73ec0192018-05-15 10:52:28 +0200886 case RuntimeSetting::Type::kCustomRenderProcessingRuntimeSetting:
887 RTC_NOTREACHED();
888 break;
889 case RuntimeSetting::Type::kNotSpecified:
890 RTC_NOTREACHED();
891 break;
Per Åhgren552d3e32020-08-12 08:46:47 +0200892 case RuntimeSetting::Type::kCaptureOutputUsed:
Per Åhgren0a144a72021-02-09 08:47:51 +0100893 bool value;
894 setting.GetBool(&value);
895 HandleCaptureOutputUsedSetting(value);
Per Åhgren552d3e32020-08-12 08:46:47 +0200896 break;
Alex Loiko73ec0192018-05-15 10:52:28 +0200897 }
Per Åhgren652ada52021-03-03 10:52:44 +0000898 ++num_settings_processed;
899 }
900
901 if (num_settings_processed >= RuntimeSettingQueueSize()) {
902 // Handle overrun of the runtime settings queue, which likely will has
903 // caused settings to be discarded.
904 HandleOverrunInCaptureRuntimeSettingsQueue();
905 }
906}
907
908void AudioProcessingImpl::HandleOverrunInCaptureRuntimeSettingsQueue() {
909 // Fall back to a safe state for the case when a setting for capture output
910 // usage setting has been missed.
Per Åhgren19775cb2021-03-12 23:08:09 +0000911 HandleCaptureOutputUsedSetting(/*capture_output_used=*/true);
Alex Loiko73ec0192018-05-15 10:52:28 +0200912}
913
914void AudioProcessingImpl::HandleRenderRuntimeSettings() {
915 RuntimeSetting setting;
916 while (render_runtime_settings_.Remove(&setting)) {
Alex Loiko62347222018-09-10 10:18:07 +0200917 if (aec_dump_) {
918 aec_dump_->WriteRuntimeSetting(setting);
919 }
Alex Loiko73ec0192018-05-15 10:52:28 +0200920 switch (setting.type()) {
Alessio Bazzica7c19a702019-11-07 13:22:00 +0100921 case RuntimeSetting::Type::kPlayoutAudioDeviceChange: // fall-through
Alessio Bazzica7587de42019-11-11 13:32:20 +0100922 case RuntimeSetting::Type::kPlayoutVolumeChange: // fall-through
Alex Loiko73ec0192018-05-15 10:52:28 +0200923 case RuntimeSetting::Type::kCustomRenderProcessingRuntimeSetting:
saza1d600522019-10-18 13:29:43 +0200924 if (submodules_.render_pre_processor) {
925 submodules_.render_pre_processor->SetRuntimeSetting(setting);
Alex Loiko73ec0192018-05-15 10:52:28 +0200926 }
927 break;
Sam Zackrissonf0d1c032019-03-27 13:28:08 +0100928 case RuntimeSetting::Type::kCapturePreGain: // fall-through
Per Åhgrendb5d7282021-03-15 16:31:04 +0000929 case RuntimeSetting::Type::kCapturePostGain: // fall-through
Sam Zackrissonf0d1c032019-03-27 13:28:08 +0100930 case RuntimeSetting::Type::kCaptureCompressionGain: // fall-through
Per Åhgren6ee75fd2019-04-26 11:33:37 +0200931 case RuntimeSetting::Type::kCaptureFixedPostGain: // fall-through
Per Åhgren552d3e32020-08-12 08:46:47 +0200932 case RuntimeSetting::Type::kCaptureOutputUsed: // fall-through
Alessio Bazzica33444dc2018-04-20 13:16:55 +0200933 case RuntimeSetting::Type::kNotSpecified:
Alessio Bazzicac054e782018-04-16 12:10:09 +0200934 RTC_NOTREACHED();
935 break;
936 }
937 }
938}
939
peah9e6a2902017-05-15 07:19:21 -0700940void AudioProcessingImpl::QueueBandedRenderAudio(AudioBuffer* audio) {
kwibergaf476c72016-11-28 15:21:39 -0800941 RTC_DCHECK_GE(160, audio->num_frames_per_band());
peah764e3642016-10-22 05:04:30 -0700942
saza1d600522019-10-18 13:29:43 +0200943 if (submodules_.echo_control_mobile) {
Per Åhgrenb6e24d72019-04-29 12:14:50 +0200944 EchoControlMobileImpl::PackRenderAudioBuffer(audio, num_output_channels(),
945 num_reverse_channels(),
946 &aecm_render_queue_buffer_);
947 RTC_DCHECK(aecm_render_signal_queue_);
948 // Insert the samples into the queue.
949 if (!aecm_render_signal_queue_->Insert(&aecm_render_queue_buffer_)) {
950 // The data queue is full and needs to be emptied.
951 EmptyQueuedRenderAudio();
peaha0624602016-10-25 04:45:24 -0700952
Per Åhgrenb6e24d72019-04-29 12:14:50 +0200953 // Retry the insert (should always work).
954 bool result =
955 aecm_render_signal_queue_->Insert(&aecm_render_queue_buffer_);
956 RTC_DCHECK(result);
957 }
peah764e3642016-10-22 05:04:30 -0700958 }
peah701d6282016-10-25 05:42:20 -0700959
Per Åhgren0695df12020-01-13 14:43:13 +0100960 if (!submodules_.agc_manager && submodules_.gain_control) {
Per Åhgrene35b32c2019-11-22 18:22:04 +0100961 GainControlImpl::PackRenderAudioBuffer(*audio, &agc_render_queue_buffer_);
peah701d6282016-10-25 05:42:20 -0700962 // Insert the samples into the queue.
963 if (!agc_render_signal_queue_->Insert(&agc_render_queue_buffer_)) {
964 // The data queue is full and needs to be emptied.
965 EmptyQueuedRenderAudio();
966
967 // Retry the insert (should always work).
968 bool result = agc_render_signal_queue_->Insert(&agc_render_queue_buffer_);
969 RTC_DCHECK(result);
970 }
971 }
peah9e6a2902017-05-15 07:19:21 -0700972}
ivoc9f4a4a02016-10-28 05:39:16 -0700973
peah9e6a2902017-05-15 07:19:21 -0700974void AudioProcessingImpl::QueueNonbandedRenderAudio(AudioBuffer* audio) {
ivoc9f4a4a02016-10-28 05:39:16 -0700975 ResidualEchoDetector::PackRenderAudioBuffer(audio, &red_render_queue_buffer_);
976
977 // Insert the samples into the queue.
978 if (!red_render_signal_queue_->Insert(&red_render_queue_buffer_)) {
979 // The data queue is full and needs to be emptied.
980 EmptyQueuedRenderAudio();
981
982 // Retry the insert (should always work).
983 bool result = red_render_signal_queue_->Insert(&red_render_queue_buffer_);
984 RTC_DCHECK(result);
985 }
peah764e3642016-10-22 05:04:30 -0700986}
987
988void AudioProcessingImpl::AllocateRenderQueue() {
peah701d6282016-10-25 05:42:20 -0700989 const size_t new_agc_render_queue_element_max_size =
peah9e6a2902017-05-15 07:19:21 -0700990 std::max(static_cast<size_t>(1), kMaxAllowedValuesOfSamplesPerBand);
peah701d6282016-10-25 05:42:20 -0700991
ivoc9f4a4a02016-10-28 05:39:16 -0700992 const size_t new_red_render_queue_element_max_size =
993 std::max(static_cast<size_t>(1), kMaxAllowedValuesOfSamplesPerFrame);
994
peaha0624602016-10-25 04:45:24 -0700995 // Reallocate the queues if the queue item sizes are too small to fit the
996 // data to put in the queues.
peah701d6282016-10-25 05:42:20 -0700997
998 if (agc_render_queue_element_max_size_ <
999 new_agc_render_queue_element_max_size) {
1000 agc_render_queue_element_max_size_ = new_agc_render_queue_element_max_size;
1001
1002 std::vector<int16_t> template_queue_element(
1003 agc_render_queue_element_max_size_);
1004
1005 agc_render_signal_queue_.reset(
1006 new SwapQueue<std::vector<int16_t>, RenderQueueItemVerifier<int16_t>>(
1007 kMaxNumFramesToBuffer, template_queue_element,
1008 RenderQueueItemVerifier<int16_t>(
1009 agc_render_queue_element_max_size_)));
1010
1011 agc_render_queue_buffer_.resize(agc_render_queue_element_max_size_);
1012 agc_capture_queue_buffer_.resize(agc_render_queue_element_max_size_);
1013 } else {
1014 agc_render_signal_queue_->Clear();
peah764e3642016-10-22 05:04:30 -07001015 }
ivoc9f4a4a02016-10-28 05:39:16 -07001016
1017 if (red_render_queue_element_max_size_ <
1018 new_red_render_queue_element_max_size) {
1019 red_render_queue_element_max_size_ = new_red_render_queue_element_max_size;
1020
1021 std::vector<float> template_queue_element(
1022 red_render_queue_element_max_size_);
1023
1024 red_render_signal_queue_.reset(
1025 new SwapQueue<std::vector<float>, RenderQueueItemVerifier<float>>(
1026 kMaxNumFramesToBuffer, template_queue_element,
1027 RenderQueueItemVerifier<float>(
1028 red_render_queue_element_max_size_)));
1029
1030 red_render_queue_buffer_.resize(red_render_queue_element_max_size_);
1031 red_capture_queue_buffer_.resize(red_render_queue_element_max_size_);
1032 } else {
1033 red_render_signal_queue_->Clear();
1034 }
peah764e3642016-10-22 05:04:30 -07001035}
1036
1037void AudioProcessingImpl::EmptyQueuedRenderAudio() {
Markus Handell0df0fae2020-07-07 15:53:34 +02001038 MutexLock lock_capture(&mutex_capture_);
Markus Handell02ba1d22020-05-14 14:31:18 +02001039 EmptyQueuedRenderAudioLocked();
1040}
1041
1042void AudioProcessingImpl::EmptyQueuedRenderAudioLocked() {
saza1d600522019-10-18 13:29:43 +02001043 if (submodules_.echo_control_mobile) {
Per Åhgrenb6e24d72019-04-29 12:14:50 +02001044 RTC_DCHECK(aecm_render_signal_queue_);
1045 while (aecm_render_signal_queue_->Remove(&aecm_capture_queue_buffer_)) {
saza1d600522019-10-18 13:29:43 +02001046 submodules_.echo_control_mobile->ProcessRenderAudio(
Per Åhgrenb6e24d72019-04-29 12:14:50 +02001047 aecm_capture_queue_buffer_);
1048 }
peah701d6282016-10-25 05:42:20 -07001049 }
1050
Per Åhgren0695df12020-01-13 14:43:13 +01001051 if (submodules_.gain_control) {
1052 while (agc_render_signal_queue_->Remove(&agc_capture_queue_buffer_)) {
1053 submodules_.gain_control->ProcessRenderAudio(agc_capture_queue_buffer_);
1054 }
peah764e3642016-10-22 05:04:30 -07001055 }
ivoc9f4a4a02016-10-28 05:39:16 -07001056
1057 while (red_render_signal_queue_->Remove(&red_capture_queue_buffer_)) {
saza1d600522019-10-18 13:29:43 +02001058 RTC_DCHECK(submodules_.echo_detector);
1059 submodules_.echo_detector->AnalyzeRenderAudio(red_capture_queue_buffer_);
ivoc9f4a4a02016-10-28 05:39:16 -07001060 }
peah764e3642016-10-22 05:04:30 -07001061}
1062
Per Åhgren645f24c2020-03-16 12:06:02 +01001063int AudioProcessingImpl::ProcessStream(const int16_t* const src,
1064 const StreamConfig& input_config,
1065 const StreamConfig& output_config,
Per Åhgrendc5522b2020-03-19 14:55:58 +01001066 int16_t* const dest) {
Per Åhgren71652f42020-03-17 13:23:58 +01001067 TRACE_EVENT0("webrtc", "AudioProcessing::ProcessStream_AudioFrame");
Sam Zackrisson12e319a2020-01-03 14:54:20 +01001068 RETURN_ON_ERR(MaybeInitializeCapture(input_config, output_config));
Oskar Sundbom4b276482019-05-23 14:28:00 +02001069
Markus Handell0df0fae2020-07-07 15:53:34 +02001070 MutexLock lock_capture(&mutex_capture_);
Alessio Bazzica0441bb62021-08-10 15:23:23 +02001071 DenormalDisabler denormal_disabler(use_denormal_disabler_);
niklase@google.com470e71d2011-07-07 08:21:25 +00001072
aleloi868f32f2017-05-23 07:20:05 -07001073 if (aec_dump_) {
Per Åhgren645f24c2020-03-16 12:06:02 +01001074 RecordUnprocessedCaptureStream(src, input_config);
aleloi868f32f2017-05-23 07:20:05 -07001075 }
1076
Per Åhgren645f24c2020-03-16 12:06:02 +01001077 capture_.capture_audio->CopyFrom(src, input_config);
Gustaf Ullberg3c918b12019-10-11 13:14:44 +02001078 if (capture_.capture_fullband_audio) {
Per Åhgren645f24c2020-03-16 12:06:02 +01001079 capture_.capture_fullband_audio->CopyFrom(src, input_config);
Gustaf Ullberg3c918b12019-10-11 13:14:44 +02001080 }
peahde65ddc2016-09-16 15:02:15 -07001081 RETURN_ON_ERR(ProcessCaptureStreamLocked());
Gustaf Ullberg8675eee2019-10-09 13:34:36 +02001082 if (submodule_states_.CaptureMultiBandProcessingPresent() ||
Per Åhgrena1351272019-08-15 12:15:46 +02001083 submodule_states_.CaptureFullBandProcessingActive()) {
Gustaf Ullberg422b9e02019-10-09 13:02:14 +02001084 if (capture_.capture_fullband_audio) {
Per Åhgren645f24c2020-03-16 12:06:02 +01001085 capture_.capture_fullband_audio->CopyTo(output_config, dest);
Gustaf Ullberg422b9e02019-10-09 13:02:14 +02001086 } else {
Per Åhgren645f24c2020-03-16 12:06:02 +01001087 capture_.capture_audio->CopyTo(output_config, dest);
Gustaf Ullberg422b9e02019-10-09 13:02:14 +02001088 }
Per Åhgrena1351272019-08-15 12:15:46 +02001089 }
Per Åhgren645f24c2020-03-16 12:06:02 +01001090
aleloi868f32f2017-05-23 07:20:05 -07001091 if (aec_dump_) {
Per Åhgren645f24c2020-03-16 12:06:02 +01001092 RecordProcessedCaptureStream(dest, output_config);
aleloi868f32f2017-05-23 07:20:05 -07001093 }
andrew@webrtc.org17e40642014-03-04 20:58:13 +00001094
1095 return kNoError;
1096}
1097
peahde65ddc2016-09-16 15:02:15 -07001098int AudioProcessingImpl::ProcessCaptureStreamLocked() {
Markus Handell02ba1d22020-05-14 14:31:18 +02001099 EmptyQueuedRenderAudioLocked();
Alex Loiko73ec0192018-05-15 10:52:28 +02001100 HandleCaptureRuntimeSettings();
Alessio Bazzica0441bb62021-08-10 15:23:23 +02001101 DenormalDisabler denormal_disabler(use_denormal_disabler_);
Alessio Bazzicac054e782018-04-16 12:10:09 +02001102
peahb58a1582016-03-15 09:34:24 -07001103 // Ensure that not both the AEC and AECM are active at the same time.
Sam Zackrisson2a959d92018-07-23 14:48:07 +00001104 // TODO(peah): Simplify once the public API Enable functions for these
1105 // are moved to APM.
Sam Zackrisson308bc642019-12-23 10:22:08 +01001106 RTC_DCHECK_LE(
1107 !!submodules_.echo_controller + !!submodules_.echo_control_mobile, 1);
peahb58a1582016-03-15 09:34:24 -07001108
peahde65ddc2016-09-16 15:02:15 -07001109 AudioBuffer* capture_buffer = capture_.capture_audio.get(); // For brevity.
Per Åhgren2e8e1c62019-12-20 00:42:22 +01001110 AudioBuffer* linear_aec_buffer = capture_.linear_aec_output.get();
ekmeyerson60d9b332015-08-14 10:35:55 -07001111
Per Åhgrenc0424252019-12-10 13:04:15 +01001112 if (submodules_.high_pass_filter &&
1113 config_.high_pass_filter.apply_in_full_band &&
1114 !constants_.enforce_split_band_hpf) {
1115 submodules_.high_pass_filter->Process(capture_buffer,
1116 /*use_split_band_data=*/false);
1117 }
1118
Per Åhgrendb5d7282021-03-15 16:31:04 +00001119 if (submodules_.capture_levels_adjuster) {
1120 // If the analog mic gain emulation is active, get the emulated analog mic
1121 // gain and pass it to the analog gain control functionality.
1122 if (config_.capture_level_adjustment.analog_mic_gain_emulation.enabled) {
1123 int level = submodules_.capture_levels_adjuster->GetAnalogMicGainLevel();
1124 if (submodules_.agc_manager) {
1125 submodules_.agc_manager->set_stream_analog_level(level);
1126 } else if (submodules_.gain_control) {
1127 int error = submodules_.gain_control->set_stream_analog_level(level);
1128 RTC_DCHECK_EQ(kNoError, error);
1129 }
1130 }
1131
1132 submodules_.capture_levels_adjuster->ApplyPreLevelAdjustment(
1133 *capture_buffer);
Alex Loikob5c9a792018-04-16 16:31:22 +02001134 }
1135
Per Åhgren928146f2019-08-20 09:19:21 +02001136 capture_input_rms_.Analyze(rtc::ArrayView<const float>(
Per Åhgrend47941e2019-08-22 11:51:13 +02001137 capture_buffer->channels_const()[0],
henrik.lundin290d43a2016-11-29 08:09:09 -08001138 capture_nonlocked_.capture_processing_format.num_frames()));
peah1b08dc32016-12-20 13:45:58 -08001139 const bool log_rms = ++capture_rms_interval_counter_ >= 1000;
1140 if (log_rms) {
1141 capture_rms_interval_counter_ = 0;
1142 RmsLevel::Levels levels = capture_input_rms_.AverageAndPeak();
henrik.lundin45bb5132016-12-06 04:28:04 -08001143 RTC_HISTOGRAM_COUNTS_LINEAR("WebRTC.Audio.ApmCaptureInputLevelAverageRms",
1144 levels.average, 1, RmsLevel::kMinLevelDb, 64);
1145 RTC_HISTOGRAM_COUNTS_LINEAR("WebRTC.Audio.ApmCaptureInputLevelPeakRms",
1146 levels.peak, 1, RmsLevel::kMinLevelDb, 64);
henrik.lundin290d43a2016-11-29 08:09:09 -08001147 }
1148
saza1d600522019-10-18 13:29:43 +02001149 if (submodules_.echo_controller) {
Per Åhgren88cf0502018-07-16 17:08:41 +02001150 // Detect and flag any change in the analog gain.
Markus Handell02ba1d22020-05-14 14:31:18 +02001151 int analog_mic_level = recommended_stream_analog_level_locked();
Per Åhgren88cf0502018-07-16 17:08:41 +02001152 capture_.echo_path_gain_change =
1153 capture_.prev_analog_mic_level != analog_mic_level &&
1154 capture_.prev_analog_mic_level != -1;
1155 capture_.prev_analog_mic_level = analog_mic_level;
1156
Per Åhgrendb5d7282021-03-15 16:31:04 +00001157 // Detect and flag any change in the capture level adjustment pre-gain.
1158 if (submodules_.capture_levels_adjuster) {
1159 float pre_adjustment_gain =
1160 submodules_.capture_levels_adjuster->GetPreAdjustmentGain();
Per Åhgrend2650d12018-10-02 17:00:59 +02001161 capture_.echo_path_gain_change =
1162 capture_.echo_path_gain_change ||
Per Åhgrendb5d7282021-03-15 16:31:04 +00001163 (capture_.prev_pre_adjustment_gain != pre_adjustment_gain &&
1164 capture_.prev_pre_adjustment_gain >= 0.f);
1165 capture_.prev_pre_adjustment_gain = pre_adjustment_gain;
Per Åhgrend2650d12018-10-02 17:00:59 +02001166 }
Fredrik Hernqvistca362852019-05-10 15:50:02 +02001167
1168 // Detect volume change.
1169 capture_.echo_path_gain_change =
1170 capture_.echo_path_gain_change ||
1171 (capture_.prev_playout_volume != capture_.playout_volume &&
1172 capture_.prev_playout_volume >= 0);
1173 capture_.prev_playout_volume = capture_.playout_volume;
1174
saza1d600522019-10-18 13:29:43 +02001175 submodules_.echo_controller->AnalyzeCapture(capture_buffer);
peahe0eae3c2016-12-14 01:16:23 -08001176 }
1177
Per Åhgren0695df12020-01-13 14:43:13 +01001178 if (submodules_.agc_manager) {
Per Åhgren3daedb62019-11-22 12:11:40 +01001179 submodules_.agc_manager->AnalyzePreProcess(capture_buffer);
pbos@webrtc.org788acd12014-12-15 09:41:24 +00001180 }
1181
peah2ace3f92016-09-10 04:42:27 -07001182 if (submodule_states_.CaptureMultiBandSubModulesActive() &&
1183 SampleRateSupportsMultiBand(
peahde65ddc2016-09-16 15:02:15 -07001184 capture_nonlocked_.capture_processing_format.sample_rate_hz())) {
1185 capture_buffer->SplitIntoFrequencyBands();
niklase@google.com470e71d2011-07-07 08:21:25 +00001186 }
1187
Per Åhgrene14cb992019-11-27 09:34:22 +01001188 const bool multi_channel_capture = config_.pipeline.multi_channel_capture &&
1189 constants_.multi_channel_capture_support;
1190 if (submodules_.echo_controller && !multi_channel_capture) {
peah522d71b2017-02-23 05:16:26 -08001191 // Force down-mixing of the number of channels after the detection of
1192 // capture signal saturation.
1193 // TODO(peah): Look into ensuring that this kind of tampering with the
1194 // AudioBuffer functionality should not be needed.
1195 capture_buffer->set_num_channels(1);
1196 }
1197
Per Åhgrenc0424252019-12-10 13:04:15 +01001198 if (submodules_.high_pass_filter &&
1199 (!config_.high_pass_filter.apply_in_full_band ||
1200 constants_.enforce_split_band_hpf)) {
1201 submodules_.high_pass_filter->Process(capture_buffer,
1202 /*use_split_band_data=*/true);
peah8271d042016-11-22 07:24:52 -08001203 }
Per Åhgren0cbb58e2019-10-29 22:59:44 +01001204
Per Åhgren0695df12020-01-13 14:43:13 +01001205 if (submodules_.gain_control) {
1206 RETURN_ON_ERR(
1207 submodules_.gain_control->AnalyzeCaptureAudio(*capture_buffer));
1208 }
Per Åhgren2e8e1c62019-12-20 00:42:22 +01001209
Per Åhgren8ad9e742020-01-30 07:40:58 +01001210 if ((!config_.noise_suppression.analyze_linear_aec_output_when_available ||
1211 !linear_aec_buffer || submodules_.echo_control_mobile) &&
1212 submodules_.noise_suppressor) {
1213 submodules_.noise_suppressor->Analyze(*capture_buffer);
saza0bad15f2019-10-16 11:46:11 +02001214 }
peahb58a1582016-03-15 09:34:24 -07001215
saza1d600522019-10-18 13:29:43 +02001216 if (submodules_.echo_control_mobile) {
Per Åhgrenb6e24d72019-04-29 12:14:50 +02001217 // Ensure that the stream delay was set before the call to the
1218 // AECM ProcessCaptureAudio function.
Sam Zackrisson2d31aea2020-01-17 10:55:09 +01001219 if (!capture_.was_stream_delay_set) {
Per Åhgrenb6e24d72019-04-29 12:14:50 +02001220 return AudioProcessing::kStreamParameterNotSetError;
Per Åhgrend0fa8202018-04-18 09:35:13 +02001221 }
1222
saza1d600522019-10-18 13:29:43 +02001223 if (submodules_.noise_suppressor) {
Per Åhgren0cbb58e2019-10-29 22:59:44 +01001224 submodules_.noise_suppressor->Process(capture_buffer);
Per Åhgrenb6e24d72019-04-29 12:14:50 +02001225 }
peahe0eae3c2016-12-14 01:16:23 -08001226
saza1d600522019-10-18 13:29:43 +02001227 RETURN_ON_ERR(submodules_.echo_control_mobile->ProcessCaptureAudio(
Per Åhgren46537a32017-06-07 10:08:10 +02001228 capture_buffer, stream_delay_ms()));
Per Åhgrenb6e24d72019-04-29 12:14:50 +02001229 } else {
saza1d600522019-10-18 13:29:43 +02001230 if (submodules_.echo_controller) {
Per Åhgrenb6e24d72019-04-29 12:14:50 +02001231 data_dumper_->DumpRaw("stream_delay", stream_delay_ms());
1232
Sam Zackrisson2d31aea2020-01-17 10:55:09 +01001233 if (capture_.was_stream_delay_set) {
saza1d600522019-10-18 13:29:43 +02001234 submodules_.echo_controller->SetAudioBufferDelay(stream_delay_ms());
Per Åhgrenb6e24d72019-04-29 12:14:50 +02001235 }
1236
saza1d600522019-10-18 13:29:43 +02001237 submodules_.echo_controller->ProcessCapture(
Per Åhgrenc20a19c2019-11-13 11:12:29 +01001238 capture_buffer, linear_aec_buffer, capture_.echo_path_gain_change);
Per Åhgrenb6e24d72019-04-29 12:14:50 +02001239 }
1240
Per Åhgren2e8e1c62019-12-20 00:42:22 +01001241 if (config_.noise_suppression.analyze_linear_aec_output_when_available &&
Per Åhgren8ad9e742020-01-30 07:40:58 +01001242 linear_aec_buffer && submodules_.noise_suppressor) {
1243 submodules_.noise_suppressor->Analyze(*linear_aec_buffer);
Per Åhgren2e8e1c62019-12-20 00:42:22 +01001244 }
1245
saza1d600522019-10-18 13:29:43 +02001246 if (submodules_.noise_suppressor) {
Per Åhgren0cbb58e2019-10-29 22:59:44 +01001247 submodules_.noise_suppressor->Process(capture_buffer);
saza0bad15f2019-10-16 11:46:11 +02001248 }
Per Åhgren46537a32017-06-07 10:08:10 +02001249 }
ivoc9f4a4a02016-10-28 05:39:16 -07001250
Sam Zackrisson4db667b2018-12-21 16:29:27 +01001251 if (config_.voice_detection.enabled) {
Sam Zackrisson4db667b2018-12-21 16:29:27 +01001252 capture_.stats.voice_detected =
saza1d600522019-10-18 13:29:43 +02001253 submodules_.voice_detector->ProcessCaptureAudio(capture_buffer);
Sam Zackrisson4db667b2018-12-21 16:29:27 +01001254 } else {
1255 capture_.stats.voice_detected = absl::nullopt;
1256 }
pbos@webrtc.org788acd12014-12-15 09:41:24 +00001257
Per Åhgren0695df12020-01-13 14:43:13 +01001258 if (submodules_.agc_manager) {
Per Åhgren3daedb62019-11-22 12:11:40 +01001259 submodules_.agc_manager->Process(capture_buffer);
1260
1261 absl::optional<int> new_digital_gain =
1262 submodules_.agc_manager->GetDigitalComressionGain();
Per Åhgren0695df12020-01-13 14:43:13 +01001263 if (new_digital_gain && submodules_.gain_control) {
Per Åhgren3daedb62019-11-22 12:11:40 +01001264 submodules_.gain_control->set_compression_gain_db(*new_digital_gain);
1265 }
pbos@webrtc.org788acd12014-12-15 09:41:24 +00001266 }
Per Åhgren0695df12020-01-13 14:43:13 +01001267
1268 if (submodules_.gain_control) {
1269 // TODO(peah): Add reporting from AEC3 whether there is echo.
1270 RETURN_ON_ERR(submodules_.gain_control->ProcessCaptureAudio(
1271 capture_buffer, /*stream_has_echo*/ false));
1272 }
niklase@google.com470e71d2011-07-07 08:21:25 +00001273
Ilya Nikolaevskiy80979352021-03-11 11:40:46 +00001274 if (submodule_states_.CaptureMultiBandProcessingPresent() &&
1275 SampleRateSupportsMultiBand(
1276 capture_nonlocked_.capture_processing_format.sample_rate_hz())) {
1277 capture_buffer->MergeFrequencyBands();
Gustaf Ullberg422b9e02019-10-09 13:02:14 +02001278 }
1279
Per Åhgren19775cb2021-03-12 23:08:09 +00001280 capture_.stats.output_rms_dbfs = absl::nullopt;
1281 if (capture_.capture_output_used) {
1282 if (capture_.capture_fullband_audio) {
1283 const auto& ec = submodules_.echo_controller;
1284 bool ec_active = ec ? ec->ActiveProcessing() : false;
1285 // Only update the fullband buffer if the multiband processing has changed
1286 // the signal. Keep the original signal otherwise.
1287 if (submodule_states_.CaptureMultiBandProcessingActive(ec_active)) {
1288 capture_buffer->CopyTo(capture_.capture_fullband_audio.get());
1289 }
1290 capture_buffer = capture_.capture_fullband_audio.get();
Ilya Nikolaevskiy80979352021-03-11 11:40:46 +00001291 }
Per Åhgren19775cb2021-03-12 23:08:09 +00001292
1293 if (config_.residual_echo_detector.enabled) {
1294 RTC_DCHECK(submodules_.echo_detector);
1295 submodules_.echo_detector->AnalyzeCaptureAudio(
1296 rtc::ArrayView<const float>(capture_buffer->channels()[0],
1297 capture_buffer->num_frames()));
1298 }
1299
1300 // TODO(aluebs): Investigate if the transient suppression placement should
1301 // be before or after the AGC.
1302 if (submodules_.transient_suppressor) {
1303 float voice_probability =
1304 submodules_.agc_manager.get()
1305 ? submodules_.agc_manager->voice_probability()
1306 : 1.f;
1307
1308 submodules_.transient_suppressor->Suppress(
1309 capture_buffer->channels()[0], capture_buffer->num_frames(),
1310 capture_buffer->num_channels(),
1311 capture_buffer->split_bands_const(0)[kBand0To8kHz],
1312 capture_buffer->num_frames_per_band(),
1313 capture_.keyboard_info.keyboard_data,
1314 capture_.keyboard_info.num_keyboard_frames, voice_probability,
1315 capture_.key_pressed);
1316 }
1317
Artem Titov0b489302021-07-28 20:50:03 +02001318 // Experimental APM sub-module that analyzes `capture_buffer`.
Per Åhgren19775cb2021-03-12 23:08:09 +00001319 if (submodules_.capture_analyzer) {
1320 submodules_.capture_analyzer->Analyze(capture_buffer);
1321 }
1322
1323 if (submodules_.gain_controller2) {
1324 submodules_.gain_controller2->NotifyAnalogLevel(
1325 recommended_stream_analog_level_locked());
1326 submodules_.gain_controller2->Process(capture_buffer);
1327 }
1328
1329 if (submodules_.capture_post_processor) {
1330 submodules_.capture_post_processor->Process(capture_buffer);
1331 }
1332
1333 // The level estimator operates on the recombined data.
1334 if (config_.level_estimation.enabled) {
1335 submodules_.output_level_estimator->ProcessStream(*capture_buffer);
1336 capture_.stats.output_rms_dbfs =
1337 submodules_.output_level_estimator->RMS();
1338 }
1339
1340 capture_output_rms_.Analyze(rtc::ArrayView<const float>(
1341 capture_buffer->channels_const()[0],
1342 capture_nonlocked_.capture_processing_format.num_frames()));
1343 if (log_rms) {
1344 RmsLevel::Levels levels = capture_output_rms_.AverageAndPeak();
1345 RTC_HISTOGRAM_COUNTS_LINEAR(
1346 "WebRTC.Audio.ApmCaptureOutputLevelAverageRms", levels.average, 1,
1347 RmsLevel::kMinLevelDb, 64);
1348 RTC_HISTOGRAM_COUNTS_LINEAR("WebRTC.Audio.ApmCaptureOutputLevelPeakRms",
1349 levels.peak, 1, RmsLevel::kMinLevelDb, 64);
1350 }
1351
1352 if (submodules_.agc_manager) {
1353 int level = recommended_stream_analog_level_locked();
1354 data_dumper_->DumpRaw("experimental_gain_control_stream_analog_level", 1,
1355 &level);
1356 }
1357
1358 // Compute echo-detector stats.
1359 if (config_.residual_echo_detector.enabled) {
1360 RTC_DCHECK(submodules_.echo_detector);
1361 auto ed_metrics = submodules_.echo_detector->GetMetrics();
1362 capture_.stats.residual_echo_likelihood = ed_metrics.echo_likelihood;
1363 capture_.stats.residual_echo_likelihood_recent_max =
1364 ed_metrics.echo_likelihood_recent_max;
1365 }
Ilya Nikolaevskiy80979352021-03-11 11:40:46 +00001366 }
1367
Per Åhgren19775cb2021-03-12 23:08:09 +00001368 // Compute echo-controller stats.
Per Åhgrencf4c8722019-12-30 14:32:14 +01001369 if (submodules_.echo_controller) {
1370 auto ec_metrics = submodules_.echo_controller->GetMetrics();
1371 capture_.stats.echo_return_loss = ec_metrics.echo_return_loss;
1372 capture_.stats.echo_return_loss_enhancement =
1373 ec_metrics.echo_return_loss_enhancement;
1374 capture_.stats.delay_ms = ec_metrics.delay_ms;
1375 }
Per Åhgrencf4c8722019-12-30 14:32:14 +01001376
1377 // Pass stats for reporting.
1378 stats_reporter_.UpdateStatistics(capture_.stats);
1379
Per Åhgrendb5d7282021-03-15 16:31:04 +00001380 if (submodules_.capture_levels_adjuster) {
1381 submodules_.capture_levels_adjuster->ApplyPostLevelAdjustment(
1382 *capture_buffer);
1383
1384 // If the analog mic gain emulation is active, retrieve the level from the
1385 // analog gain control and set it to mic gain emulator.
1386 if (config_.capture_level_adjustment.analog_mic_gain_emulation.enabled) {
1387 if (submodules_.agc_manager) {
1388 submodules_.capture_levels_adjuster->SetAnalogMicGainLevel(
1389 submodules_.agc_manager->stream_analog_level());
1390 } else if (submodules_.gain_control) {
1391 submodules_.capture_levels_adjuster->SetAnalogMicGainLevel(
1392 submodules_.gain_control->stream_analog_level());
1393 }
1394 }
1395 }
1396
Per Åhgren55bc0772021-03-12 14:18:36 +00001397 // Temporarily set the output to zero after the stream has been unmuted
1398 // (capture output is again used). The purpose of this is to avoid clicks and
1399 // artefacts in the audio that results when the processing again is
1400 // reactivated after unmuting.
1401 if (!capture_.capture_output_used_last_frame &&
1402 capture_.capture_output_used) {
1403 for (size_t ch = 0; ch < capture_buffer->num_channels(); ++ch) {
1404 rtc::ArrayView<float> channel_view(capture_buffer->channels()[ch],
1405 capture_buffer->num_frames());
1406 std::fill(channel_view.begin(), channel_view.end(), 0.f);
1407 }
1408 }
1409 capture_.capture_output_used_last_frame = capture_.capture_output_used;
1410
peahdf3efa82015-11-28 12:35:15 -08001411 capture_.was_stream_delay_set = false;
niklase@google.com470e71d2011-07-07 08:21:25 +00001412 return kNoError;
1413}
1414
Gustaf Ullberg8c51f2e2019-10-22 15:21:31 +02001415int AudioProcessingImpl::AnalyzeReverseStream(
1416 const float* const* data,
1417 const StreamConfig& reverse_config) {
1418 TRACE_EVENT0("webrtc", "AudioProcessing::AnalyzeReverseStream_StreamConfig");
Markus Handell0df0fae2020-07-07 15:53:34 +02001419 MutexLock lock(&mutex_render_);
Gustaf Ullberg8c51f2e2019-10-22 15:21:31 +02001420 return AnalyzeReverseStreamLocked(data, reverse_config, reverse_config);
1421}
1422
peahde65ddc2016-09-16 15:02:15 -07001423int AudioProcessingImpl::ProcessReverseStream(const float* const* src,
1424 const StreamConfig& input_config,
1425 const StreamConfig& output_config,
1426 float* const* dest) {
peah369f8282015-12-17 06:42:29 -08001427 TRACE_EVENT0("webrtc", "AudioProcessing::ProcessReverseStream_StreamConfig");
Markus Handell0df0fae2020-07-07 15:53:34 +02001428 MutexLock lock(&mutex_render_);
Alessio Bazzica0441bb62021-08-10 15:23:23 +02001429 DenormalDisabler denormal_disabler(use_denormal_disabler_);
1430
peahde65ddc2016-09-16 15:02:15 -07001431 RETURN_ON_ERR(AnalyzeReverseStreamLocked(src, input_config, output_config));
Alex Loiko5825aa62017-12-18 16:02:40 +01001432 if (submodule_states_.RenderMultiBandProcessingActive() ||
1433 submodule_states_.RenderFullBandProcessingActive()) {
peahdf3efa82015-11-28 12:35:15 -08001434 render_.render_audio->CopyTo(formats_.api_format.reverse_output_stream(),
1435 dest);
peah2ace3f92016-09-10 04:42:27 -07001436 } else if (formats_.api_format.reverse_input_stream() !=
1437 formats_.api_format.reverse_output_stream()) {
peahde65ddc2016-09-16 15:02:15 -07001438 render_.render_converter->Convert(src, input_config.num_samples(), dest,
1439 output_config.num_samples());
ekmeyerson60d9b332015-08-14 10:35:55 -07001440 } else {
peahde65ddc2016-09-16 15:02:15 -07001441 CopyAudioIfNeeded(src, input_config.num_frames(),
1442 input_config.num_channels(), dest);
ekmeyerson60d9b332015-08-14 10:35:55 -07001443 }
1444
1445 return kNoError;
Michael Graczyk86c6d332015-07-23 11:41:39 -07001446}
1447
peahdf3efa82015-11-28 12:35:15 -08001448int AudioProcessingImpl::AnalyzeReverseStreamLocked(
ekmeyerson60d9b332015-08-14 10:35:55 -07001449 const float* const* src,
peahde65ddc2016-09-16 15:02:15 -07001450 const StreamConfig& input_config,
1451 const StreamConfig& output_config) {
peahdf3efa82015-11-28 12:35:15 -08001452 if (src == nullptr) {
andrew@webrtc.org17e40642014-03-04 20:58:13 +00001453 return kNullPointerError;
1454 }
andrew@webrtc.org17e40642014-03-04 20:58:13 +00001455
peahde65ddc2016-09-16 15:02:15 -07001456 if (input_config.num_channels() == 0) {
Michael Graczyk86c6d332015-07-23 11:41:39 -07001457 return kBadNumberChannelsError;
andrew@webrtc.org17e40642014-03-04 20:58:13 +00001458 }
1459
peahdf3efa82015-11-28 12:35:15 -08001460 ProcessingConfig processing_config = formats_.api_format;
peahde65ddc2016-09-16 15:02:15 -07001461 processing_config.reverse_input_stream() = input_config;
1462 processing_config.reverse_output_stream() = output_config;
Michael Graczyk86c6d332015-07-23 11:41:39 -07001463
peahdf3efa82015-11-28 12:35:15 -08001464 RETURN_ON_ERR(MaybeInitializeRender(processing_config));
Fredrik Solenbergbbf21a32018-04-12 22:44:09 +02001465 RTC_DCHECK_EQ(input_config.num_frames(),
1466 formats_.api_format.reverse_input_stream().num_frames());
Michael Graczyk86c6d332015-07-23 11:41:39 -07001467
Alessio Bazzica0441bb62021-08-10 15:23:23 +02001468 DenormalDisabler denormal_disabler(use_denormal_disabler_);
1469
aleloi868f32f2017-05-23 07:20:05 -07001470 if (aec_dump_) {
1471 const size_t channel_size =
1472 formats_.api_format.reverse_input_stream().num_frames();
1473 const size_t num_channels =
1474 formats_.api_format.reverse_input_stream().num_channels();
1475 aec_dump_->WriteRenderStreamMessage(
Alex Loikoe36e8bb2018-02-16 11:54:07 +01001476 AudioFrameView<const float>(src, num_channels, channel_size));
aleloi868f32f2017-05-23 07:20:05 -07001477 }
peahdf3efa82015-11-28 12:35:15 -08001478 render_.render_audio->CopyFrom(src,
1479 formats_.api_format.reverse_input_stream());
peahde65ddc2016-09-16 15:02:15 -07001480 return ProcessRenderStreamLocked();
ekmeyerson60d9b332015-08-14 10:35:55 -07001481}
1482
Per Åhgren645f24c2020-03-16 12:06:02 +01001483int AudioProcessingImpl::ProcessReverseStream(const int16_t* const src,
1484 const StreamConfig& input_config,
1485 const StreamConfig& output_config,
1486 int16_t* const dest) {
Per Åhgren71652f42020-03-17 13:23:58 +01001487 TRACE_EVENT0("webrtc", "AudioProcessing::ProcessReverseStream_AudioFrame");
Per Åhgren2507f8c2020-03-19 12:33:29 +01001488
1489 if (input_config.num_channels() <= 0) {
1490 return AudioProcessing::Error::kBadNumberChannelsError;
1491 }
1492
Markus Handell0df0fae2020-07-07 15:53:34 +02001493 MutexLock lock(&mutex_render_);
Alessio Bazzica0441bb62021-08-10 15:23:23 +02001494 DenormalDisabler denormal_disabler(use_denormal_disabler_);
1495
peahdf3efa82015-11-28 12:35:15 -08001496 ProcessingConfig processing_config = formats_.api_format;
ekmeyerson60d9b332015-08-14 10:35:55 -07001497 processing_config.reverse_input_stream().set_sample_rate_hz(
Per Åhgren645f24c2020-03-16 12:06:02 +01001498 input_config.sample_rate_hz());
ekmeyerson60d9b332015-08-14 10:35:55 -07001499 processing_config.reverse_input_stream().set_num_channels(
Per Åhgren645f24c2020-03-16 12:06:02 +01001500 input_config.num_channels());
ekmeyerson60d9b332015-08-14 10:35:55 -07001501 processing_config.reverse_output_stream().set_sample_rate_hz(
Per Åhgren645f24c2020-03-16 12:06:02 +01001502 output_config.sample_rate_hz());
ekmeyerson60d9b332015-08-14 10:35:55 -07001503 processing_config.reverse_output_stream().set_num_channels(
Per Åhgren645f24c2020-03-16 12:06:02 +01001504 output_config.num_channels());
Michael Graczyk86c6d332015-07-23 11:41:39 -07001505
peahdf3efa82015-11-28 12:35:15 -08001506 RETURN_ON_ERR(MaybeInitializeRender(processing_config));
Per Åhgren645f24c2020-03-16 12:06:02 +01001507 if (input_config.num_frames() !=
peahdf3efa82015-11-28 12:35:15 -08001508 formats_.api_format.reverse_input_stream().num_frames()) {
andrew@webrtc.org17e40642014-03-04 20:58:13 +00001509 return kBadDataLengthError;
1510 }
niklase@google.com470e71d2011-07-07 08:21:25 +00001511
aleloi868f32f2017-05-23 07:20:05 -07001512 if (aec_dump_) {
Per Åhgren645f24c2020-03-16 12:06:02 +01001513 aec_dump_->WriteRenderStreamMessage(src, input_config.num_frames(),
1514 input_config.num_channels());
aleloi868f32f2017-05-23 07:20:05 -07001515 }
1516
Per Åhgren645f24c2020-03-16 12:06:02 +01001517 render_.render_audio->CopyFrom(src, input_config);
peahde65ddc2016-09-16 15:02:15 -07001518 RETURN_ON_ERR(ProcessRenderStreamLocked());
Per Åhgrena1351272019-08-15 12:15:46 +02001519 if (submodule_states_.RenderMultiBandProcessingActive() ||
1520 submodule_states_.RenderFullBandProcessingActive()) {
Per Åhgren645f24c2020-03-16 12:06:02 +01001521 render_.render_audio->CopyTo(output_config, dest);
Per Åhgrena1351272019-08-15 12:15:46 +02001522 }
aluebsb0319552016-03-17 20:39:53 -07001523 return kNoError;
andrew@webrtc.org17e40642014-03-04 20:58:13 +00001524}
niklase@google.com470e71d2011-07-07 08:21:25 +00001525
peahde65ddc2016-09-16 15:02:15 -07001526int AudioProcessingImpl::ProcessRenderStreamLocked() {
1527 AudioBuffer* render_buffer = render_.render_audio.get(); // For brevity.
peah9e6a2902017-05-15 07:19:21 -07001528
Alex Loiko73ec0192018-05-15 10:52:28 +02001529 HandleRenderRuntimeSettings();
Alessio Bazzica0441bb62021-08-10 15:23:23 +02001530 DenormalDisabler denormal_disabler(use_denormal_disabler_);
Alex Loiko73ec0192018-05-15 10:52:28 +02001531
saza1d600522019-10-18 13:29:43 +02001532 if (submodules_.render_pre_processor) {
1533 submodules_.render_pre_processor->Process(render_buffer);
Alex Loiko5825aa62017-12-18 16:02:40 +01001534 }
1535
Alessio Bazzicad2b97402018-08-09 14:23:11 +02001536 QueueNonbandedRenderAudio(render_buffer);
1537
peah2ace3f92016-09-10 04:42:27 -07001538 if (submodule_states_.RenderMultiBandSubModulesActive() &&
peahde65ddc2016-09-16 15:02:15 -07001539 SampleRateSupportsMultiBand(
1540 formats_.render_processing_format.sample_rate_hz())) {
1541 render_buffer->SplitIntoFrequencyBands();
niklase@google.com470e71d2011-07-07 08:21:25 +00001542 }
1543
peahce4d9152017-05-19 01:28:05 -07001544 if (submodule_states_.RenderMultiBandSubModulesActive()) {
1545 QueueBandedRenderAudio(render_buffer);
1546 }
1547
Alessio Bazzicad2b97402018-08-09 14:23:11 +02001548 // TODO(peah): Perform the queuing inside QueueRenderAudiuo().
saza1d600522019-10-18 13:29:43 +02001549 if (submodules_.echo_controller) {
1550 submodules_.echo_controller->AnalyzeRender(render_buffer);
peahe0eae3c2016-12-14 01:16:23 -08001551 }
niklase@google.com470e71d2011-07-07 08:21:25 +00001552
peah2ace3f92016-09-10 04:42:27 -07001553 if (submodule_states_.RenderMultiBandProcessingActive() &&
peahde65ddc2016-09-16 15:02:15 -07001554 SampleRateSupportsMultiBand(
1555 formats_.render_processing_format.sample_rate_hz())) {
1556 render_buffer->MergeFrequencyBands();
ekmeyerson60d9b332015-08-14 10:35:55 -07001557 }
1558
andrew@webrtc.org17e40642014-03-04 20:58:13 +00001559 return kNoError;
niklase@google.com470e71d2011-07-07 08:21:25 +00001560}
1561
1562int AudioProcessingImpl::set_stream_delay_ms(int delay) {
Markus Handell0df0fae2020-07-07 15:53:34 +02001563 MutexLock lock(&mutex_capture_);
andrew@webrtc.org5f23d642012-05-29 21:14:06 +00001564 Error retval = kNoError;
peahdf3efa82015-11-28 12:35:15 -08001565 capture_.was_stream_delay_set = true;
andrew@webrtc.org6f9f8172012-03-06 19:03:39 +00001566
niklase@google.com470e71d2011-07-07 08:21:25 +00001567 if (delay < 0) {
andrew@webrtc.org5f23d642012-05-29 21:14:06 +00001568 delay = 0;
1569 retval = kBadStreamParameterWarning;
niklase@google.com470e71d2011-07-07 08:21:25 +00001570 }
1571
1572 // TODO(ajm): the max is rather arbitrarily chosen; investigate.
1573 if (delay > 500) {
andrew@webrtc.org5f23d642012-05-29 21:14:06 +00001574 delay = 500;
1575 retval = kBadStreamParameterWarning;
niklase@google.com470e71d2011-07-07 08:21:25 +00001576 }
1577
peahdf3efa82015-11-28 12:35:15 -08001578 capture_nonlocked_.stream_delay_ms = delay;
andrew@webrtc.org5f23d642012-05-29 21:14:06 +00001579 return retval;
niklase@google.com470e71d2011-07-07 08:21:25 +00001580}
1581
Per Åhgrenc20a19c2019-11-13 11:12:29 +01001582bool AudioProcessingImpl::GetLinearAecOutput(
1583 rtc::ArrayView<std::array<float, 160>> linear_output) const {
Markus Handell0df0fae2020-07-07 15:53:34 +02001584 MutexLock lock(&mutex_capture_);
Per Åhgrenc20a19c2019-11-13 11:12:29 +01001585 AudioBuffer* linear_aec_buffer = capture_.linear_aec_output.get();
1586
1587 RTC_DCHECK(linear_aec_buffer);
1588 if (linear_aec_buffer) {
1589 RTC_DCHECK_EQ(1, linear_aec_buffer->num_bands());
1590 RTC_DCHECK_EQ(linear_output.size(), linear_aec_buffer->num_channels());
1591
1592 for (size_t ch = 0; ch < linear_aec_buffer->num_channels(); ++ch) {
1593 RTC_DCHECK_EQ(linear_output[ch].size(), linear_aec_buffer->num_frames());
1594 rtc::ArrayView<const float> channel_view =
1595 rtc::ArrayView<const float>(linear_aec_buffer->channels_const()[ch],
1596 linear_aec_buffer->num_frames());
Gustaf Ullberg45436972020-11-13 14:30:30 +01001597 FloatS16ToFloat(channel_view.data(), channel_view.size(),
1598 linear_output[ch].data());
Per Åhgrenc20a19c2019-11-13 11:12:29 +01001599 }
1600 return true;
1601 }
1602 RTC_LOG(LS_ERROR) << "No linear AEC output available";
1603 RTC_NOTREACHED();
1604 return false;
1605}
1606
niklase@google.com470e71d2011-07-07 08:21:25 +00001607int AudioProcessingImpl::stream_delay_ms() const {
peahdf3efa82015-11-28 12:35:15 -08001608 // Used as callback from submodules, hence locking is not allowed.
1609 return capture_nonlocked_.stream_delay_ms;
niklase@google.com470e71d2011-07-07 08:21:25 +00001610}
1611
andrew@webrtc.org17e40642014-03-04 20:58:13 +00001612void AudioProcessingImpl::set_stream_key_pressed(bool key_pressed) {
Markus Handell0df0fae2020-07-07 15:53:34 +02001613 MutexLock lock(&mutex_capture_);
peahdf3efa82015-11-28 12:35:15 -08001614 capture_.key_pressed = key_pressed;
andrew@webrtc.org17e40642014-03-04 20:58:13 +00001615}
1616
Sam Zackrissonf0d1c032019-03-27 13:28:08 +01001617void AudioProcessingImpl::set_stream_analog_level(int level) {
Markus Handell0df0fae2020-07-07 15:53:34 +02001618 MutexLock lock_capture(&mutex_capture_);
Per Åhgren0e3198e2019-11-18 08:52:22 +01001619
Per Åhgrendb5d7282021-03-15 16:31:04 +00001620 if (config_.capture_level_adjustment.analog_mic_gain_emulation.enabled) {
1621 // If the analog mic gain is emulated internally, simply cache the level for
1622 // later reporting back as the recommended stream analog level to use.
1623 capture_.cached_stream_analog_level_ = level;
1624 return;
1625 }
1626
Per Åhgren0e3198e2019-11-18 08:52:22 +01001627 if (submodules_.agc_manager) {
1628 submodules_.agc_manager->set_stream_analog_level(level);
1629 data_dumper_->DumpRaw("experimental_gain_control_set_stream_analog_level",
1630 1, &level);
Per Åhgrendb5d7282021-03-15 16:31:04 +00001631 return;
1632 }
1633
1634 if (submodules_.gain_control) {
Per Åhgren0e3198e2019-11-18 08:52:22 +01001635 int error = submodules_.gain_control->set_stream_analog_level(level);
1636 RTC_DCHECK_EQ(kNoError, error);
Per Åhgrendb5d7282021-03-15 16:31:04 +00001637 return;
Per Åhgren0e3198e2019-11-18 08:52:22 +01001638 }
Per Åhgrendb5d7282021-03-15 16:31:04 +00001639
1640 // If no analog mic gain control functionality is in place, cache the level
1641 // for later reporting back as the recommended stream analog level to use.
1642 capture_.cached_stream_analog_level_ = level;
Sam Zackrissonf0d1c032019-03-27 13:28:08 +01001643}
1644
1645int AudioProcessingImpl::recommended_stream_analog_level() const {
Markus Handell0df0fae2020-07-07 15:53:34 +02001646 MutexLock lock_capture(&mutex_capture_);
Markus Handell02ba1d22020-05-14 14:31:18 +02001647 return recommended_stream_analog_level_locked();
1648}
1649
1650int AudioProcessingImpl::recommended_stream_analog_level_locked() const {
Per Åhgrendb5d7282021-03-15 16:31:04 +00001651 if (config_.capture_level_adjustment.analog_mic_gain_emulation.enabled) {
Per Åhgren0695df12020-01-13 14:43:13 +01001652 return capture_.cached_stream_analog_level_;
Per Åhgren0e3198e2019-11-18 08:52:22 +01001653 }
Per Åhgrendb5d7282021-03-15 16:31:04 +00001654
1655 if (submodules_.agc_manager) {
1656 return submodules_.agc_manager->stream_analog_level();
1657 }
1658
1659 if (submodules_.gain_control) {
1660 return submodules_.gain_control->stream_analog_level();
1661 }
1662
1663 return capture_.cached_stream_analog_level_;
Sam Zackrissonf0d1c032019-03-27 13:28:08 +01001664}
1665
Per Åhgren09e9a832020-05-11 11:03:47 +02001666bool AudioProcessingImpl::CreateAndAttachAecDump(const std::string& file_name,
1667 int64_t max_log_size_bytes,
1668 rtc::TaskQueue* worker_queue) {
1669 std::unique_ptr<AecDump> aec_dump =
1670 AecDumpFactory::Create(file_name, max_log_size_bytes, worker_queue);
1671 if (!aec_dump) {
1672 return false;
1673 }
1674
1675 AttachAecDump(std::move(aec_dump));
1676 return true;
1677}
1678
1679bool AudioProcessingImpl::CreateAndAttachAecDump(FILE* handle,
1680 int64_t max_log_size_bytes,
1681 rtc::TaskQueue* worker_queue) {
1682 std::unique_ptr<AecDump> aec_dump =
1683 AecDumpFactory::Create(handle, max_log_size_bytes, worker_queue);
1684 if (!aec_dump) {
1685 return false;
1686 }
1687
1688 AttachAecDump(std::move(aec_dump));
1689 return true;
1690}
1691
aleloi868f32f2017-05-23 07:20:05 -07001692void AudioProcessingImpl::AttachAecDump(std::unique_ptr<AecDump> aec_dump) {
1693 RTC_DCHECK(aec_dump);
Markus Handell0df0fae2020-07-07 15:53:34 +02001694 MutexLock lock_render(&mutex_render_);
1695 MutexLock lock_capture(&mutex_capture_);
aleloi868f32f2017-05-23 07:20:05 -07001696
1697 // The previously attached AecDump will be destroyed with the
1698 // 'aec_dump' parameter, which is after locks are released.
1699 aec_dump_.swap(aec_dump);
1700 WriteAecDumpConfigMessage(true);
Minyue Li656d6092018-08-10 15:38:52 +02001701 aec_dump_->WriteInitMessage(formats_.api_format, rtc::TimeUTCMillis());
aleloi868f32f2017-05-23 07:20:05 -07001702}
1703
1704void AudioProcessingImpl::DetachAecDump() {
1705 // The d-tor of a task-queue based AecDump blocks until all pending
1706 // tasks are done. This construction avoids blocking while holding
1707 // the render and capture locks.
1708 std::unique_ptr<AecDump> aec_dump = nullptr;
1709 {
Markus Handell0df0fae2020-07-07 15:53:34 +02001710 MutexLock lock_render(&mutex_render_);
1711 MutexLock lock_capture(&mutex_capture_);
aleloi868f32f2017-05-23 07:20:05 -07001712 aec_dump = std::move(aec_dump_);
1713 }
1714}
1715
peah8271d042016-11-22 07:24:52 -08001716AudioProcessing::Config AudioProcessingImpl::GetConfig() const {
Markus Handell0df0fae2020-07-07 15:53:34 +02001717 MutexLock lock_render(&mutex_render_);
1718 MutexLock lock_capture(&mutex_capture_);
peah8271d042016-11-22 07:24:52 -08001719 return config_;
1720}
1721
peah2ace3f92016-09-10 04:42:27 -07001722bool AudioProcessingImpl::UpdateActiveSubmoduleStates() {
1723 return submodule_states_.Update(
Per Åhgren62ea0aa2019-12-09 10:18:44 +01001724 config_.high_pass_filter.enabled, !!submodules_.echo_control_mobile,
Per Åhgren8ad9e742020-01-30 07:40:58 +01001725 config_.residual_echo_detector.enabled, !!submodules_.noise_suppressor,
Per Åhgren0695df12020-01-13 14:43:13 +01001726 !!submodules_.gain_control, !!submodules_.gain_controller2,
Per Åhgrendb5d7282021-03-15 16:31:04 +00001727 config_.pre_amplifier.enabled || config_.capture_level_adjustment.enabled,
1728 capture_nonlocked_.echo_controller_enabled,
Per Åhgrenc0734712020-01-02 15:15:36 +01001729 config_.voice_detection.enabled, !!submodules_.transient_suppressor);
ekmeyerson60d9b332015-08-14 10:35:55 -07001730}
1731
Per Åhgrenc0734712020-01-02 15:15:36 +01001732void AudioProcessingImpl::InitializeTransientSuppressor() {
Gustaf Ullberga399c822021-05-18 12:17:56 +02001733 if (config_.transient_suppression.enabled &&
1734 !constants_.transient_suppressor_forced_off) {
sazaaa42ecd2020-04-01 15:24:40 +02001735 // Attempt to create a transient suppressor, if one is not already created.
Per Åhgrenc0734712020-01-02 15:15:36 +01001736 if (!submodules_.transient_suppressor) {
Sam Zackrissonb37e59d2020-04-27 08:39:33 +02001737 submodules_.transient_suppressor =
1738 CreateTransientSuppressor(submodule_creation_overrides_);
pbos@webrtc.org788acd12014-12-15 09:41:24 +00001739 }
sazaaa42ecd2020-04-01 15:24:40 +02001740 if (submodules_.transient_suppressor) {
1741 submodules_.transient_suppressor->Initialize(
1742 proc_fullband_sample_rate_hz(), capture_nonlocked_.split_rate,
1743 num_proc_channels());
1744 } else {
1745 RTC_LOG(LS_WARNING)
1746 << "No transient suppressor created (probably disabled)";
1747 }
Per Åhgrenc0734712020-01-02 15:15:36 +01001748 } else {
1749 submodules_.transient_suppressor.reset();
pbos@webrtc.org788acd12014-12-15 09:41:24 +00001750 }
pbos@webrtc.org788acd12014-12-15 09:41:24 +00001751}
1752
Per Åhgren0f14db22020-01-03 14:27:14 +01001753void AudioProcessingImpl::InitializeHighPassFilter(bool forced_reset) {
Per Åhgrenb8106462019-12-04 08:34:12 +01001754 bool high_pass_filter_needed_by_aec =
1755 config_.echo_canceller.enabled &&
1756 config_.echo_canceller.enforce_high_pass_filtering &&
1757 !config_.echo_canceller.mobile_mode;
1758 if (submodule_states_.HighPassFilteringRequired() ||
1759 high_pass_filter_needed_by_aec) {
Per Åhgrenc0424252019-12-10 13:04:15 +01001760 bool use_full_band = config_.high_pass_filter.apply_in_full_band &&
1761 !constants_.enforce_split_band_hpf;
1762 int rate = use_full_band ? proc_fullband_sample_rate_hz()
1763 : proc_split_sample_rate_hz();
1764 size_t num_channels =
1765 use_full_band ? num_output_channels() : num_proc_channels();
1766
Per Åhgren0f14db22020-01-03 14:27:14 +01001767 if (!submodules_.high_pass_filter ||
1768 rate != submodules_.high_pass_filter->sample_rate_hz() ||
1769 forced_reset ||
1770 num_channels != submodules_.high_pass_filter->num_channels()) {
1771 submodules_.high_pass_filter.reset(
1772 new HighPassFilter(rate, num_channels));
1773 }
peah8271d042016-11-22 07:24:52 -08001774 } else {
saza1d600522019-10-18 13:29:43 +02001775 submodules_.high_pass_filter.reset();
peah8271d042016-11-22 07:24:52 -08001776 }
1777}
alessiob3ec96df2017-05-22 06:57:06 -07001778
Sam Zackrisson0824c6f2019-10-07 14:03:56 +02001779void AudioProcessingImpl::InitializeVoiceDetector() {
1780 if (config_.voice_detection.enabled) {
saza1d600522019-10-18 13:29:43 +02001781 submodules_.voice_detector = std::make_unique<VoiceDetection>(
Sam Zackrisson0824c6f2019-10-07 14:03:56 +02001782 proc_split_sample_rate_hz(), VoiceDetection::kVeryLowLikelihood);
1783 } else {
saza1d600522019-10-18 13:29:43 +02001784 submodules_.voice_detector.reset();
Sam Zackrisson0824c6f2019-10-07 14:03:56 +02001785 }
1786}
Gustaf Ullberg8eb9c7d2017-10-14 08:28:46 +02001787void AudioProcessingImpl::InitializeEchoController() {
Per Åhgrenb6e24d72019-04-29 12:14:50 +02001788 bool use_echo_controller =
1789 echo_control_factory_ ||
Per Åhgren62ea0aa2019-12-09 10:18:44 +01001790 (config_.echo_canceller.enabled && !config_.echo_canceller.mobile_mode);
Per Åhgrenb6e24d72019-04-29 12:14:50 +02001791
1792 if (use_echo_controller) {
1793 // Create and activate the echo controller.
Per Åhgren200feba2019-03-06 04:16:46 +01001794 if (echo_control_factory_) {
Per Åhgren4e5c7092019-11-01 20:44:11 +01001795 submodules_.echo_controller = echo_control_factory_->Create(
1796 proc_sample_rate_hz(), num_reverse_channels(), num_proc_channels());
Gustaf Ullberg2c6f3732019-11-07 17:15:12 +01001797 RTC_DCHECK(submodules_.echo_controller);
Per Åhgren200feba2019-03-06 04:16:46 +01001798 } else {
Per Åhgrenb2b58d82019-12-02 14:59:40 +01001799 EchoCanceller3Config config =
1800 use_setup_specific_default_aec3_config_
1801 ? EchoCanceller3::CreateDefaultConfig(num_reverse_channels(),
1802 num_proc_channels())
1803 : EchoCanceller3Config();
saza1d600522019-10-18 13:29:43 +02001804 submodules_.echo_controller = std::make_unique<EchoCanceller3>(
Per Åhgrenb2b58d82019-12-02 14:59:40 +01001805 config, proc_sample_rate_hz(), num_reverse_channels(),
Sam Zackrissonfeee1e42019-09-20 07:50:35 +02001806 num_proc_channels());
Per Åhgren200feba2019-03-06 04:16:46 +01001807 }
1808
Per Åhgrenc20a19c2019-11-13 11:12:29 +01001809 // Setup the storage for returning the linear AEC output.
1810 if (config_.echo_canceller.export_linear_aec_output) {
1811 constexpr int kLinearOutputRateHz = 16000;
1812 capture_.linear_aec_output = std::make_unique<AudioBuffer>(
1813 kLinearOutputRateHz, num_proc_channels(), kLinearOutputRateHz,
1814 num_proc_channels(), kLinearOutputRateHz, num_proc_channels());
1815 } else {
1816 capture_.linear_aec_output.reset();
1817 }
1818
Per Åhgren200feba2019-03-06 04:16:46 +01001819 capture_nonlocked_.echo_controller_enabled = true;
Per Åhgren200feba2019-03-06 04:16:46 +01001820
saza1d600522019-10-18 13:29:43 +02001821 submodules_.echo_control_mobile.reset();
Per Åhgrenb6e24d72019-04-29 12:14:50 +02001822 aecm_render_signal_queue_.reset();
Per Åhgrenf204faf2019-04-25 15:18:06 +02001823 return;
peahe0eae3c2016-12-14 01:16:23 -08001824 }
Per Åhgrenf204faf2019-04-25 15:18:06 +02001825
saza1d600522019-10-18 13:29:43 +02001826 submodules_.echo_controller.reset();
Per Åhgrenf204faf2019-04-25 15:18:06 +02001827 capture_nonlocked_.echo_controller_enabled = false;
Per Åhgrenc20a19c2019-11-13 11:12:29 +01001828 capture_.linear_aec_output.reset();
Per Åhgrenf204faf2019-04-25 15:18:06 +02001829
1830 if (!config_.echo_canceller.enabled) {
saza1d600522019-10-18 13:29:43 +02001831 submodules_.echo_control_mobile.reset();
Per Åhgrenb6e24d72019-04-29 12:14:50 +02001832 aecm_render_signal_queue_.reset();
Per Åhgrenf204faf2019-04-25 15:18:06 +02001833 return;
1834 }
1835
1836 if (config_.echo_canceller.mobile_mode) {
1837 // Create and activate AECM.
Per Åhgrenb6e24d72019-04-29 12:14:50 +02001838 size_t max_element_size =
1839 std::max(static_cast<size_t>(1),
1840 kMaxAllowedValuesOfSamplesPerBand *
1841 EchoControlMobileImpl::NumCancellersRequired(
1842 num_output_channels(), num_reverse_channels()));
1843
1844 std::vector<int16_t> template_queue_element(max_element_size);
1845
1846 aecm_render_signal_queue_.reset(
1847 new SwapQueue<std::vector<int16_t>, RenderQueueItemVerifier<int16_t>>(
1848 kMaxNumFramesToBuffer, template_queue_element,
1849 RenderQueueItemVerifier<int16_t>(max_element_size)));
1850
1851 aecm_render_queue_buffer_.resize(max_element_size);
1852 aecm_capture_queue_buffer_.resize(max_element_size);
1853
saza1d600522019-10-18 13:29:43 +02001854 submodules_.echo_control_mobile.reset(new EchoControlMobileImpl());
Per Åhgrenb6e24d72019-04-29 12:14:50 +02001855
saza1d600522019-10-18 13:29:43 +02001856 submodules_.echo_control_mobile->Initialize(proc_split_sample_rate_hz(),
1857 num_reverse_channels(),
1858 num_output_channels());
Per Åhgrenf204faf2019-04-25 15:18:06 +02001859 return;
1860 }
1861
saza1d600522019-10-18 13:29:43 +02001862 submodules_.echo_control_mobile.reset();
Per Åhgrenb6e24d72019-04-29 12:14:50 +02001863 aecm_render_signal_queue_.reset();
peahe0eae3c2016-12-14 01:16:23 -08001864}
peah8271d042016-11-22 07:24:52 -08001865
Per Åhgren0695df12020-01-13 14:43:13 +01001866void AudioProcessingImpl::InitializeGainController1() {
1867 if (!config_.gain_controller1.enabled) {
1868 submodules_.agc_manager.reset();
1869 submodules_.gain_control.reset();
1870 return;
1871 }
1872
1873 if (!submodules_.gain_control) {
1874 submodules_.gain_control.reset(new GainControlImpl());
1875 }
1876
1877 submodules_.gain_control->Initialize(num_proc_channels(),
1878 proc_sample_rate_hz());
1879
1880 if (!config_.gain_controller1.analog_gain_controller.enabled) {
1881 int error = submodules_.gain_control->set_mode(
1882 Agc1ConfigModeToInterfaceMode(config_.gain_controller1.mode));
1883 RTC_DCHECK_EQ(kNoError, error);
1884 error = submodules_.gain_control->set_target_level_dbfs(
1885 config_.gain_controller1.target_level_dbfs);
1886 RTC_DCHECK_EQ(kNoError, error);
1887 error = submodules_.gain_control->set_compression_gain_db(
1888 config_.gain_controller1.compression_gain_db);
1889 RTC_DCHECK_EQ(kNoError, error);
1890 error = submodules_.gain_control->enable_limiter(
1891 config_.gain_controller1.enable_limiter);
1892 RTC_DCHECK_EQ(kNoError, error);
1893 error = submodules_.gain_control->set_analog_level_limits(
1894 config_.gain_controller1.analog_level_minimum,
1895 config_.gain_controller1.analog_level_maximum);
1896 RTC_DCHECK_EQ(kNoError, error);
1897
1898 submodules_.agc_manager.reset();
1899 return;
1900 }
1901
1902 if (!submodules_.agc_manager.get() ||
1903 submodules_.agc_manager->num_channels() !=
1904 static_cast<int>(num_proc_channels()) ||
1905 submodules_.agc_manager->sample_rate_hz() !=
1906 capture_nonlocked_.split_rate) {
1907 int stream_analog_level = -1;
1908 const bool re_creation = !!submodules_.agc_manager;
1909 if (re_creation) {
1910 stream_analog_level = submodules_.agc_manager->stream_analog_level();
1911 }
1912 submodules_.agc_manager.reset(new AgcManagerDirect(
1913 num_proc_channels(),
1914 config_.gain_controller1.analog_gain_controller.startup_min_volume,
1915 config_.gain_controller1.analog_gain_controller.clipped_level_min,
Per Åhgren0695df12020-01-13 14:43:13 +01001916 !config_.gain_controller1.analog_gain_controller
1917 .enable_digital_adaptive,
Hanna Silenb8dc7fa2021-05-20 17:37:56 +02001918 capture_nonlocked_.split_rate,
1919 config_.gain_controller1.analog_gain_controller.clipped_level_step,
1920 config_.gain_controller1.analog_gain_controller.clipped_ratio_threshold,
Hanna Silena0047152021-06-03 03:29:38 +02001921 config_.gain_controller1.analog_gain_controller.clipped_wait_frames,
1922 config_.gain_controller1.analog_gain_controller.clipping_predictor));
Per Åhgren0695df12020-01-13 14:43:13 +01001923 if (re_creation) {
1924 submodules_.agc_manager->set_stream_analog_level(stream_analog_level);
1925 }
1926 }
1927 submodules_.agc_manager->Initialize();
1928 submodules_.agc_manager->SetupDigitalGainControl(
1929 submodules_.gain_control.get());
Per Åhgren0a144a72021-02-09 08:47:51 +01001930 submodules_.agc_manager->HandleCaptureOutputUsedChange(
1931 capture_.capture_output_used);
Per Åhgren0695df12020-01-13 14:43:13 +01001932}
1933
alessiob3ec96df2017-05-22 06:57:06 -07001934void AudioProcessingImpl::InitializeGainController2() {
Alessio Bazzica270f7b52017-10-13 11:05:17 +02001935 if (config_.gain_controller2.enabled) {
Per Åhgren2bd85ab2020-01-03 10:36:34 +01001936 if (!submodules_.gain_controller2) {
1937 // TODO(alessiob): Move the injected gain controller once injection is
1938 // implemented.
1939 submodules_.gain_controller2.reset(new GainController2());
1940 }
1941
Alessio Bazzicad66a6052021-04-29 16:13:25 +02001942 submodules_.gain_controller2->Initialize(proc_fullband_sample_rate_hz(),
1943 num_input_channels());
Per Åhgren2bd85ab2020-01-03 10:36:34 +01001944 submodules_.gain_controller2->ApplyConfig(config_.gain_controller2);
1945 } else {
1946 submodules_.gain_controller2.reset();
alessiob3ec96df2017-05-22 06:57:06 -07001947 }
1948}
1949
saza0bad15f2019-10-16 11:46:11 +02001950void AudioProcessingImpl::InitializeNoiseSuppressor() {
Per Åhgren0cbb58e2019-10-29 22:59:44 +01001951 submodules_.noise_suppressor.reset();
1952
saza0bad15f2019-10-16 11:46:11 +02001953 if (config_.noise_suppression.enabled) {
sazaaa42ecd2020-04-01 15:24:40 +02001954 auto map_level =
1955 [](AudioProcessing::Config::NoiseSuppression::Level level) {
1956 using NoiseSuppresionConfig =
1957 AudioProcessing::Config::NoiseSuppression;
1958 switch (level) {
1959 case NoiseSuppresionConfig::kLow:
1960 return NsConfig::SuppressionLevel::k6dB;
1961 case NoiseSuppresionConfig::kModerate:
1962 return NsConfig::SuppressionLevel::k12dB;
1963 case NoiseSuppresionConfig::kHigh:
1964 return NsConfig::SuppressionLevel::k18dB;
1965 case NoiseSuppresionConfig::kVeryHigh:
1966 return NsConfig::SuppressionLevel::k21dB;
sazaaa42ecd2020-04-01 15:24:40 +02001967 }
Karl Wibergc95b9392020-11-08 00:49:37 +01001968 RTC_CHECK_NOTREACHED();
sazaaa42ecd2020-04-01 15:24:40 +02001969 };
Per Åhgren0cbb58e2019-10-29 22:59:44 +01001970
sazaaa42ecd2020-04-01 15:24:40 +02001971 NsConfig cfg;
1972 cfg.target_level = map_level(config_.noise_suppression.level);
1973 submodules_.noise_suppressor = std::make_unique<NoiseSuppressor>(
1974 cfg, proc_sample_rate_hz(), num_proc_channels());
saza0bad15f2019-10-16 11:46:11 +02001975 }
1976}
1977
Per Åhgrendb5d7282021-03-15 16:31:04 +00001978void AudioProcessingImpl::InitializeCaptureLevelsAdjuster() {
1979 if (config_.pre_amplifier.enabled ||
1980 config_.capture_level_adjustment.enabled) {
1981 // Use both the pre-amplifier and the capture level adjustment gains as
1982 // pre-gains.
1983 float pre_gain = 1.f;
1984 if (config_.pre_amplifier.enabled) {
1985 pre_gain *= config_.pre_amplifier.fixed_gain_factor;
1986 }
1987 if (config_.capture_level_adjustment.enabled) {
1988 pre_gain *= config_.capture_level_adjustment.pre_gain_factor;
1989 }
1990
1991 submodules_.capture_levels_adjuster =
1992 std::make_unique<CaptureLevelsAdjuster>(
1993 config_.capture_level_adjustment.analog_mic_gain_emulation.enabled,
1994 config_.capture_level_adjustment.analog_mic_gain_emulation
1995 .initial_level,
1996 pre_gain, config_.capture_level_adjustment.post_gain_factor);
Alex Loikob5c9a792018-04-16 16:31:22 +02001997 } else {
Per Åhgrendb5d7282021-03-15 16:31:04 +00001998 submodules_.capture_levels_adjuster.reset();
Alex Loikob5c9a792018-04-16 16:31:22 +02001999 }
2000}
2001
ivoc9f4a4a02016-10-28 05:39:16 -07002002void AudioProcessingImpl::InitializeResidualEchoDetector() {
saza1d600522019-10-18 13:29:43 +02002003 RTC_DCHECK(submodules_.echo_detector);
2004 submodules_.echo_detector->Initialize(
Gustaf Ullberg422b9e02019-10-09 13:02:14 +02002005 proc_fullband_sample_rate_hz(), 1,
Ivo Creusenb1facc12018-04-12 16:15:58 +02002006 formats_.render_processing_format.sample_rate_hz(), 1);
ivoc9f4a4a02016-10-28 05:39:16 -07002007}
2008
Valeriia Nemychnikovaf06eb572018-08-29 10:37:09 +02002009void AudioProcessingImpl::InitializeAnalyzer() {
saza1d600522019-10-18 13:29:43 +02002010 if (submodules_.capture_analyzer) {
2011 submodules_.capture_analyzer->Initialize(proc_fullband_sample_rate_hz(),
2012 num_proc_channels());
Valeriia Nemychnikovaf06eb572018-08-29 10:37:09 +02002013 }
2014}
2015
Sam Zackrisson0beac582017-09-25 12:04:02 +02002016void AudioProcessingImpl::InitializePostProcessor() {
saza1d600522019-10-18 13:29:43 +02002017 if (submodules_.capture_post_processor) {
2018 submodules_.capture_post_processor->Initialize(
Gustaf Ullberg422b9e02019-10-09 13:02:14 +02002019 proc_fullband_sample_rate_hz(), num_proc_channels());
Sam Zackrisson0beac582017-09-25 12:04:02 +02002020 }
2021}
2022
Alex Loiko5825aa62017-12-18 16:02:40 +01002023void AudioProcessingImpl::InitializePreProcessor() {
saza1d600522019-10-18 13:29:43 +02002024 if (submodules_.render_pre_processor) {
2025 submodules_.render_pre_processor->Initialize(
Alex Loiko5825aa62017-12-18 16:02:40 +01002026 formats_.render_processing_format.sample_rate_hz(),
2027 formats_.render_processing_format.num_channels());
2028 }
2029}
2030
aleloi868f32f2017-05-23 07:20:05 -07002031void AudioProcessingImpl::WriteAecDumpConfigMessage(bool forced) {
2032 if (!aec_dump_) {
2033 return;
2034 }
Per Åhgrenf204faf2019-04-25 15:18:06 +02002035
2036 std::string experiments_description = "";
aleloi868f32f2017-05-23 07:20:05 -07002037 // TODO(peah): Add semicolon-separated concatenations of experiment
2038 // descriptions for other submodules.
Per Åhgren0695df12020-01-13 14:43:13 +01002039 if (config_.gain_controller1.analog_gain_controller.clipped_level_min !=
2040 kClippedLevelMin) {
aleloi868f32f2017-05-23 07:20:05 -07002041 experiments_description += "AgcClippingLevelExperiment;";
2042 }
Sam Zackrisson701bd172020-02-18 14:50:28 +01002043 if (!!submodules_.capture_post_processor) {
2044 experiments_description += "CapturePostProcessor;";
2045 }
2046 if (!!submodules_.render_pre_processor) {
2047 experiments_description += "RenderPreProcessor;";
2048 }
Gustaf Ullbergce045ac2017-10-16 13:49:04 +02002049 if (capture_nonlocked_.echo_controller_enabled) {
2050 experiments_description += "EchoController;";
aleloi868f32f2017-05-23 07:20:05 -07002051 }
Alessio Bazzica270f7b52017-10-13 11:05:17 +02002052 if (config_.gain_controller2.enabled) {
2053 experiments_description += "GainController2;";
2054 }
aleloi868f32f2017-05-23 07:20:05 -07002055
2056 InternalAPMConfig apm_config;
2057
Per Åhgren200feba2019-03-06 04:16:46 +01002058 apm_config.aec_enabled = config_.echo_canceller.enabled;
Per Åhgren62ea0aa2019-12-09 10:18:44 +01002059 apm_config.aec_delay_agnostic_enabled = false;
2060 apm_config.aec_extended_filter_enabled = false;
2061 apm_config.aec_suppression_level = 0;
aleloi868f32f2017-05-23 07:20:05 -07002062
saza1d600522019-10-18 13:29:43 +02002063 apm_config.aecm_enabled = !!submodules_.echo_control_mobile;
aleloi868f32f2017-05-23 07:20:05 -07002064 apm_config.aecm_comfort_noise_enabled =
saza1d600522019-10-18 13:29:43 +02002065 submodules_.echo_control_mobile &&
2066 submodules_.echo_control_mobile->is_comfort_noise_enabled();
Per Åhgrenb6e24d72019-04-29 12:14:50 +02002067 apm_config.aecm_routing_mode =
saza1d600522019-10-18 13:29:43 +02002068 submodules_.echo_control_mobile
2069 ? static_cast<int>(submodules_.echo_control_mobile->routing_mode())
Per Åhgrenb6e24d72019-04-29 12:14:50 +02002070 : 0;
aleloi868f32f2017-05-23 07:20:05 -07002071
Per Åhgren0695df12020-01-13 14:43:13 +01002072 apm_config.agc_enabled = !!submodules_.gain_control;
2073
2074 apm_config.agc_mode = submodules_.gain_control
2075 ? static_cast<int>(submodules_.gain_control->mode())
2076 : GainControl::kAdaptiveAnalog;
aleloi868f32f2017-05-23 07:20:05 -07002077 apm_config.agc_limiter_enabled =
Per Åhgren0695df12020-01-13 14:43:13 +01002078 submodules_.gain_control ? submodules_.gain_control->is_limiter_enabled()
2079 : false;
Per Åhgren0e3198e2019-11-18 08:52:22 +01002080 apm_config.noise_robust_agc_enabled = !!submodules_.agc_manager;
aleloi868f32f2017-05-23 07:20:05 -07002081
2082 apm_config.hpf_enabled = config_.high_pass_filter.enabled;
2083
saza0bad15f2019-10-16 11:46:11 +02002084 apm_config.ns_enabled = config_.noise_suppression.enabled;
2085 apm_config.ns_level = static_cast<int>(config_.noise_suppression.level);
aleloi868f32f2017-05-23 07:20:05 -07002086
2087 apm_config.transient_suppression_enabled =
Per Åhgrenc0734712020-01-02 15:15:36 +01002088 config_.transient_suppression.enabled;
aleloi868f32f2017-05-23 07:20:05 -07002089 apm_config.experiments_description = experiments_description;
Alex Loiko5feb30e2018-04-16 13:52:32 +02002090 apm_config.pre_amplifier_enabled = config_.pre_amplifier.enabled;
2091 apm_config.pre_amplifier_fixed_gain_factor =
2092 config_.pre_amplifier.fixed_gain_factor;
aleloi868f32f2017-05-23 07:20:05 -07002093
2094 if (!forced && apm_config == apm_config_for_aec_dump_) {
2095 return;
2096 }
2097 aec_dump_->WriteConfig(apm_config);
2098 apm_config_for_aec_dump_ = apm_config;
2099}
2100
2101void AudioProcessingImpl::RecordUnprocessedCaptureStream(
2102 const float* const* src) {
2103 RTC_DCHECK(aec_dump_);
2104 WriteAecDumpConfigMessage(false);
2105
2106 const size_t channel_size = formats_.api_format.input_stream().num_frames();
2107 const size_t num_channels = formats_.api_format.input_stream().num_channels();
2108 aec_dump_->AddCaptureStreamInput(
Alex Loikoe36e8bb2018-02-16 11:54:07 +01002109 AudioFrameView<const float>(src, num_channels, channel_size));
aleloi868f32f2017-05-23 07:20:05 -07002110 RecordAudioProcessingState();
2111}
2112
2113void AudioProcessingImpl::RecordUnprocessedCaptureStream(
Per Åhgren645f24c2020-03-16 12:06:02 +01002114 const int16_t* const data,
2115 const StreamConfig& config) {
aleloi868f32f2017-05-23 07:20:05 -07002116 RTC_DCHECK(aec_dump_);
2117 WriteAecDumpConfigMessage(false);
2118
Per Åhgren645f24c2020-03-16 12:06:02 +01002119 aec_dump_->AddCaptureStreamInput(data, config.num_channels(),
2120 config.num_frames());
aleloi868f32f2017-05-23 07:20:05 -07002121 RecordAudioProcessingState();
2122}
2123
2124void AudioProcessingImpl::RecordProcessedCaptureStream(
2125 const float* const* processed_capture_stream) {
2126 RTC_DCHECK(aec_dump_);
2127
2128 const size_t channel_size = formats_.api_format.output_stream().num_frames();
2129 const size_t num_channels =
2130 formats_.api_format.output_stream().num_channels();
Alex Loikoe36e8bb2018-02-16 11:54:07 +01002131 aec_dump_->AddCaptureStreamOutput(AudioFrameView<const float>(
2132 processed_capture_stream, num_channels, channel_size));
aleloi868f32f2017-05-23 07:20:05 -07002133 aec_dump_->WriteCaptureStreamMessage();
2134}
2135
2136void AudioProcessingImpl::RecordProcessedCaptureStream(
Per Åhgren645f24c2020-03-16 12:06:02 +01002137 const int16_t* const data,
2138 const StreamConfig& config) {
aleloi868f32f2017-05-23 07:20:05 -07002139 RTC_DCHECK(aec_dump_);
2140
Per Åhgren645f24c2020-03-16 12:06:02 +01002141 aec_dump_->AddCaptureStreamOutput(data, config.num_channels(),
Per Åhgren088329f2020-03-18 21:59:52 +01002142 config.num_frames());
aleloi868f32f2017-05-23 07:20:05 -07002143 aec_dump_->WriteCaptureStreamMessage();
2144}
2145
2146void AudioProcessingImpl::RecordAudioProcessingState() {
2147 RTC_DCHECK(aec_dump_);
2148 AecDump::AudioProcessingState audio_proc_state;
2149 audio_proc_state.delay = capture_nonlocked_.stream_delay_ms;
Per Åhgren62ea0aa2019-12-09 10:18:44 +01002150 audio_proc_state.drift = 0;
Markus Handell02ba1d22020-05-14 14:31:18 +02002151 audio_proc_state.level = recommended_stream_analog_level_locked();
aleloi868f32f2017-05-23 07:20:05 -07002152 audio_proc_state.keypress = capture_.key_pressed;
2153 aec_dump_->AddAudioProcessingState(audio_proc_state);
2154}
2155
Per Åhgrenc0734712020-01-02 15:15:36 +01002156AudioProcessingImpl::ApmCaptureState::ApmCaptureState()
Sam Zackrisson2d31aea2020-01-17 10:55:09 +01002157 : was_stream_delay_set(false),
Per Åhgren0a144a72021-02-09 08:47:51 +01002158 capture_output_used(true),
Per Åhgren55bc0772021-03-12 14:18:36 +00002159 capture_output_used_last_frame(true),
kwiberg83ffe452016-08-29 14:46:07 -07002160 key_pressed(false),
peahde65ddc2016-09-16 15:02:15 -07002161 capture_processing_format(kSampleRate16kHz),
peah67995532017-04-10 14:12:41 -07002162 split_rate(kSampleRate16kHz),
Per Åhgren88cf0502018-07-16 17:08:41 +02002163 echo_path_gain_change(false),
Per Åhgrend2650d12018-10-02 17:00:59 +02002164 prev_analog_mic_level(-1),
Per Åhgrendb5d7282021-03-15 16:31:04 +00002165 prev_pre_adjustment_gain(-1.f),
Fredrik Hernqvistca362852019-05-10 15:50:02 +02002166 playout_volume(-1),
2167 prev_playout_volume(-1) {}
kwiberg83ffe452016-08-29 14:46:07 -07002168
2169AudioProcessingImpl::ApmCaptureState::~ApmCaptureState() = default;
2170
Per Åhgrena1351272019-08-15 12:15:46 +02002171void AudioProcessingImpl::ApmCaptureState::KeyboardInfo::Extract(
2172 const float* const* data,
2173 const StreamConfig& stream_config) {
2174 if (stream_config.has_keyboard()) {
2175 keyboard_data = data[stream_config.num_channels()];
2176 } else {
2177 keyboard_data = NULL;
2178 }
2179 num_keyboard_frames = stream_config.num_frames();
2180}
2181
kwiberg83ffe452016-08-29 14:46:07 -07002182AudioProcessingImpl::ApmRenderState::ApmRenderState() = default;
2183
2184AudioProcessingImpl::ApmRenderState::~ApmRenderState() = default;
2185
Per Åhgrencf4c8722019-12-30 14:32:14 +01002186AudioProcessingImpl::ApmStatsReporter::ApmStatsReporter()
2187 : stats_message_queue_(1) {}
2188
2189AudioProcessingImpl::ApmStatsReporter::~ApmStatsReporter() = default;
2190
2191AudioProcessingStats AudioProcessingImpl::ApmStatsReporter::GetStatistics() {
Markus Handell0df0fae2020-07-07 15:53:34 +02002192 MutexLock lock_stats(&mutex_stats_);
Per Åhgrencf4c8722019-12-30 14:32:14 +01002193 bool new_stats_available = stats_message_queue_.Remove(&cached_stats_);
2194 // If the message queue is full, return the cached stats.
2195 static_cast<void>(new_stats_available);
2196
2197 return cached_stats_;
2198}
2199
2200void AudioProcessingImpl::ApmStatsReporter::UpdateStatistics(
2201 const AudioProcessingStats& new_stats) {
2202 AudioProcessingStats stats_to_queue = new_stats;
2203 bool stats_message_passed = stats_message_queue_.Insert(&stats_to_queue);
2204 // If the message queue is full, discard the new stats.
2205 static_cast<void>(stats_message_passed);
2206}
2207
niklase@google.com470e71d2011-07-07 08:21:25 +00002208} // namespace webrtc