blob: 8a3ddf5ba6c7eb751a17c908fbeda7e52a76e1b0 [file] [log] [blame]
niklase@google.com470e71d2011-07-07 08:21:25 +00001/*
andrew@webrtc.org40654032012-01-30 20:51:15 +00002 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
niklase@google.com470e71d2011-07-07 08:21:25 +00003 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020011#include "modules/audio_processing/audio_processing_impl.h"
niklase@google.com470e71d2011-07-07 08:21:25 +000012
Michael Graczyk86c6d332015-07-23 11:41:39 -070013#include <algorithm>
Yves Gerey988cc082018-10-23 12:03:01 +020014#include <cstdint>
Mirko Bonadei317a1f02019-09-17 17:06:18 +020015#include <memory>
alessiob3ec96df2017-05-22 06:57:06 -070016#include <string>
Yves Gerey988cc082018-10-23 12:03:01 +020017#include <type_traits>
18#include <utility>
niklase@google.com470e71d2011-07-07 08:21:25 +000019
Yves Gerey988cc082018-10-23 12:03:01 +020020#include "absl/types/optional.h"
21#include "api/array_view.h"
Per Åhgren645f24c2020-03-16 12:06:02 +010022#include "api/audio/audio_frame.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020023#include "common_audio/audio_converter.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020024#include "common_audio/include/audio_util.h"
Per Åhgren09e9a832020-05-11 11:03:47 +020025#include "modules/audio_processing/aec_dump/aec_dump_factory.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020026#include "modules/audio_processing/audio_buffer.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020027#include "modules/audio_processing/common.h"
Yves Gerey988cc082018-10-23 12:03:01 +020028#include "modules/audio_processing/include/audio_frame_view.h"
Per Åhgren13735822018-02-12 21:42:56 +010029#include "modules/audio_processing/logging/apm_data_dumper.h"
Sam Zackrissonb37e59d2020-04-27 08:39:33 +020030#include "modules/audio_processing/optionally_built_submodule_creators.h"
Steve Anton10542f22019-01-11 09:11:00 -080031#include "rtc_base/atomic_ops.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020032#include "rtc_base/checks.h"
Steve Anton10542f22019-01-11 09:11:00 -080033#include "rtc_base/constructor_magic.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020034#include "rtc_base/logging.h"
Steve Anton10542f22019-01-11 09:11:00 -080035#include "rtc_base/ref_counted_object.h"
36#include "rtc_base/time_utils.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020037#include "rtc_base/trace_event.h"
Alessio Bazzica0441bb62021-08-10 15:23:23 +020038#include "system_wrappers/include/denormal_disabler.h"
Sam Zackrissonfeee1e42019-09-20 07:50:35 +020039#include "system_wrappers/include/field_trial.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020040#include "system_wrappers/include/metrics.h"
andrew@webrtc.org7bf26462011-12-03 00:03:31 +000041
Michael Graczyk86c6d332015-07-23 11:41:39 -070042#define RETURN_ON_ERR(expr) \
43 do { \
44 int err = (expr); \
45 if (err != kNoError) { \
46 return err; \
47 } \
andrew@webrtc.org60730cf2014-01-07 17:45:09 +000048 } while (0)
49
niklase@google.com470e71d2011-07-07 08:21:25 +000050namespace webrtc {
aluebsdf6416a2016-03-16 18:26:35 -070051
Michael Graczyk86c6d332015-07-23 11:41:39 -070052namespace {
53
54static bool LayoutHasKeyboard(AudioProcessing::ChannelLayout layout) {
55 switch (layout) {
56 case AudioProcessing::kMono:
57 case AudioProcessing::kStereo:
58 return false;
59 case AudioProcessing::kMonoAndKeyboard:
60 case AudioProcessing::kStereoAndKeyboard:
61 return true;
62 }
63
Artem Titovd3251962021-11-15 16:57:07 +010064 RTC_DCHECK_NOTREACHED();
Michael Graczyk86c6d332015-07-23 11:41:39 -070065 return false;
66}
aluebsdf6416a2016-03-16 18:26:35 -070067
peah2ace3f92016-09-10 04:42:27 -070068bool SampleRateSupportsMultiBand(int sample_rate_hz) {
aluebsdf6416a2016-03-16 18:26:35 -070069 return sample_rate_hz == AudioProcessing::kSampleRate32kHz ||
70 sample_rate_hz == AudioProcessing::kSampleRate48kHz;
71}
72
Per Åhgrenc0424252019-12-10 13:04:15 +010073// Checks whether the high-pass filter should be done in the full-band.
74bool EnforceSplitBandHpf() {
75 return field_trial::IsEnabled("WebRTC-FullBandHpfKillSwitch");
76}
77
Per Åhgrenb2b58d82019-12-02 14:59:40 +010078// Checks whether AEC3 should be allowed to decide what the default
79// configuration should be based on the render and capture channel configuration
80// at hand.
81bool UseSetupSpecificDefaultAec3Congfig() {
82 return !field_trial::IsEnabled(
83 "WebRTC-Aec3SetupSpecificDefaultConfigDefaultsKillSwitch");
84}
85
Per Åhgrenc8626b62019-08-23 15:49:51 +020086// Identify the native processing rate that best handles a sample rate.
Per Åhgrenfcbe4072019-09-15 00:27:58 +020087int SuitableProcessRate(int minimum_rate,
88 int max_splitting_rate,
89 bool band_splitting_required) {
Per Åhgrenc8626b62019-08-23 15:49:51 +020090 const int uppermost_native_rate =
Per Åhgrenfcbe4072019-09-15 00:27:58 +020091 band_splitting_required ? max_splitting_rate : 48000;
Per Åhgrenc8626b62019-08-23 15:49:51 +020092 for (auto rate : {16000, 32000, 48000}) {
peah2ace3f92016-09-10 04:42:27 -070093 if (rate >= uppermost_native_rate) {
94 return uppermost_native_rate;
95 }
96 if (rate >= minimum_rate) {
aluebsdf6416a2016-03-16 18:26:35 -070097 return rate;
98 }
99 }
Artem Titovd3251962021-11-15 16:57:07 +0100100 RTC_DCHECK_NOTREACHED();
peah2ace3f92016-09-10 04:42:27 -0700101 return uppermost_native_rate;
aluebsdf6416a2016-03-16 18:26:35 -0700102}
103
Sam Zackrissonf0d1c032019-03-27 13:28:08 +0100104GainControl::Mode Agc1ConfigModeToInterfaceMode(
105 AudioProcessing::Config::GainController1::Mode mode) {
106 using Agc1Config = AudioProcessing::Config::GainController1;
107 switch (mode) {
108 case Agc1Config::kAdaptiveAnalog:
109 return GainControl::kAdaptiveAnalog;
110 case Agc1Config::kAdaptiveDigital:
111 return GainControl::kAdaptiveDigital;
112 case Agc1Config::kFixedDigital:
113 return GainControl::kFixedDigital;
114 }
Karl Wibergc95b9392020-11-08 00:49:37 +0100115 RTC_CHECK_NOTREACHED();
Sam Zackrissonf0d1c032019-03-27 13:28:08 +0100116}
117
Per Åhgren19775cb2021-03-12 23:08:09 +0000118bool MinimizeProcessingForUnusedOutput() {
119 return !field_trial::IsEnabled("WebRTC-MutedStateKillSwitch");
120}
121
peah9e6a2902017-05-15 07:19:21 -0700122// Maximum lengths that frame of samples being passed from the render side to
123// the capture side can have (does not apply to AEC3).
124static const size_t kMaxAllowedValuesOfSamplesPerBand = 160;
125static const size_t kMaxAllowedValuesOfSamplesPerFrame = 480;
126
peah764e3642016-10-22 05:04:30 -0700127// Maximum number of frames to buffer in the render queue.
128// TODO(peah): Decrease this once we properly handle hugely unbalanced
129// reverse and forward call numbers.
130static const size_t kMaxNumFramesToBuffer = 100;
Alessio Bazzica3438a932020-10-14 12:47:50 +0200131
Michael Graczyk86c6d332015-07-23 11:41:39 -0700132} // namespace
andrew@webrtc.org60730cf2014-01-07 17:45:09 +0000133
134// Throughout webrtc, it's assumed that success is represented by zero.
kwiberg@webrtc.org2ebfac52015-01-14 10:51:54 +0000135static_assert(AudioProcessing::kNoError == 0, "kNoError must be zero");
andrew@webrtc.org60730cf2014-01-07 17:45:09 +0000136
saza1d600522019-10-18 13:29:43 +0200137AudioProcessingImpl::SubmoduleStates::SubmoduleStates(
Alex Loiko5825aa62017-12-18 16:02:40 +0100138 bool capture_post_processor_enabled,
Valeriia Nemychnikovaf06eb572018-08-29 10:37:09 +0200139 bool render_pre_processor_enabled,
140 bool capture_analyzer_enabled)
Alex Loiko5825aa62017-12-18 16:02:40 +0100141 : capture_post_processor_enabled_(capture_post_processor_enabled),
Valeriia Nemychnikovaf06eb572018-08-29 10:37:09 +0200142 render_pre_processor_enabled_(render_pre_processor_enabled),
143 capture_analyzer_enabled_(capture_analyzer_enabled) {}
peah2ace3f92016-09-10 04:42:27 -0700144
saza1d600522019-10-18 13:29:43 +0200145bool AudioProcessingImpl::SubmoduleStates::Update(
Sam Zackrissoncb1b5562018-09-28 14:15:09 +0200146 bool high_pass_filter_enabled,
peah2ace3f92016-09-10 04:42:27 -0700147 bool mobile_echo_controller_enabled,
ivoc9f4a4a02016-10-28 05:39:16 -0700148 bool residual_echo_detector_enabled,
peah2ace3f92016-09-10 04:42:27 -0700149 bool noise_suppressor_enabled,
peah2ace3f92016-09-10 04:42:27 -0700150 bool adaptive_gain_controller_enabled,
alessiob3ec96df2017-05-22 06:57:06 -0700151 bool gain_controller2_enabled,
Per Åhgrendb5d7282021-03-15 16:31:04 +0000152 bool gain_adjustment_enabled,
Gustaf Ullbergce045ac2017-10-16 13:49:04 +0200153 bool echo_controller_enabled,
Sam Zackrisson0824c6f2019-10-07 14:03:56 +0200154 bool voice_detector_enabled,
peah2ace3f92016-09-10 04:42:27 -0700155 bool transient_suppressor_enabled) {
156 bool changed = false;
Sam Zackrissoncb1b5562018-09-28 14:15:09 +0200157 changed |= (high_pass_filter_enabled != high_pass_filter_enabled_);
peah2ace3f92016-09-10 04:42:27 -0700158 changed |=
159 (mobile_echo_controller_enabled != mobile_echo_controller_enabled_);
ivoc9f4a4a02016-10-28 05:39:16 -0700160 changed |=
161 (residual_echo_detector_enabled != residual_echo_detector_enabled_);
peah2ace3f92016-09-10 04:42:27 -0700162 changed |= (noise_suppressor_enabled != noise_suppressor_enabled_);
163 changed |=
peah2ace3f92016-09-10 04:42:27 -0700164 (adaptive_gain_controller_enabled != adaptive_gain_controller_enabled_);
Per Åhgren6ee75fd2019-04-26 11:33:37 +0200165 changed |= (gain_controller2_enabled != gain_controller2_enabled_);
Per Åhgrendb5d7282021-03-15 16:31:04 +0000166 changed |= (gain_adjustment_enabled != gain_adjustment_enabled_);
Gustaf Ullbergce045ac2017-10-16 13:49:04 +0200167 changed |= (echo_controller_enabled != echo_controller_enabled_);
Sam Zackrisson0824c6f2019-10-07 14:03:56 +0200168 changed |= (voice_detector_enabled != voice_detector_enabled_);
peah2ace3f92016-09-10 04:42:27 -0700169 changed |= (transient_suppressor_enabled != transient_suppressor_enabled_);
170 if (changed) {
Sam Zackrissoncb1b5562018-09-28 14:15:09 +0200171 high_pass_filter_enabled_ = high_pass_filter_enabled;
peah2ace3f92016-09-10 04:42:27 -0700172 mobile_echo_controller_enabled_ = mobile_echo_controller_enabled;
ivoc9f4a4a02016-10-28 05:39:16 -0700173 residual_echo_detector_enabled_ = residual_echo_detector_enabled;
peah2ace3f92016-09-10 04:42:27 -0700174 noise_suppressor_enabled_ = noise_suppressor_enabled;
peah2ace3f92016-09-10 04:42:27 -0700175 adaptive_gain_controller_enabled_ = adaptive_gain_controller_enabled;
alessiob3ec96df2017-05-22 06:57:06 -0700176 gain_controller2_enabled_ = gain_controller2_enabled;
Per Åhgrendb5d7282021-03-15 16:31:04 +0000177 gain_adjustment_enabled_ = gain_adjustment_enabled;
Gustaf Ullbergce045ac2017-10-16 13:49:04 +0200178 echo_controller_enabled_ = echo_controller_enabled;
Sam Zackrisson0824c6f2019-10-07 14:03:56 +0200179 voice_detector_enabled_ = voice_detector_enabled;
peah2ace3f92016-09-10 04:42:27 -0700180 transient_suppressor_enabled_ = transient_suppressor_enabled;
181 }
182
183 changed |= first_update_;
184 first_update_ = false;
185 return changed;
186}
187
saza1d600522019-10-18 13:29:43 +0200188bool AudioProcessingImpl::SubmoduleStates::CaptureMultiBandSubModulesActive()
peah2ace3f92016-09-10 04:42:27 -0700189 const {
Gustaf Ullberg8675eee2019-10-09 13:34:36 +0200190 return CaptureMultiBandProcessingPresent() || voice_detector_enabled_;
peah2ace3f92016-09-10 04:42:27 -0700191}
192
saza1d600522019-10-18 13:29:43 +0200193bool AudioProcessingImpl::SubmoduleStates::CaptureMultiBandProcessingPresent()
194 const {
Gustaf Ullberg8675eee2019-10-09 13:34:36 +0200195 // If echo controller is present, assume it performs active processing.
196 return CaptureMultiBandProcessingActive(/*ec_processing_active=*/true);
197}
198
saza1d600522019-10-18 13:29:43 +0200199bool AudioProcessingImpl::SubmoduleStates::CaptureMultiBandProcessingActive(
Gustaf Ullberg8675eee2019-10-09 13:34:36 +0200200 bool ec_processing_active) const {
Per Åhgren62ea0aa2019-12-09 10:18:44 +0100201 return high_pass_filter_enabled_ || mobile_echo_controller_enabled_ ||
202 noise_suppressor_enabled_ || adaptive_gain_controller_enabled_ ||
Gustaf Ullberg8675eee2019-10-09 13:34:36 +0200203 (echo_controller_enabled_ && ec_processing_active);
peah2ace3f92016-09-10 04:42:27 -0700204}
205
saza1d600522019-10-18 13:29:43 +0200206bool AudioProcessingImpl::SubmoduleStates::CaptureFullBandProcessingActive()
peah23ac8b42017-05-23 05:33:56 -0700207 const {
Alex Loikob5c9a792018-04-16 16:31:22 +0200208 return gain_controller2_enabled_ || capture_post_processor_enabled_ ||
Per Åhgrendb5d7282021-03-15 16:31:04 +0000209 gain_adjustment_enabled_;
peah23ac8b42017-05-23 05:33:56 -0700210}
211
saza1d600522019-10-18 13:29:43 +0200212bool AudioProcessingImpl::SubmoduleStates::CaptureAnalyzerActive() const {
Valeriia Nemychnikovaf06eb572018-08-29 10:37:09 +0200213 return capture_analyzer_enabled_;
214}
215
saza1d600522019-10-18 13:29:43 +0200216bool AudioProcessingImpl::SubmoduleStates::RenderMultiBandSubModulesActive()
peah2ace3f92016-09-10 04:42:27 -0700217 const {
Per Åhgren62ea0aa2019-12-09 10:18:44 +0100218 return RenderMultiBandProcessingActive() || mobile_echo_controller_enabled_ ||
219 adaptive_gain_controller_enabled_ || echo_controller_enabled_;
peah2ace3f92016-09-10 04:42:27 -0700220}
221
saza1d600522019-10-18 13:29:43 +0200222bool AudioProcessingImpl::SubmoduleStates::RenderFullBandProcessingActive()
Alex Loiko5825aa62017-12-18 16:02:40 +0100223 const {
224 return render_pre_processor_enabled_;
225}
226
saza1d600522019-10-18 13:29:43 +0200227bool AudioProcessingImpl::SubmoduleStates::RenderMultiBandProcessingActive()
peah2ace3f92016-09-10 04:42:27 -0700228 const {
peah2ace3f92016-09-10 04:42:27 -0700229 return false;
peah2ace3f92016-09-10 04:42:27 -0700230}
231
saza1d600522019-10-18 13:29:43 +0200232bool AudioProcessingImpl::SubmoduleStates::HighPassFilteringRequired() const {
Per Åhgren62ea0aa2019-12-09 10:18:44 +0100233 return high_pass_filter_enabled_ || mobile_echo_controller_enabled_ ||
234 noise_suppressor_enabled_;
Sam Zackrissoncb1b5562018-09-28 14:15:09 +0200235}
236
Alessio Bazzicabe1b8982021-09-17 08:26:10 +0200237AudioProcessingImpl::AudioProcessingImpl()
Alessio Bazzica20a9ac62021-10-14 10:55:08 +0200238 : AudioProcessingImpl(/*config=*/{},
239 /*capture_post_processor=*/nullptr,
Sam Zackrissonfeee1e42019-09-20 07:50:35 +0200240 /*render_pre_processor=*/nullptr,
241 /*echo_control_factory=*/nullptr,
242 /*echo_detector=*/nullptr,
243 /*capture_analyzer=*/nullptr) {}
aluebs@webrtc.orgd82f55d2015-01-15 18:07:21 +0000244
Per Åhgren13735822018-02-12 21:42:56 +0100245int AudioProcessingImpl::instance_count_ = 0;
246
Sam Zackrisson0beac582017-09-25 12:04:02 +0200247AudioProcessingImpl::AudioProcessingImpl(
Alessio Bazzica20a9ac62021-10-14 10:55:08 +0200248 const AudioProcessing::Config& config,
Alex Loiko5825aa62017-12-18 16:02:40 +0100249 std::unique_ptr<CustomProcessing> capture_post_processor,
250 std::unique_ptr<CustomProcessing> render_pre_processor,
Gustaf Ullberg002ef282017-10-12 15:13:17 +0200251 std::unique_ptr<EchoControlFactory> echo_control_factory,
Valeriia Nemychnikovaf06eb572018-08-29 10:37:09 +0200252 rtc::scoped_refptr<EchoDetector> echo_detector,
253 std::unique_ptr<CustomAudioAnalyzer> capture_analyzer)
Per Åhgren13735822018-02-12 21:42:56 +0100254 : data_dumper_(
255 new ApmDataDumper(rtc::AtomicOps::Increment(&instance_count_))),
Per Åhgrenb2b58d82019-12-02 14:59:40 +0100256 use_setup_specific_default_aec3_config_(
257 UseSetupSpecificDefaultAec3Congfig()),
Alessio Bazzica0441bb62021-08-10 15:23:23 +0200258 use_denormal_disabler_(
259 !field_trial::IsEnabled("WebRTC-ApmDenormalDisablerKillSwitch")),
Per Åhgren652ada52021-03-03 10:52:44 +0000260 capture_runtime_settings_(RuntimeSettingQueueSize()),
261 render_runtime_settings_(RuntimeSettingQueueSize()),
Alex Loiko73ec0192018-05-15 10:52:28 +0200262 capture_runtime_settings_enqueuer_(&capture_runtime_settings_),
263 render_runtime_settings_enqueuer_(&render_runtime_settings_),
Gustaf Ullberg002ef282017-10-12 15:13:17 +0200264 echo_control_factory_(std::move(echo_control_factory)),
Alessio Bazzica20a9ac62021-10-14 10:55:08 +0200265 config_(config),
Valeriia Nemychnikovaf06eb572018-08-29 10:37:09 +0200266 submodule_states_(!!capture_post_processor,
267 !!render_pre_processor,
268 !!capture_analyzer),
saza1d600522019-10-18 13:29:43 +0200269 submodules_(std::move(capture_post_processor),
270 std::move(render_pre_processor),
271 std::move(echo_detector),
Per Åhgren3daedb62019-11-22 12:11:40 +0100272 std::move(capture_analyzer)),
Per Åhgren0695df12020-01-13 14:43:13 +0100273 constants_(!field_trial::IsEnabled(
Sam Zackrissonfeee1e42019-09-20 07:50:35 +0200274 "WebRTC-ApmExperimentalMultiChannelRenderKillSwitch"),
275 !field_trial::IsEnabled(
Per Åhgrenc0424252019-12-10 13:04:15 +0100276 "WebRTC-ApmExperimentalMultiChannelCaptureKillSwitch"),
Per Åhgren19775cb2021-03-12 23:08:09 +0000277 EnforceSplitBandHpf(),
Gustaf Ullberga399c822021-05-18 12:17:56 +0200278 MinimizeProcessingForUnusedOutput(),
279 field_trial::IsEnabled("WebRTC-TransientSuppressorForcedOff")),
Per Åhgren19775cb2021-03-12 23:08:09 +0000280 capture_(),
Alessio Bazzicacc22f512018-08-30 13:01:34 +0200281 capture_nonlocked_() {
Sam Zackrisson72cc71c2019-10-21 12:54:02 +0200282 RTC_LOG(LS_INFO) << "Injected APM submodules:"
Jonas Olssonb2b20312020-01-14 12:11:31 +0100283 "\nEcho control factory: "
284 << !!echo_control_factory_
Sam Zackrisson72cc71c2019-10-21 12:54:02 +0200285 << "\nEcho detector: " << !!submodules_.echo_detector
286 << "\nCapture analyzer: " << !!submodules_.capture_analyzer
287 << "\nCapture post processor: "
288 << !!submodules_.capture_post_processor
289 << "\nRender pre processor: "
290 << !!submodules_.render_pre_processor;
Alessio Bazzica0441bb62021-08-10 15:23:23 +0200291 RTC_LOG(LS_INFO) << "Denormal disabler: "
292 << (DenormalDisabler::IsSupported() ? "supported"
293 : "unsupported");
Sam Zackrisson72cc71c2019-10-21 12:54:02 +0200294
Sam Zackrisson421c8592019-02-11 13:39:46 +0100295 // Mark Echo Controller enabled if a factory is injected.
296 capture_nonlocked_.echo_controller_enabled =
297 static_cast<bool>(echo_control_factory_);
niklase@google.com470e71d2011-07-07 08:21:25 +0000298
Sam Zackrisson421c8592019-02-11 13:39:46 +0100299 // If no echo detector is injected, use the ResidualEchoDetector.
saza1d600522019-10-18 13:29:43 +0200300 if (!submodules_.echo_detector) {
Tommi87f70902021-04-27 14:43:08 +0200301 submodules_.echo_detector = rtc::make_ref_counted<ResidualEchoDetector>();
peahdf3efa82015-11-28 12:35:15 -0800302 }
pbos@webrtc.org788acd12014-12-15 09:41:24 +0000303
Per Åhgren0ade9832020-09-01 23:57:20 +0200304 Initialize();
niklase@google.com470e71d2011-07-07 08:21:25 +0000305}
306
Per Åhgren0e3198e2019-11-18 08:52:22 +0100307AudioProcessingImpl::~AudioProcessingImpl() = default;
niklase@google.com470e71d2011-07-07 08:21:25 +0000308
niklase@google.com470e71d2011-07-07 08:21:25 +0000309int AudioProcessingImpl::Initialize() {
peahdf3efa82015-11-28 12:35:15 -0800310 // Run in a single-threaded manner during initialization.
Markus Handell0df0fae2020-07-07 15:53:34 +0200311 MutexLock lock_render(&mutex_render_);
312 MutexLock lock_capture(&mutex_capture_);
Per Åhgren0ade9832020-09-01 23:57:20 +0200313 InitializeLocked();
314 return kNoError;
niklase@google.com470e71d2011-07-07 08:21:25 +0000315}
316
peahde65ddc2016-09-16 15:02:15 -0700317int AudioProcessingImpl::Initialize(int capture_input_sample_rate_hz,
318 int capture_output_sample_rate_hz,
319 int render_input_sample_rate_hz,
320 ChannelLayout capture_input_layout,
321 ChannelLayout capture_output_layout,
322 ChannelLayout render_input_layout) {
Michael Graczyk86c6d332015-07-23 11:41:39 -0700323 const ProcessingConfig processing_config = {
peahde65ddc2016-09-16 15:02:15 -0700324 {{capture_input_sample_rate_hz, ChannelsFromLayout(capture_input_layout),
325 LayoutHasKeyboard(capture_input_layout)},
326 {capture_output_sample_rate_hz,
327 ChannelsFromLayout(capture_output_layout),
328 LayoutHasKeyboard(capture_output_layout)},
329 {render_input_sample_rate_hz, ChannelsFromLayout(render_input_layout),
330 LayoutHasKeyboard(render_input_layout)},
331 {render_input_sample_rate_hz, ChannelsFromLayout(render_input_layout),
332 LayoutHasKeyboard(render_input_layout)}}};
Michael Graczyk86c6d332015-07-23 11:41:39 -0700333
334 return Initialize(processing_config);
335}
336
337int AudioProcessingImpl::Initialize(const ProcessingConfig& processing_config) {
peahdf3efa82015-11-28 12:35:15 -0800338 // Run in a single-threaded manner during initialization.
Markus Handell0df0fae2020-07-07 15:53:34 +0200339 MutexLock lock_render(&mutex_render_);
340 MutexLock lock_capture(&mutex_capture_);
Michael Graczyk86c6d332015-07-23 11:41:39 -0700341 return InitializeLocked(processing_config);
andrew@webrtc.orga8b97372014-03-10 22:26:12 +0000342}
343
peahdf3efa82015-11-28 12:35:15 -0800344int AudioProcessingImpl::MaybeInitializeRender(
peah81b9bfe2015-11-27 02:47:28 -0800345 const ProcessingConfig& processing_config) {
peahdf3efa82015-11-28 12:35:15 -0800346 // Called from both threads. Thread check is therefore not possible.
Oskar Sundbom4b276482019-05-23 14:28:00 +0200347 if (processing_config == formats_.api_format) {
peah192164e2015-11-17 02:16:45 -0800348 return kNoError;
349 }
peahdf3efa82015-11-28 12:35:15 -0800350
Markus Handell0df0fae2020-07-07 15:53:34 +0200351 MutexLock lock_capture(&mutex_capture_);
peah192164e2015-11-17 02:16:45 -0800352 return InitializeLocked(processing_config);
353}
354
Per Åhgren0ade9832020-09-01 23:57:20 +0200355void AudioProcessingImpl::InitializeLocked() {
Per Åhgren4bdced52017-06-27 16:00:38 +0200356 UpdateActiveSubmoduleStates();
357
Per Åhgrend47941e2019-08-22 11:51:13 +0200358 const int render_audiobuffer_sample_rate_hz =
peahdf3efa82015-11-28 12:35:15 -0800359 formats_.api_format.reverse_output_stream().num_frames() == 0
Per Åhgrend47941e2019-08-22 11:51:13 +0200360 ? formats_.render_processing_format.sample_rate_hz()
361 : formats_.api_format.reverse_output_stream().sample_rate_hz();
peahdf3efa82015-11-28 12:35:15 -0800362 if (formats_.api_format.reverse_input_stream().num_channels() > 0) {
363 render_.render_audio.reset(new AudioBuffer(
Per Åhgrend47941e2019-08-22 11:51:13 +0200364 formats_.api_format.reverse_input_stream().sample_rate_hz(),
peahdf3efa82015-11-28 12:35:15 -0800365 formats_.api_format.reverse_input_stream().num_channels(),
Per Åhgrend47941e2019-08-22 11:51:13 +0200366 formats_.render_processing_format.sample_rate_hz(),
peahde65ddc2016-09-16 15:02:15 -0700367 formats_.render_processing_format.num_channels(),
Per Åhgrend47941e2019-08-22 11:51:13 +0200368 render_audiobuffer_sample_rate_hz,
369 formats_.render_processing_format.num_channels()));
peah2ace3f92016-09-10 04:42:27 -0700370 if (formats_.api_format.reverse_input_stream() !=
371 formats_.api_format.reverse_output_stream()) {
kwibergc2b785d2016-02-24 05:22:32 -0800372 render_.render_converter = AudioConverter::Create(
peahdf3efa82015-11-28 12:35:15 -0800373 formats_.api_format.reverse_input_stream().num_channels(),
374 formats_.api_format.reverse_input_stream().num_frames(),
375 formats_.api_format.reverse_output_stream().num_channels(),
kwibergc2b785d2016-02-24 05:22:32 -0800376 formats_.api_format.reverse_output_stream().num_frames());
ekmeyerson60d9b332015-08-14 10:35:55 -0700377 } else {
peahdf3efa82015-11-28 12:35:15 -0800378 render_.render_converter.reset(nullptr);
ekmeyerson60d9b332015-08-14 10:35:55 -0700379 }
Michael Graczyk86c6d332015-07-23 11:41:39 -0700380 } else {
peahdf3efa82015-11-28 12:35:15 -0800381 render_.render_audio.reset(nullptr);
382 render_.render_converter.reset(nullptr);
Michael Graczyk86c6d332015-07-23 11:41:39 -0700383 }
peahce4d9152017-05-19 01:28:05 -0700384
Per Åhgrend47941e2019-08-22 11:51:13 +0200385 capture_.capture_audio.reset(new AudioBuffer(
386 formats_.api_format.input_stream().sample_rate_hz(),
387 formats_.api_format.input_stream().num_channels(),
388 capture_nonlocked_.capture_processing_format.sample_rate_hz(),
389 formats_.api_format.output_stream().num_channels(),
390 formats_.api_format.output_stream().sample_rate_hz(),
391 formats_.api_format.output_stream().num_channels()));
niklase@google.com470e71d2011-07-07 08:21:25 +0000392
Gustaf Ullberg422b9e02019-10-09 13:02:14 +0200393 if (capture_nonlocked_.capture_processing_format.sample_rate_hz() <
394 formats_.api_format.output_stream().sample_rate_hz() &&
395 formats_.api_format.output_stream().sample_rate_hz() == 48000) {
396 capture_.capture_fullband_audio.reset(
397 new AudioBuffer(formats_.api_format.input_stream().sample_rate_hz(),
398 formats_.api_format.input_stream().num_channels(),
399 formats_.api_format.output_stream().sample_rate_hz(),
400 formats_.api_format.output_stream().num_channels(),
401 formats_.api_format.output_stream().sample_rate_hz(),
402 formats_.api_format.output_stream().num_channels()));
403 } else {
404 capture_.capture_fullband_audio.reset();
405 }
406
peah764e3642016-10-22 05:04:30 -0700407 AllocateRenderQueue();
408
Per Åhgren0695df12020-01-13 14:43:13 +0100409 InitializeGainController1();
Per Åhgrenc0734712020-01-02 15:15:36 +0100410 InitializeTransientSuppressor();
Per Åhgren0f14db22020-01-03 14:27:14 +0100411 InitializeHighPassFilter(true);
Sam Zackrisson0824c6f2019-10-07 14:03:56 +0200412 InitializeVoiceDetector();
ivoc9f4a4a02016-10-28 05:39:16 -0700413 InitializeResidualEchoDetector();
Gustaf Ullberg8eb9c7d2017-10-14 08:28:46 +0200414 InitializeEchoController();
Alessio Bazzica38901042021-10-14 12:14:21 +0200415 InitializeGainController2(/*config_has_changed=*/true);
saza0bad15f2019-10-16 11:46:11 +0200416 InitializeNoiseSuppressor();
Valeriia Nemychnikovaf06eb572018-08-29 10:37:09 +0200417 InitializeAnalyzer();
Sam Zackrisson0beac582017-09-25 12:04:02 +0200418 InitializePostProcessor();
Alex Loiko5825aa62017-12-18 16:02:40 +0100419 InitializePreProcessor();
Per Åhgrendb5d7282021-03-15 16:31:04 +0000420 InitializeCaptureLevelsAdjuster();
solenberg70f99032015-12-08 11:07:32 -0800421
aleloi868f32f2017-05-23 07:20:05 -0700422 if (aec_dump_) {
Minyue Li656d6092018-08-10 15:38:52 +0200423 aec_dump_->WriteInitMessage(formats_.api_format, rtc::TimeUTCMillis());
aleloi868f32f2017-05-23 07:20:05 -0700424 }
niklase@google.com470e71d2011-07-07 08:21:25 +0000425}
426
Michael Graczyk86c6d332015-07-23 11:41:39 -0700427int AudioProcessingImpl::InitializeLocked(const ProcessingConfig& config) {
Per Åhgren4bdced52017-06-27 16:00:38 +0200428 UpdateActiveSubmoduleStates();
429
Michael Graczyk86c6d332015-07-23 11:41:39 -0700430 for (const auto& stream : config.streams) {
Michael Graczyk86c6d332015-07-23 11:41:39 -0700431 if (stream.num_channels() > 0 && stream.sample_rate_hz() <= 0) {
432 return kBadSampleRateError;
433 }
andrew@webrtc.orga8b97372014-03-10 22:26:12 +0000434 }
Michael Graczyk86c6d332015-07-23 11:41:39 -0700435
Peter Kasting69558702016-01-12 16:26:35 -0800436 const size_t num_in_channels = config.input_stream().num_channels();
437 const size_t num_out_channels = config.output_stream().num_channels();
Michael Graczyk86c6d332015-07-23 11:41:39 -0700438
439 // Need at least one input channel.
440 // Need either one output channel or as many outputs as there are inputs.
441 if (num_in_channels == 0 ||
442 !(num_out_channels == 1 || num_out_channels == num_in_channels)) {
Michael Graczykc2047542015-07-22 21:06:11 -0700443 return kBadNumberChannelsError;
444 }
445
peahdf3efa82015-11-28 12:35:15 -0800446 formats_.api_format = config;
andrew@webrtc.orgddbb8a22014-04-22 21:00:04 +0000447
Per Åhgrenfcbe4072019-09-15 00:27:58 +0200448 // Choose maximum rate to use for the split filtering.
449 RTC_DCHECK(config_.pipeline.maximum_internal_processing_rate == 48000 ||
450 config_.pipeline.maximum_internal_processing_rate == 32000);
451 int max_splitting_rate = 48000;
452 if (config_.pipeline.maximum_internal_processing_rate == 32000) {
453 max_splitting_rate = config_.pipeline.maximum_internal_processing_rate;
454 }
455
Per Åhgrenc8626b62019-08-23 15:49:51 +0200456 int capture_processing_rate = SuitableProcessRate(
peah423d2362016-04-09 16:06:52 -0700457 std::min(formats_.api_format.input_stream().sample_rate_hz(),
peah2ace3f92016-09-10 04:42:27 -0700458 formats_.api_format.output_stream().sample_rate_hz()),
Per Åhgrenfcbe4072019-09-15 00:27:58 +0200459 max_splitting_rate,
peah2ace3f92016-09-10 04:42:27 -0700460 submodule_states_.CaptureMultiBandSubModulesActive() ||
461 submodule_states_.RenderMultiBandSubModulesActive());
Per Åhgrenc8626b62019-08-23 15:49:51 +0200462 RTC_DCHECK_NE(8000, capture_processing_rate);
andrew@webrtc.orga8b97372014-03-10 22:26:12 +0000463
peahde65ddc2016-09-16 15:02:15 -0700464 capture_nonlocked_.capture_processing_format =
465 StreamConfig(capture_processing_rate);
peah2ace3f92016-09-10 04:42:27 -0700466
peah2ce640f2017-04-07 03:57:48 -0700467 int render_processing_rate;
Gustaf Ullbergbd83b912017-10-18 12:32:42 +0200468 if (!capture_nonlocked_.echo_controller_enabled) {
Per Åhgrenc8626b62019-08-23 15:49:51 +0200469 render_processing_rate = SuitableProcessRate(
peah2ce640f2017-04-07 03:57:48 -0700470 std::min(formats_.api_format.reverse_input_stream().sample_rate_hz(),
471 formats_.api_format.reverse_output_stream().sample_rate_hz()),
Per Åhgrenfcbe4072019-09-15 00:27:58 +0200472 max_splitting_rate,
peah2ce640f2017-04-07 03:57:48 -0700473 submodule_states_.CaptureMultiBandSubModulesActive() ||
474 submodule_states_.RenderMultiBandSubModulesActive());
475 } else {
476 render_processing_rate = capture_processing_rate;
477 }
478
peahde65ddc2016-09-16 15:02:15 -0700479 // If the forward sample rate is 8 kHz, the render stream is also processed
aluebseb3603b2016-04-20 15:27:58 -0700480 // at this rate.
peahde65ddc2016-09-16 15:02:15 -0700481 if (capture_nonlocked_.capture_processing_format.sample_rate_hz() ==
482 kSampleRate8kHz) {
483 render_processing_rate = kSampleRate8kHz;
andrew@webrtc.orga8b97372014-03-10 22:26:12 +0000484 } else {
peahde65ddc2016-09-16 15:02:15 -0700485 render_processing_rate =
486 std::max(render_processing_rate, static_cast<int>(kSampleRate16kHz));
andrew@webrtc.orgddbb8a22014-04-22 21:00:04 +0000487 }
488
Per Åhgrenc8626b62019-08-23 15:49:51 +0200489 RTC_DCHECK_NE(8000, render_processing_rate);
490
peahce4d9152017-05-19 01:28:05 -0700491 if (submodule_states_.RenderMultiBandSubModulesActive()) {
Sam Zackrissonfeee1e42019-09-20 07:50:35 +0200492 // By default, downmix the render stream to mono for analysis. This has been
493 // demonstrated to work well for AEC in most practical scenarios.
Per Åhgrene14cb992019-11-27 09:34:22 +0100494 const bool multi_channel_render = config_.pipeline.multi_channel_render &&
495 constants_.multi_channel_render_support;
Sam Zackrissonfeee1e42019-09-20 07:50:35 +0200496 int render_processing_num_channels =
Per Åhgrene14cb992019-11-27 09:34:22 +0100497 multi_channel_render
Sam Zackrissonfeee1e42019-09-20 07:50:35 +0200498 ? formats_.api_format.reverse_input_stream().num_channels()
499 : 1;
500 formats_.render_processing_format =
501 StreamConfig(render_processing_rate, render_processing_num_channels);
peahce4d9152017-05-19 01:28:05 -0700502 } else {
503 formats_.render_processing_format = StreamConfig(
504 formats_.api_format.reverse_input_stream().sample_rate_hz(),
505 formats_.api_format.reverse_input_stream().num_channels());
506 }
andrew@webrtc.orgddbb8a22014-04-22 21:00:04 +0000507
peahde65ddc2016-09-16 15:02:15 -0700508 if (capture_nonlocked_.capture_processing_format.sample_rate_hz() ==
509 kSampleRate32kHz ||
510 capture_nonlocked_.capture_processing_format.sample_rate_hz() ==
511 kSampleRate48kHz) {
peahdf3efa82015-11-28 12:35:15 -0800512 capture_nonlocked_.split_rate = kSampleRate16kHz;
andrew@webrtc.orgddbb8a22014-04-22 21:00:04 +0000513 } else {
peahdf3efa82015-11-28 12:35:15 -0800514 capture_nonlocked_.split_rate =
peahde65ddc2016-09-16 15:02:15 -0700515 capture_nonlocked_.capture_processing_format.sample_rate_hz();
andrew@webrtc.orga8b97372014-03-10 22:26:12 +0000516 }
517
Per Åhgren0ade9832020-09-01 23:57:20 +0200518 InitializeLocked();
519 return kNoError;
andrew@webrtc.orga8b97372014-03-10 22:26:12 +0000520}
521
peah88ac8532016-09-12 16:47:25 -0700522void AudioProcessingImpl::ApplyConfig(const AudioProcessing::Config& config) {
Sam Zackrisson72cc71c2019-10-21 12:54:02 +0200523 RTC_LOG(LS_INFO) << "AudioProcessing::ApplyConfig: " << config.ToString();
524
peah88ac8532016-09-12 16:47:25 -0700525 // Run in a single-threaded manner when applying the settings.
Markus Handell0df0fae2020-07-07 15:53:34 +0200526 MutexLock lock_render(&mutex_render_);
527 MutexLock lock_capture(&mutex_capture_);
peah88ac8532016-09-12 16:47:25 -0700528
Sam Zackrissonfeee1e42019-09-20 07:50:35 +0200529 const bool pipeline_config_changed =
Per Åhgrene14cb992019-11-27 09:34:22 +0100530 config_.pipeline.multi_channel_render !=
531 config.pipeline.multi_channel_render ||
532 config_.pipeline.multi_channel_capture !=
Per Åhgrenc0424252019-12-10 13:04:15 +0100533 config.pipeline.multi_channel_capture ||
534 config_.pipeline.maximum_internal_processing_rate !=
535 config.pipeline.maximum_internal_processing_rate;
Sam Zackrissonfeee1e42019-09-20 07:50:35 +0200536
Per Åhgren200feba2019-03-06 04:16:46 +0100537 const bool aec_config_changed =
538 config_.echo_canceller.enabled != config.echo_canceller.enabled ||
Per Åhgren62ea0aa2019-12-09 10:18:44 +0100539 config_.echo_canceller.mobile_mode != config.echo_canceller.mobile_mode;
Per Åhgren200feba2019-03-06 04:16:46 +0100540
Sam Zackrissonf0d1c032019-03-27 13:28:08 +0100541 const bool agc1_config_changed =
Alessio Bazzica3438a932020-10-14 12:47:50 +0200542 config_.gain_controller1 != config.gain_controller1;
Sam Zackrissonf0d1c032019-03-27 13:28:08 +0100543
Per Åhgren2bd85ab2020-01-03 10:36:34 +0100544 const bool agc2_config_changed =
Alessio Bazzica3438a932020-10-14 12:47:50 +0200545 config_.gain_controller2 != config.gain_controller2;
Per Åhgren2bd85ab2020-01-03 10:36:34 +0100546
Sam Zackrisson0824c6f2019-10-07 14:03:56 +0200547 const bool voice_detection_config_changed =
548 config_.voice_detection.enabled != config.voice_detection.enabled;
549
saza0bad15f2019-10-16 11:46:11 +0200550 const bool ns_config_changed =
551 config_.noise_suppression.enabled != config.noise_suppression.enabled ||
552 config_.noise_suppression.level != config.noise_suppression.level;
553
Per Åhgrenc0734712020-01-02 15:15:36 +0100554 const bool ts_config_changed = config_.transient_suppression.enabled !=
555 config.transient_suppression.enabled;
556
Per Åhgren0f14db22020-01-03 14:27:14 +0100557 const bool pre_amplifier_config_changed =
558 config_.pre_amplifier.enabled != config.pre_amplifier.enabled ||
559 config_.pre_amplifier.fixed_gain_factor !=
560 config.pre_amplifier.fixed_gain_factor;
561
Per Åhgrendb5d7282021-03-15 16:31:04 +0000562 const bool gain_adjustment_config_changed =
563 config_.capture_level_adjustment != config.capture_level_adjustment;
564
Yves Gerey499bc6c2018-10-10 18:29:07 +0200565 config_ = config;
566
Per Åhgren200feba2019-03-06 04:16:46 +0100567 if (aec_config_changed) {
568 InitializeEchoController();
569 }
Sam Zackrissoncdf0e6d2018-09-17 11:05:17 +0200570
saza0bad15f2019-10-16 11:46:11 +0200571 if (ns_config_changed) {
572 InitializeNoiseSuppressor();
573 }
Sam Zackrisson23513132019-01-11 15:10:32 +0100574
Per Åhgrenc0734712020-01-02 15:15:36 +0100575 if (ts_config_changed) {
576 InitializeTransientSuppressor();
577 }
578
Per Åhgren0f14db22020-01-03 14:27:14 +0100579 InitializeHighPassFilter(false);
peah8271d042016-11-22 07:24:52 -0800580
Sam Zackrissonf0d1c032019-03-27 13:28:08 +0100581 if (agc1_config_changed) {
Per Åhgren0695df12020-01-13 14:43:13 +0100582 InitializeGainController1();
Sam Zackrissonf0d1c032019-03-27 13:28:08 +0100583 }
584
Sam Zackrissonab1aee02018-03-05 15:59:06 +0100585 const bool config_ok = GainController2::Validate(config_.gain_controller2);
alessiob3ec96df2017-05-22 06:57:06 -0700586 if (!config_ok) {
Alessio Bazzica0c83e152020-10-14 12:49:54 +0200587 RTC_LOG(LS_ERROR)
588 << "Invalid Gain Controller 2 config; using the default config.";
alessiob3ec96df2017-05-22 06:57:06 -0700589 config_.gain_controller2 = AudioProcessing::Config::GainController2();
590 }
Per Åhgren0f14db22020-01-03 14:27:14 +0100591
Alessio Bazzica38901042021-10-14 12:14:21 +0200592 InitializeGainController2(agc2_config_changed);
Per Åhgren0f14db22020-01-03 14:27:14 +0100593
Per Åhgrendb5d7282021-03-15 16:31:04 +0000594 if (pre_amplifier_config_changed || gain_adjustment_config_changed) {
595 InitializeCaptureLevelsAdjuster();
Per Åhgren0f14db22020-01-03 14:27:14 +0100596 }
Sam Zackrissonb24c00f2018-11-26 16:18:25 +0100597
Sam Zackrisson0824c6f2019-10-07 14:03:56 +0200598 if (voice_detection_config_changed) {
599 InitializeVoiceDetector();
Sam Zackrisson4db667b2018-12-21 16:29:27 +0100600 }
Sam Zackrissonfeee1e42019-09-20 07:50:35 +0200601
602 // Reinitialization must happen after all submodule configuration to avoid
603 // additional reinitializations on the next capture / render processing call.
604 if (pipeline_config_changed) {
605 InitializeLocked(formats_.api_format);
606 }
peah88ac8532016-09-12 16:47:25 -0700607}
608
Sam Zackrissonb37e59d2020-04-27 08:39:33 +0200609void AudioProcessingImpl::OverrideSubmoduleCreationForTesting(
610 const ApmSubmoduleCreationOverrides& overrides) {
Markus Handell0df0fae2020-07-07 15:53:34 +0200611 MutexLock lock(&mutex_capture_);
Sam Zackrissonb37e59d2020-04-27 08:39:33 +0200612 submodule_creation_overrides_ = overrides;
613}
614
andrew@webrtc.orgddbb8a22014-04-22 21:00:04 +0000615int AudioProcessingImpl::proc_sample_rate_hz() const {
peahdf3efa82015-11-28 12:35:15 -0800616 // Used as callback from submodules, hence locking is not allowed.
peahde65ddc2016-09-16 15:02:15 -0700617 return capture_nonlocked_.capture_processing_format.sample_rate_hz();
niklase@google.com470e71d2011-07-07 08:21:25 +0000618}
619
Gustaf Ullberg422b9e02019-10-09 13:02:14 +0200620int AudioProcessingImpl::proc_fullband_sample_rate_hz() const {
621 return capture_.capture_fullband_audio
622 ? capture_.capture_fullband_audio->num_frames() * 100
623 : capture_nonlocked_.capture_processing_format.sample_rate_hz();
624}
625
andrew@webrtc.orgddbb8a22014-04-22 21:00:04 +0000626int AudioProcessingImpl::proc_split_sample_rate_hz() const {
peahdf3efa82015-11-28 12:35:15 -0800627 // Used as callback from submodules, hence locking is not allowed.
628 return capture_nonlocked_.split_rate;
niklase@google.com470e71d2011-07-07 08:21:25 +0000629}
630
Peter Kasting69558702016-01-12 16:26:35 -0800631size_t AudioProcessingImpl::num_reverse_channels() const {
peahdf3efa82015-11-28 12:35:15 -0800632 // Used as callback from submodules, hence locking is not allowed.
peahde65ddc2016-09-16 15:02:15 -0700633 return formats_.render_processing_format.num_channels();
niklase@google.com470e71d2011-07-07 08:21:25 +0000634}
635
Peter Kasting69558702016-01-12 16:26:35 -0800636size_t AudioProcessingImpl::num_input_channels() const {
peahdf3efa82015-11-28 12:35:15 -0800637 // Used as callback from submodules, hence locking is not allowed.
638 return formats_.api_format.input_stream().num_channels();
niklase@google.com470e71d2011-07-07 08:21:25 +0000639}
640
Peter Kasting69558702016-01-12 16:26:35 -0800641size_t AudioProcessingImpl::num_proc_channels() const {
aluebsb2328d12016-01-11 20:32:29 -0800642 // Used as callback from submodules, hence locking is not allowed.
Per Åhgrene14cb992019-11-27 09:34:22 +0100643 const bool multi_channel_capture = config_.pipeline.multi_channel_capture &&
644 constants_.multi_channel_capture_support;
645 if (capture_nonlocked_.echo_controller_enabled && !multi_channel_capture) {
Sam Zackrissonfeee1e42019-09-20 07:50:35 +0200646 return 1;
647 }
648 return num_output_channels();
aluebsb2328d12016-01-11 20:32:29 -0800649}
650
Peter Kasting69558702016-01-12 16:26:35 -0800651size_t AudioProcessingImpl::num_output_channels() const {
peahdf3efa82015-11-28 12:35:15 -0800652 // Used as callback from submodules, hence locking is not allowed.
653 return formats_.api_format.output_stream().num_channels();
niklase@google.com470e71d2011-07-07 08:21:25 +0000654}
655
andrew@webrtc.org17342e52014-02-12 22:28:31 +0000656void AudioProcessingImpl::set_output_will_be_muted(bool muted) {
Markus Handell0df0fae2020-07-07 15:53:34 +0200657 MutexLock lock(&mutex_capture_);
Per Åhgren0a144a72021-02-09 08:47:51 +0100658 HandleCaptureOutputUsedSetting(!muted);
659}
660
661void AudioProcessingImpl::HandleCaptureOutputUsedSetting(
662 bool capture_output_used) {
Per Åhgren19775cb2021-03-12 23:08:09 +0000663 capture_.capture_output_used =
664 capture_output_used || !constants_.minimize_processing_for_unused_output;
665
saza1d600522019-10-18 13:29:43 +0200666 if (submodules_.agc_manager.get()) {
Per Åhgren0a144a72021-02-09 08:47:51 +0100667 submodules_.agc_manager->HandleCaptureOutputUsedChange(
668 capture_.capture_output_used);
pbos@webrtc.org788acd12014-12-15 09:41:24 +0000669 }
Per Åhgren652ada52021-03-03 10:52:44 +0000670 if (submodules_.echo_controller) {
671 submodules_.echo_controller->SetCaptureOutputUsage(
672 capture_.capture_output_used);
673 }
Per Åhgren15179a92021-03-12 14:12:44 +0000674 if (submodules_.noise_suppressor) {
675 submodules_.noise_suppressor->SetCaptureOutputUsage(
676 capture_.capture_output_used);
677 }
andrew@webrtc.org17342e52014-02-12 22:28:31 +0000678}
679
Alessio Bazzicac054e782018-04-16 12:10:09 +0200680void AudioProcessingImpl::SetRuntimeSetting(RuntimeSetting setting) {
Per Åhgren0a144a72021-02-09 08:47:51 +0100681 PostRuntimeSetting(setting);
682}
683
684bool AudioProcessingImpl::PostRuntimeSetting(RuntimeSetting setting) {
Alex Loiko73ec0192018-05-15 10:52:28 +0200685 switch (setting.type()) {
686 case RuntimeSetting::Type::kCustomRenderProcessingRuntimeSetting:
Alessio Bazzica7c19a702019-11-07 13:22:00 +0100687 case RuntimeSetting::Type::kPlayoutAudioDeviceChange:
Per Åhgren0a144a72021-02-09 08:47:51 +0100688 return render_runtime_settings_enqueuer_.Enqueue(setting);
Alex Loiko73ec0192018-05-15 10:52:28 +0200689 case RuntimeSetting::Type::kCapturePreGain:
Per Åhgrendb5d7282021-03-15 16:31:04 +0000690 case RuntimeSetting::Type::kCapturePostGain:
Sam Zackrissonf0d1c032019-03-27 13:28:08 +0100691 case RuntimeSetting::Type::kCaptureCompressionGain:
Per Åhgren6ee75fd2019-04-26 11:33:37 +0200692 case RuntimeSetting::Type::kCaptureFixedPostGain:
Per Åhgren552d3e32020-08-12 08:46:47 +0200693 case RuntimeSetting::Type::kCaptureOutputUsed:
Per Åhgren0a144a72021-02-09 08:47:51 +0100694 return capture_runtime_settings_enqueuer_.Enqueue(setting);
695 case RuntimeSetting::Type::kPlayoutVolumeChange: {
696 bool enqueueing_successful;
697 enqueueing_successful =
698 capture_runtime_settings_enqueuer_.Enqueue(setting);
699 enqueueing_successful =
700 render_runtime_settings_enqueuer_.Enqueue(setting) &&
701 enqueueing_successful;
702 return enqueueing_successful;
703 }
Alessio Bazzica7c19a702019-11-07 13:22:00 +0100704 case RuntimeSetting::Type::kNotSpecified:
Artem Titovd3251962021-11-15 16:57:07 +0100705 RTC_DCHECK_NOTREACHED();
Per Åhgren0a144a72021-02-09 08:47:51 +0100706 return true;
Alex Loiko73ec0192018-05-15 10:52:28 +0200707 }
708 // The language allows the enum to have a non-enumerator
709 // value. Check that this doesn't happen.
Artem Titovd3251962021-11-15 16:57:07 +0100710 RTC_DCHECK_NOTREACHED();
Per Åhgren0a144a72021-02-09 08:47:51 +0100711 return true;
Alessio Bazzicac054e782018-04-16 12:10:09 +0200712}
713
714AudioProcessingImpl::RuntimeSettingEnqueuer::RuntimeSettingEnqueuer(
715 SwapQueue<RuntimeSetting>* runtime_settings)
Alessio Bazzica33444dc2018-04-20 13:16:55 +0200716 : runtime_settings_(*runtime_settings) {
717 RTC_DCHECK(runtime_settings);
Alessio Bazzicac054e782018-04-16 12:10:09 +0200718}
719
720AudioProcessingImpl::RuntimeSettingEnqueuer::~RuntimeSettingEnqueuer() =
721 default;
722
Per Åhgren0a144a72021-02-09 08:47:51 +0100723bool AudioProcessingImpl::RuntimeSettingEnqueuer::Enqueue(
Alessio Bazzicac054e782018-04-16 12:10:09 +0200724 RuntimeSetting setting) {
Per Åhgren652ada52021-03-03 10:52:44 +0000725 const bool successful_insert = runtime_settings_.Insert(&setting);
726
727 if (!successful_insert) {
Alessio Bazzica0b10f462020-10-23 12:40:30 +0200728 RTC_HISTOGRAM_BOOLEAN("WebRTC.Audio.ApmRuntimeSettingCannotEnqueue", 1);
Alessio Bazzicac054e782018-04-16 12:10:09 +0200729 RTC_LOG(LS_ERROR) << "Cannot enqueue a new runtime setting.";
Alessio Bazzica0b10f462020-10-23 12:40:30 +0200730 }
Per Åhgren652ada52021-03-03 10:52:44 +0000731 return successful_insert;
Alessio Bazzicac054e782018-04-16 12:10:09 +0200732}
andrew@webrtc.org17342e52014-02-12 22:28:31 +0000733
Sam Zackrisson12e319a2020-01-03 14:54:20 +0100734int AudioProcessingImpl::MaybeInitializeCapture(
735 const StreamConfig& input_config,
736 const StreamConfig& output_config) {
peahdf3efa82015-11-28 12:35:15 -0800737 ProcessingConfig processing_config;
peah2ace3f92016-09-10 04:42:27 -0700738 bool reinitialization_required = false;
peahdf3efa82015-11-28 12:35:15 -0800739 {
Sam Zackrisson12e319a2020-01-03 14:54:20 +0100740 // Acquire the capture lock in order to access api_format. The lock is
741 // released immediately, as we may need to acquire the render lock as part
742 // of the conditional reinitialization.
Markus Handell0df0fae2020-07-07 15:53:34 +0200743 MutexLock lock_capture(&mutex_capture_);
peahdf3efa82015-11-28 12:35:15 -0800744 processing_config = formats_.api_format;
peah2ace3f92016-09-10 04:42:27 -0700745 reinitialization_required = UpdateActiveSubmoduleStates();
niklase@google.com470e71d2011-07-07 08:21:25 +0000746 }
andrew@webrtc.org17e40642014-03-04 20:58:13 +0000747
Oskar Sundbom4b276482019-05-23 14:28:00 +0200748 if (processing_config.input_stream() != input_config) {
749 processing_config.input_stream() = input_config;
750 reinitialization_required = true;
peahdf3efa82015-11-28 12:35:15 -0800751 }
Oskar Sundbom4b276482019-05-23 14:28:00 +0200752
753 if (processing_config.output_stream() != output_config) {
754 processing_config.output_stream() = output_config;
755 reinitialization_required = true;
756 }
757
758 if (reinitialization_required) {
Markus Handell0df0fae2020-07-07 15:53:34 +0200759 MutexLock lock_render(&mutex_render_);
760 MutexLock lock_capture(&mutex_capture_);
Oskar Sundbom4b276482019-05-23 14:28:00 +0200761 RETURN_ON_ERR(InitializeLocked(processing_config));
762 }
Sam Zackrisson12e319a2020-01-03 14:54:20 +0100763 return kNoError;
764}
765
766int AudioProcessingImpl::ProcessStream(const float* const* src,
767 const StreamConfig& input_config,
768 const StreamConfig& output_config,
769 float* const* dest) {
770 TRACE_EVENT0("webrtc", "AudioProcessing::ProcessStream_StreamConfig");
771 if (!src || !dest) {
772 return kNullPointerError;
773 }
774
775 RETURN_ON_ERR(MaybeInitializeCapture(input_config, output_config));
Oskar Sundbom4b276482019-05-23 14:28:00 +0200776
Markus Handell0df0fae2020-07-07 15:53:34 +0200777 MutexLock lock_capture(&mutex_capture_);
Alessio Bazzica0441bb62021-08-10 15:23:23 +0200778 DenormalDisabler denormal_disabler(use_denormal_disabler_);
andrew@webrtc.org17e40642014-03-04 20:58:13 +0000779
aleloi868f32f2017-05-23 07:20:05 -0700780 if (aec_dump_) {
781 RecordUnprocessedCaptureStream(src);
782 }
783
Per Åhgrena1351272019-08-15 12:15:46 +0200784 capture_.keyboard_info.Extract(src, formats_.api_format.input_stream());
peahdf3efa82015-11-28 12:35:15 -0800785 capture_.capture_audio->CopyFrom(src, formats_.api_format.input_stream());
Gustaf Ullberg8675eee2019-10-09 13:34:36 +0200786 if (capture_.capture_fullband_audio) {
787 capture_.capture_fullband_audio->CopyFrom(
788 src, formats_.api_format.input_stream());
789 }
peahde65ddc2016-09-16 15:02:15 -0700790 RETURN_ON_ERR(ProcessCaptureStreamLocked());
Gustaf Ullberg422b9e02019-10-09 13:02:14 +0200791 if (capture_.capture_fullband_audio) {
792 capture_.capture_fullband_audio->CopyTo(formats_.api_format.output_stream(),
793 dest);
794 } else {
795 capture_.capture_audio->CopyTo(formats_.api_format.output_stream(), dest);
796 }
andrew@webrtc.org17e40642014-03-04 20:58:13 +0000797
aleloi868f32f2017-05-23 07:20:05 -0700798 if (aec_dump_) {
799 RecordProcessedCaptureStream(dest);
800 }
andrew@webrtc.org17e40642014-03-04 20:58:13 +0000801 return kNoError;
802}
803
Alex Loiko73ec0192018-05-15 10:52:28 +0200804void AudioProcessingImpl::HandleCaptureRuntimeSettings() {
Alessio Bazzicac054e782018-04-16 12:10:09 +0200805 RuntimeSetting setting;
Per Åhgren652ada52021-03-03 10:52:44 +0000806 int num_settings_processed = 0;
Alex Loiko73ec0192018-05-15 10:52:28 +0200807 while (capture_runtime_settings_.Remove(&setting)) {
Alex Loiko62347222018-09-10 10:18:07 +0200808 if (aec_dump_) {
809 aec_dump_->WriteRuntimeSetting(setting);
810 }
Alessio Bazzicac054e782018-04-16 12:10:09 +0200811 switch (setting.type()) {
812 case RuntimeSetting::Type::kCapturePreGain:
Per Åhgrendb5d7282021-03-15 16:31:04 +0000813 if (config_.pre_amplifier.enabled ||
814 config_.capture_level_adjustment.enabled) {
Alex Loikob5c9a792018-04-16 16:31:22 +0200815 float value;
816 setting.GetFloat(&value);
Per Åhgrendb5d7282021-03-15 16:31:04 +0000817 // If the pre-amplifier is used, apply the new gain to the
818 // pre-amplifier regardless if the capture level adjustment is
819 // activated. This approach allows both functionalities to coexist
820 // until they have been properly merged.
821 if (config_.pre_amplifier.enabled) {
822 config_.pre_amplifier.fixed_gain_factor = value;
823 } else {
824 config_.capture_level_adjustment.pre_gain_factor = value;
825 }
826
827 // Use both the pre-amplifier and the capture level adjustment gains
828 // as pre-gains.
829 float gain = 1.f;
830 if (config_.pre_amplifier.enabled) {
831 gain *= config_.pre_amplifier.fixed_gain_factor;
832 }
833 if (config_.capture_level_adjustment.enabled) {
834 gain *= config_.capture_level_adjustment.pre_gain_factor;
835 }
836
837 submodules_.capture_levels_adjuster->SetPreGain(gain);
838 }
839 // TODO(bugs.chromium.org/9138): Log setting handling by Aec Dump.
840 break;
841 case RuntimeSetting::Type::kCapturePostGain:
842 if (config_.capture_level_adjustment.enabled) {
843 float value;
844 setting.GetFloat(&value);
845 config_.capture_level_adjustment.post_gain_factor = value;
846 submodules_.capture_levels_adjuster->SetPostGain(
847 config_.capture_level_adjustment.post_gain_factor);
Alex Loikob5c9a792018-04-16 16:31:22 +0200848 }
849 // TODO(bugs.chromium.org/9138): Log setting handling by Aec Dump.
Alessio Bazzicac054e782018-04-16 12:10:09 +0200850 break;
Sam Zackrissonf0d1c032019-03-27 13:28:08 +0100851 case RuntimeSetting::Type::kCaptureCompressionGain: {
Per Åhgren0e3198e2019-11-18 08:52:22 +0100852 if (!submodules_.agc_manager) {
853 float value;
854 setting.GetFloat(&value);
855 int int_value = static_cast<int>(value + .5f);
856 config_.gain_controller1.compression_gain_db = int_value;
Per Åhgren0695df12020-01-13 14:43:13 +0100857 if (submodules_.gain_control) {
858 int error =
859 submodules_.gain_control->set_compression_gain_db(int_value);
860 RTC_DCHECK_EQ(kNoError, error);
861 }
Per Åhgren0e3198e2019-11-18 08:52:22 +0100862 }
Sam Zackrissonf0d1c032019-03-27 13:28:08 +0100863 break;
864 }
Per Åhgren6ee75fd2019-04-26 11:33:37 +0200865 case RuntimeSetting::Type::kCaptureFixedPostGain: {
Per Åhgren2bd85ab2020-01-03 10:36:34 +0100866 if (submodules_.gain_controller2) {
Per Åhgren6ee75fd2019-04-26 11:33:37 +0200867 float value;
868 setting.GetFloat(&value);
869 config_.gain_controller2.fixed_digital.gain_db = value;
Alessio Bazzica38901042021-10-14 12:14:21 +0200870 submodules_.gain_controller2->SetFixedGainDb(value);
Per Åhgren6ee75fd2019-04-26 11:33:37 +0200871 }
872 break;
873 }
Fredrik Hernqvistca362852019-05-10 15:50:02 +0200874 case RuntimeSetting::Type::kPlayoutVolumeChange: {
875 int value;
876 setting.GetInt(&value);
877 capture_.playout_volume = value;
878 break;
879 }
Alessio Bazzica7c19a702019-11-07 13:22:00 +0100880 case RuntimeSetting::Type::kPlayoutAudioDeviceChange:
Artem Titovd3251962021-11-15 16:57:07 +0100881 RTC_DCHECK_NOTREACHED();
Alessio Bazzica7c19a702019-11-07 13:22:00 +0100882 break;
Alex Loiko73ec0192018-05-15 10:52:28 +0200883 case RuntimeSetting::Type::kCustomRenderProcessingRuntimeSetting:
Artem Titovd3251962021-11-15 16:57:07 +0100884 RTC_DCHECK_NOTREACHED();
Alex Loiko73ec0192018-05-15 10:52:28 +0200885 break;
886 case RuntimeSetting::Type::kNotSpecified:
Artem Titovd3251962021-11-15 16:57:07 +0100887 RTC_DCHECK_NOTREACHED();
Alex Loiko73ec0192018-05-15 10:52:28 +0200888 break;
Per Åhgren552d3e32020-08-12 08:46:47 +0200889 case RuntimeSetting::Type::kCaptureOutputUsed:
Per Åhgren0a144a72021-02-09 08:47:51 +0100890 bool value;
891 setting.GetBool(&value);
892 HandleCaptureOutputUsedSetting(value);
Per Åhgren552d3e32020-08-12 08:46:47 +0200893 break;
Alex Loiko73ec0192018-05-15 10:52:28 +0200894 }
Per Åhgren652ada52021-03-03 10:52:44 +0000895 ++num_settings_processed;
896 }
897
898 if (num_settings_processed >= RuntimeSettingQueueSize()) {
899 // Handle overrun of the runtime settings queue, which likely will has
900 // caused settings to be discarded.
901 HandleOverrunInCaptureRuntimeSettingsQueue();
902 }
903}
904
905void AudioProcessingImpl::HandleOverrunInCaptureRuntimeSettingsQueue() {
906 // Fall back to a safe state for the case when a setting for capture output
907 // usage setting has been missed.
Per Åhgren19775cb2021-03-12 23:08:09 +0000908 HandleCaptureOutputUsedSetting(/*capture_output_used=*/true);
Alex Loiko73ec0192018-05-15 10:52:28 +0200909}
910
911void AudioProcessingImpl::HandleRenderRuntimeSettings() {
912 RuntimeSetting setting;
913 while (render_runtime_settings_.Remove(&setting)) {
Alex Loiko62347222018-09-10 10:18:07 +0200914 if (aec_dump_) {
915 aec_dump_->WriteRuntimeSetting(setting);
916 }
Alex Loiko73ec0192018-05-15 10:52:28 +0200917 switch (setting.type()) {
Alessio Bazzica7c19a702019-11-07 13:22:00 +0100918 case RuntimeSetting::Type::kPlayoutAudioDeviceChange: // fall-through
Alessio Bazzica7587de42019-11-11 13:32:20 +0100919 case RuntimeSetting::Type::kPlayoutVolumeChange: // fall-through
Alex Loiko73ec0192018-05-15 10:52:28 +0200920 case RuntimeSetting::Type::kCustomRenderProcessingRuntimeSetting:
saza1d600522019-10-18 13:29:43 +0200921 if (submodules_.render_pre_processor) {
922 submodules_.render_pre_processor->SetRuntimeSetting(setting);
Alex Loiko73ec0192018-05-15 10:52:28 +0200923 }
924 break;
Sam Zackrissonf0d1c032019-03-27 13:28:08 +0100925 case RuntimeSetting::Type::kCapturePreGain: // fall-through
Per Åhgrendb5d7282021-03-15 16:31:04 +0000926 case RuntimeSetting::Type::kCapturePostGain: // fall-through
Sam Zackrissonf0d1c032019-03-27 13:28:08 +0100927 case RuntimeSetting::Type::kCaptureCompressionGain: // fall-through
Per Åhgren6ee75fd2019-04-26 11:33:37 +0200928 case RuntimeSetting::Type::kCaptureFixedPostGain: // fall-through
Per Åhgren552d3e32020-08-12 08:46:47 +0200929 case RuntimeSetting::Type::kCaptureOutputUsed: // fall-through
Alessio Bazzica33444dc2018-04-20 13:16:55 +0200930 case RuntimeSetting::Type::kNotSpecified:
Artem Titovd3251962021-11-15 16:57:07 +0100931 RTC_DCHECK_NOTREACHED();
Alessio Bazzicac054e782018-04-16 12:10:09 +0200932 break;
933 }
934 }
935}
936
peah9e6a2902017-05-15 07:19:21 -0700937void AudioProcessingImpl::QueueBandedRenderAudio(AudioBuffer* audio) {
kwibergaf476c72016-11-28 15:21:39 -0800938 RTC_DCHECK_GE(160, audio->num_frames_per_band());
peah764e3642016-10-22 05:04:30 -0700939
saza1d600522019-10-18 13:29:43 +0200940 if (submodules_.echo_control_mobile) {
Per Åhgrenb6e24d72019-04-29 12:14:50 +0200941 EchoControlMobileImpl::PackRenderAudioBuffer(audio, num_output_channels(),
942 num_reverse_channels(),
943 &aecm_render_queue_buffer_);
944 RTC_DCHECK(aecm_render_signal_queue_);
945 // Insert the samples into the queue.
946 if (!aecm_render_signal_queue_->Insert(&aecm_render_queue_buffer_)) {
947 // The data queue is full and needs to be emptied.
948 EmptyQueuedRenderAudio();
peaha0624602016-10-25 04:45:24 -0700949
Per Åhgrenb6e24d72019-04-29 12:14:50 +0200950 // Retry the insert (should always work).
951 bool result =
952 aecm_render_signal_queue_->Insert(&aecm_render_queue_buffer_);
953 RTC_DCHECK(result);
954 }
peah764e3642016-10-22 05:04:30 -0700955 }
peah701d6282016-10-25 05:42:20 -0700956
Per Åhgren0695df12020-01-13 14:43:13 +0100957 if (!submodules_.agc_manager && submodules_.gain_control) {
Per Åhgrene35b32c2019-11-22 18:22:04 +0100958 GainControlImpl::PackRenderAudioBuffer(*audio, &agc_render_queue_buffer_);
peah701d6282016-10-25 05:42:20 -0700959 // Insert the samples into the queue.
960 if (!agc_render_signal_queue_->Insert(&agc_render_queue_buffer_)) {
961 // The data queue is full and needs to be emptied.
962 EmptyQueuedRenderAudio();
963
964 // Retry the insert (should always work).
965 bool result = agc_render_signal_queue_->Insert(&agc_render_queue_buffer_);
966 RTC_DCHECK(result);
967 }
968 }
peah9e6a2902017-05-15 07:19:21 -0700969}
ivoc9f4a4a02016-10-28 05:39:16 -0700970
peah9e6a2902017-05-15 07:19:21 -0700971void AudioProcessingImpl::QueueNonbandedRenderAudio(AudioBuffer* audio) {
ivoc9f4a4a02016-10-28 05:39:16 -0700972 ResidualEchoDetector::PackRenderAudioBuffer(audio, &red_render_queue_buffer_);
973
974 // Insert the samples into the queue.
975 if (!red_render_signal_queue_->Insert(&red_render_queue_buffer_)) {
976 // The data queue is full and needs to be emptied.
977 EmptyQueuedRenderAudio();
978
979 // Retry the insert (should always work).
980 bool result = red_render_signal_queue_->Insert(&red_render_queue_buffer_);
981 RTC_DCHECK(result);
982 }
peah764e3642016-10-22 05:04:30 -0700983}
984
985void AudioProcessingImpl::AllocateRenderQueue() {
peah701d6282016-10-25 05:42:20 -0700986 const size_t new_agc_render_queue_element_max_size =
peah9e6a2902017-05-15 07:19:21 -0700987 std::max(static_cast<size_t>(1), kMaxAllowedValuesOfSamplesPerBand);
peah701d6282016-10-25 05:42:20 -0700988
ivoc9f4a4a02016-10-28 05:39:16 -0700989 const size_t new_red_render_queue_element_max_size =
990 std::max(static_cast<size_t>(1), kMaxAllowedValuesOfSamplesPerFrame);
991
peaha0624602016-10-25 04:45:24 -0700992 // Reallocate the queues if the queue item sizes are too small to fit the
993 // data to put in the queues.
peah701d6282016-10-25 05:42:20 -0700994
995 if (agc_render_queue_element_max_size_ <
996 new_agc_render_queue_element_max_size) {
997 agc_render_queue_element_max_size_ = new_agc_render_queue_element_max_size;
998
999 std::vector<int16_t> template_queue_element(
1000 agc_render_queue_element_max_size_);
1001
1002 agc_render_signal_queue_.reset(
1003 new SwapQueue<std::vector<int16_t>, RenderQueueItemVerifier<int16_t>>(
1004 kMaxNumFramesToBuffer, template_queue_element,
1005 RenderQueueItemVerifier<int16_t>(
1006 agc_render_queue_element_max_size_)));
1007
1008 agc_render_queue_buffer_.resize(agc_render_queue_element_max_size_);
1009 agc_capture_queue_buffer_.resize(agc_render_queue_element_max_size_);
1010 } else {
1011 agc_render_signal_queue_->Clear();
peah764e3642016-10-22 05:04:30 -07001012 }
ivoc9f4a4a02016-10-28 05:39:16 -07001013
1014 if (red_render_queue_element_max_size_ <
1015 new_red_render_queue_element_max_size) {
1016 red_render_queue_element_max_size_ = new_red_render_queue_element_max_size;
1017
1018 std::vector<float> template_queue_element(
1019 red_render_queue_element_max_size_);
1020
1021 red_render_signal_queue_.reset(
1022 new SwapQueue<std::vector<float>, RenderQueueItemVerifier<float>>(
1023 kMaxNumFramesToBuffer, template_queue_element,
1024 RenderQueueItemVerifier<float>(
1025 red_render_queue_element_max_size_)));
1026
1027 red_render_queue_buffer_.resize(red_render_queue_element_max_size_);
1028 red_capture_queue_buffer_.resize(red_render_queue_element_max_size_);
1029 } else {
1030 red_render_signal_queue_->Clear();
1031 }
peah764e3642016-10-22 05:04:30 -07001032}
1033
1034void AudioProcessingImpl::EmptyQueuedRenderAudio() {
Markus Handell0df0fae2020-07-07 15:53:34 +02001035 MutexLock lock_capture(&mutex_capture_);
Markus Handell02ba1d22020-05-14 14:31:18 +02001036 EmptyQueuedRenderAudioLocked();
1037}
1038
1039void AudioProcessingImpl::EmptyQueuedRenderAudioLocked() {
saza1d600522019-10-18 13:29:43 +02001040 if (submodules_.echo_control_mobile) {
Per Åhgrenb6e24d72019-04-29 12:14:50 +02001041 RTC_DCHECK(aecm_render_signal_queue_);
1042 while (aecm_render_signal_queue_->Remove(&aecm_capture_queue_buffer_)) {
saza1d600522019-10-18 13:29:43 +02001043 submodules_.echo_control_mobile->ProcessRenderAudio(
Per Åhgrenb6e24d72019-04-29 12:14:50 +02001044 aecm_capture_queue_buffer_);
1045 }
peah701d6282016-10-25 05:42:20 -07001046 }
1047
Per Åhgren0695df12020-01-13 14:43:13 +01001048 if (submodules_.gain_control) {
1049 while (agc_render_signal_queue_->Remove(&agc_capture_queue_buffer_)) {
1050 submodules_.gain_control->ProcessRenderAudio(agc_capture_queue_buffer_);
1051 }
peah764e3642016-10-22 05:04:30 -07001052 }
ivoc9f4a4a02016-10-28 05:39:16 -07001053
1054 while (red_render_signal_queue_->Remove(&red_capture_queue_buffer_)) {
saza1d600522019-10-18 13:29:43 +02001055 RTC_DCHECK(submodules_.echo_detector);
1056 submodules_.echo_detector->AnalyzeRenderAudio(red_capture_queue_buffer_);
ivoc9f4a4a02016-10-28 05:39:16 -07001057 }
peah764e3642016-10-22 05:04:30 -07001058}
1059
Per Åhgren645f24c2020-03-16 12:06:02 +01001060int AudioProcessingImpl::ProcessStream(const int16_t* const src,
1061 const StreamConfig& input_config,
1062 const StreamConfig& output_config,
Per Åhgrendc5522b2020-03-19 14:55:58 +01001063 int16_t* const dest) {
Per Åhgren71652f42020-03-17 13:23:58 +01001064 TRACE_EVENT0("webrtc", "AudioProcessing::ProcessStream_AudioFrame");
Sam Zackrisson12e319a2020-01-03 14:54:20 +01001065 RETURN_ON_ERR(MaybeInitializeCapture(input_config, output_config));
Oskar Sundbom4b276482019-05-23 14:28:00 +02001066
Markus Handell0df0fae2020-07-07 15:53:34 +02001067 MutexLock lock_capture(&mutex_capture_);
Alessio Bazzica0441bb62021-08-10 15:23:23 +02001068 DenormalDisabler denormal_disabler(use_denormal_disabler_);
niklase@google.com470e71d2011-07-07 08:21:25 +00001069
aleloi868f32f2017-05-23 07:20:05 -07001070 if (aec_dump_) {
Per Åhgren645f24c2020-03-16 12:06:02 +01001071 RecordUnprocessedCaptureStream(src, input_config);
aleloi868f32f2017-05-23 07:20:05 -07001072 }
1073
Per Åhgren645f24c2020-03-16 12:06:02 +01001074 capture_.capture_audio->CopyFrom(src, input_config);
Gustaf Ullberg3c918b12019-10-11 13:14:44 +02001075 if (capture_.capture_fullband_audio) {
Per Åhgren645f24c2020-03-16 12:06:02 +01001076 capture_.capture_fullband_audio->CopyFrom(src, input_config);
Gustaf Ullberg3c918b12019-10-11 13:14:44 +02001077 }
peahde65ddc2016-09-16 15:02:15 -07001078 RETURN_ON_ERR(ProcessCaptureStreamLocked());
Gustaf Ullberg8675eee2019-10-09 13:34:36 +02001079 if (submodule_states_.CaptureMultiBandProcessingPresent() ||
Per Åhgrena1351272019-08-15 12:15:46 +02001080 submodule_states_.CaptureFullBandProcessingActive()) {
Gustaf Ullberg422b9e02019-10-09 13:02:14 +02001081 if (capture_.capture_fullband_audio) {
Per Åhgren645f24c2020-03-16 12:06:02 +01001082 capture_.capture_fullband_audio->CopyTo(output_config, dest);
Gustaf Ullberg422b9e02019-10-09 13:02:14 +02001083 } else {
Per Åhgren645f24c2020-03-16 12:06:02 +01001084 capture_.capture_audio->CopyTo(output_config, dest);
Gustaf Ullberg422b9e02019-10-09 13:02:14 +02001085 }
Per Åhgrena1351272019-08-15 12:15:46 +02001086 }
Per Åhgren645f24c2020-03-16 12:06:02 +01001087
aleloi868f32f2017-05-23 07:20:05 -07001088 if (aec_dump_) {
Per Åhgren645f24c2020-03-16 12:06:02 +01001089 RecordProcessedCaptureStream(dest, output_config);
aleloi868f32f2017-05-23 07:20:05 -07001090 }
andrew@webrtc.org17e40642014-03-04 20:58:13 +00001091
1092 return kNoError;
1093}
1094
peahde65ddc2016-09-16 15:02:15 -07001095int AudioProcessingImpl::ProcessCaptureStreamLocked() {
Markus Handell02ba1d22020-05-14 14:31:18 +02001096 EmptyQueuedRenderAudioLocked();
Alex Loiko73ec0192018-05-15 10:52:28 +02001097 HandleCaptureRuntimeSettings();
Alessio Bazzica0441bb62021-08-10 15:23:23 +02001098 DenormalDisabler denormal_disabler(use_denormal_disabler_);
Alessio Bazzicac054e782018-04-16 12:10:09 +02001099
peahb58a1582016-03-15 09:34:24 -07001100 // Ensure that not both the AEC and AECM are active at the same time.
Sam Zackrisson2a959d92018-07-23 14:48:07 +00001101 // TODO(peah): Simplify once the public API Enable functions for these
1102 // are moved to APM.
Sam Zackrisson308bc642019-12-23 10:22:08 +01001103 RTC_DCHECK_LE(
1104 !!submodules_.echo_controller + !!submodules_.echo_control_mobile, 1);
peahb58a1582016-03-15 09:34:24 -07001105
peahde65ddc2016-09-16 15:02:15 -07001106 AudioBuffer* capture_buffer = capture_.capture_audio.get(); // For brevity.
Per Åhgren2e8e1c62019-12-20 00:42:22 +01001107 AudioBuffer* linear_aec_buffer = capture_.linear_aec_output.get();
ekmeyerson60d9b332015-08-14 10:35:55 -07001108
Per Åhgrenc0424252019-12-10 13:04:15 +01001109 if (submodules_.high_pass_filter &&
1110 config_.high_pass_filter.apply_in_full_band &&
1111 !constants_.enforce_split_band_hpf) {
1112 submodules_.high_pass_filter->Process(capture_buffer,
1113 /*use_split_band_data=*/false);
1114 }
1115
Per Åhgrendb5d7282021-03-15 16:31:04 +00001116 if (submodules_.capture_levels_adjuster) {
1117 // If the analog mic gain emulation is active, get the emulated analog mic
1118 // gain and pass it to the analog gain control functionality.
1119 if (config_.capture_level_adjustment.analog_mic_gain_emulation.enabled) {
1120 int level = submodules_.capture_levels_adjuster->GetAnalogMicGainLevel();
1121 if (submodules_.agc_manager) {
1122 submodules_.agc_manager->set_stream_analog_level(level);
1123 } else if (submodules_.gain_control) {
1124 int error = submodules_.gain_control->set_stream_analog_level(level);
1125 RTC_DCHECK_EQ(kNoError, error);
1126 }
1127 }
1128
1129 submodules_.capture_levels_adjuster->ApplyPreLevelAdjustment(
1130 *capture_buffer);
Alex Loikob5c9a792018-04-16 16:31:22 +02001131 }
1132
Per Åhgren928146f2019-08-20 09:19:21 +02001133 capture_input_rms_.Analyze(rtc::ArrayView<const float>(
Per Åhgrend47941e2019-08-22 11:51:13 +02001134 capture_buffer->channels_const()[0],
henrik.lundin290d43a2016-11-29 08:09:09 -08001135 capture_nonlocked_.capture_processing_format.num_frames()));
peah1b08dc32016-12-20 13:45:58 -08001136 const bool log_rms = ++capture_rms_interval_counter_ >= 1000;
1137 if (log_rms) {
1138 capture_rms_interval_counter_ = 0;
1139 RmsLevel::Levels levels = capture_input_rms_.AverageAndPeak();
henrik.lundin45bb5132016-12-06 04:28:04 -08001140 RTC_HISTOGRAM_COUNTS_LINEAR("WebRTC.Audio.ApmCaptureInputLevelAverageRms",
1141 levels.average, 1, RmsLevel::kMinLevelDb, 64);
1142 RTC_HISTOGRAM_COUNTS_LINEAR("WebRTC.Audio.ApmCaptureInputLevelPeakRms",
1143 levels.peak, 1, RmsLevel::kMinLevelDb, 64);
henrik.lundin290d43a2016-11-29 08:09:09 -08001144 }
1145
Hanna Silen5c7d5c92021-10-06 17:32:17 +02001146 // Detect an analog gain change.
1147 int analog_mic_level = recommended_stream_analog_level_locked();
1148 const bool analog_mic_level_changed =
1149 capture_.prev_analog_mic_level != analog_mic_level &&
1150 capture_.prev_analog_mic_level != -1;
1151 capture_.prev_analog_mic_level = analog_mic_level;
Hanna Silen529131d2021-10-29 14:55:45 +02001152 analog_gain_stats_reporter_.UpdateStatistics(analog_mic_level);
Hanna Silen5c7d5c92021-10-06 17:32:17 +02001153
saza1d600522019-10-18 13:29:43 +02001154 if (submodules_.echo_controller) {
Hanna Silen5c7d5c92021-10-06 17:32:17 +02001155 capture_.echo_path_gain_change = analog_mic_level_changed;
Per Åhgren88cf0502018-07-16 17:08:41 +02001156
Per Åhgrendb5d7282021-03-15 16:31:04 +00001157 // Detect and flag any change in the capture level adjustment pre-gain.
1158 if (submodules_.capture_levels_adjuster) {
1159 float pre_adjustment_gain =
1160 submodules_.capture_levels_adjuster->GetPreAdjustmentGain();
Per Åhgrend2650d12018-10-02 17:00:59 +02001161 capture_.echo_path_gain_change =
1162 capture_.echo_path_gain_change ||
Per Åhgrendb5d7282021-03-15 16:31:04 +00001163 (capture_.prev_pre_adjustment_gain != pre_adjustment_gain &&
1164 capture_.prev_pre_adjustment_gain >= 0.f);
1165 capture_.prev_pre_adjustment_gain = pre_adjustment_gain;
Per Åhgrend2650d12018-10-02 17:00:59 +02001166 }
Fredrik Hernqvistca362852019-05-10 15:50:02 +02001167
1168 // Detect volume change.
1169 capture_.echo_path_gain_change =
1170 capture_.echo_path_gain_change ||
1171 (capture_.prev_playout_volume != capture_.playout_volume &&
1172 capture_.prev_playout_volume >= 0);
1173 capture_.prev_playout_volume = capture_.playout_volume;
1174
saza1d600522019-10-18 13:29:43 +02001175 submodules_.echo_controller->AnalyzeCapture(capture_buffer);
peahe0eae3c2016-12-14 01:16:23 -08001176 }
1177
Per Åhgren0695df12020-01-13 14:43:13 +01001178 if (submodules_.agc_manager) {
Per Åhgren3daedb62019-11-22 12:11:40 +01001179 submodules_.agc_manager->AnalyzePreProcess(capture_buffer);
pbos@webrtc.org788acd12014-12-15 09:41:24 +00001180 }
1181
peah2ace3f92016-09-10 04:42:27 -07001182 if (submodule_states_.CaptureMultiBandSubModulesActive() &&
1183 SampleRateSupportsMultiBand(
peahde65ddc2016-09-16 15:02:15 -07001184 capture_nonlocked_.capture_processing_format.sample_rate_hz())) {
1185 capture_buffer->SplitIntoFrequencyBands();
niklase@google.com470e71d2011-07-07 08:21:25 +00001186 }
1187
Per Åhgrene14cb992019-11-27 09:34:22 +01001188 const bool multi_channel_capture = config_.pipeline.multi_channel_capture &&
1189 constants_.multi_channel_capture_support;
1190 if (submodules_.echo_controller && !multi_channel_capture) {
peah522d71b2017-02-23 05:16:26 -08001191 // Force down-mixing of the number of channels after the detection of
1192 // capture signal saturation.
1193 // TODO(peah): Look into ensuring that this kind of tampering with the
1194 // AudioBuffer functionality should not be needed.
1195 capture_buffer->set_num_channels(1);
1196 }
1197
Per Åhgrenc0424252019-12-10 13:04:15 +01001198 if (submodules_.high_pass_filter &&
1199 (!config_.high_pass_filter.apply_in_full_band ||
1200 constants_.enforce_split_band_hpf)) {
1201 submodules_.high_pass_filter->Process(capture_buffer,
1202 /*use_split_band_data=*/true);
peah8271d042016-11-22 07:24:52 -08001203 }
Per Åhgren0cbb58e2019-10-29 22:59:44 +01001204
Per Åhgren0695df12020-01-13 14:43:13 +01001205 if (submodules_.gain_control) {
1206 RETURN_ON_ERR(
1207 submodules_.gain_control->AnalyzeCaptureAudio(*capture_buffer));
1208 }
Per Åhgren2e8e1c62019-12-20 00:42:22 +01001209
Per Åhgren8ad9e742020-01-30 07:40:58 +01001210 if ((!config_.noise_suppression.analyze_linear_aec_output_when_available ||
1211 !linear_aec_buffer || submodules_.echo_control_mobile) &&
1212 submodules_.noise_suppressor) {
1213 submodules_.noise_suppressor->Analyze(*capture_buffer);
saza0bad15f2019-10-16 11:46:11 +02001214 }
peahb58a1582016-03-15 09:34:24 -07001215
saza1d600522019-10-18 13:29:43 +02001216 if (submodules_.echo_control_mobile) {
Per Åhgrenb6e24d72019-04-29 12:14:50 +02001217 // Ensure that the stream delay was set before the call to the
1218 // AECM ProcessCaptureAudio function.
Sam Zackrisson2d31aea2020-01-17 10:55:09 +01001219 if (!capture_.was_stream_delay_set) {
Per Åhgrenb6e24d72019-04-29 12:14:50 +02001220 return AudioProcessing::kStreamParameterNotSetError;
Per Åhgrend0fa8202018-04-18 09:35:13 +02001221 }
1222
saza1d600522019-10-18 13:29:43 +02001223 if (submodules_.noise_suppressor) {
Per Åhgren0cbb58e2019-10-29 22:59:44 +01001224 submodules_.noise_suppressor->Process(capture_buffer);
Per Åhgrenb6e24d72019-04-29 12:14:50 +02001225 }
peahe0eae3c2016-12-14 01:16:23 -08001226
saza1d600522019-10-18 13:29:43 +02001227 RETURN_ON_ERR(submodules_.echo_control_mobile->ProcessCaptureAudio(
Per Åhgren46537a32017-06-07 10:08:10 +02001228 capture_buffer, stream_delay_ms()));
Per Åhgrenb6e24d72019-04-29 12:14:50 +02001229 } else {
saza1d600522019-10-18 13:29:43 +02001230 if (submodules_.echo_controller) {
Per Åhgrenb6e24d72019-04-29 12:14:50 +02001231 data_dumper_->DumpRaw("stream_delay", stream_delay_ms());
1232
Sam Zackrisson2d31aea2020-01-17 10:55:09 +01001233 if (capture_.was_stream_delay_set) {
saza1d600522019-10-18 13:29:43 +02001234 submodules_.echo_controller->SetAudioBufferDelay(stream_delay_ms());
Per Åhgrenb6e24d72019-04-29 12:14:50 +02001235 }
1236
saza1d600522019-10-18 13:29:43 +02001237 submodules_.echo_controller->ProcessCapture(
Per Åhgrenc20a19c2019-11-13 11:12:29 +01001238 capture_buffer, linear_aec_buffer, capture_.echo_path_gain_change);
Per Åhgrenb6e24d72019-04-29 12:14:50 +02001239 }
1240
Per Åhgren2e8e1c62019-12-20 00:42:22 +01001241 if (config_.noise_suppression.analyze_linear_aec_output_when_available &&
Per Åhgren8ad9e742020-01-30 07:40:58 +01001242 linear_aec_buffer && submodules_.noise_suppressor) {
1243 submodules_.noise_suppressor->Analyze(*linear_aec_buffer);
Per Åhgren2e8e1c62019-12-20 00:42:22 +01001244 }
1245
saza1d600522019-10-18 13:29:43 +02001246 if (submodules_.noise_suppressor) {
Per Åhgren0cbb58e2019-10-29 22:59:44 +01001247 submodules_.noise_suppressor->Process(capture_buffer);
saza0bad15f2019-10-16 11:46:11 +02001248 }
Per Åhgren46537a32017-06-07 10:08:10 +02001249 }
ivoc9f4a4a02016-10-28 05:39:16 -07001250
Sam Zackrisson4db667b2018-12-21 16:29:27 +01001251 if (config_.voice_detection.enabled) {
Sam Zackrisson4db667b2018-12-21 16:29:27 +01001252 capture_.stats.voice_detected =
saza1d600522019-10-18 13:29:43 +02001253 submodules_.voice_detector->ProcessCaptureAudio(capture_buffer);
Sam Zackrisson4db667b2018-12-21 16:29:27 +01001254 } else {
1255 capture_.stats.voice_detected = absl::nullopt;
1256 }
pbos@webrtc.org788acd12014-12-15 09:41:24 +00001257
Per Åhgren0695df12020-01-13 14:43:13 +01001258 if (submodules_.agc_manager) {
Per Åhgren3daedb62019-11-22 12:11:40 +01001259 submodules_.agc_manager->Process(capture_buffer);
1260
1261 absl::optional<int> new_digital_gain =
1262 submodules_.agc_manager->GetDigitalComressionGain();
Per Åhgren0695df12020-01-13 14:43:13 +01001263 if (new_digital_gain && submodules_.gain_control) {
Per Åhgren3daedb62019-11-22 12:11:40 +01001264 submodules_.gain_control->set_compression_gain_db(*new_digital_gain);
1265 }
pbos@webrtc.org788acd12014-12-15 09:41:24 +00001266 }
Per Åhgren0695df12020-01-13 14:43:13 +01001267
1268 if (submodules_.gain_control) {
1269 // TODO(peah): Add reporting from AEC3 whether there is echo.
1270 RETURN_ON_ERR(submodules_.gain_control->ProcessCaptureAudio(
1271 capture_buffer, /*stream_has_echo*/ false));
1272 }
niklase@google.com470e71d2011-07-07 08:21:25 +00001273
Ilya Nikolaevskiy80979352021-03-11 11:40:46 +00001274 if (submodule_states_.CaptureMultiBandProcessingPresent() &&
1275 SampleRateSupportsMultiBand(
1276 capture_nonlocked_.capture_processing_format.sample_rate_hz())) {
1277 capture_buffer->MergeFrequencyBands();
Gustaf Ullberg422b9e02019-10-09 13:02:14 +02001278 }
1279
Per Åhgren19775cb2021-03-12 23:08:09 +00001280 if (capture_.capture_output_used) {
1281 if (capture_.capture_fullband_audio) {
1282 const auto& ec = submodules_.echo_controller;
1283 bool ec_active = ec ? ec->ActiveProcessing() : false;
1284 // Only update the fullband buffer if the multiband processing has changed
1285 // the signal. Keep the original signal otherwise.
1286 if (submodule_states_.CaptureMultiBandProcessingActive(ec_active)) {
1287 capture_buffer->CopyTo(capture_.capture_fullband_audio.get());
1288 }
1289 capture_buffer = capture_.capture_fullband_audio.get();
Ilya Nikolaevskiy80979352021-03-11 11:40:46 +00001290 }
Per Åhgren19775cb2021-03-12 23:08:09 +00001291
1292 if (config_.residual_echo_detector.enabled) {
1293 RTC_DCHECK(submodules_.echo_detector);
1294 submodules_.echo_detector->AnalyzeCaptureAudio(
1295 rtc::ArrayView<const float>(capture_buffer->channels()[0],
1296 capture_buffer->num_frames()));
1297 }
1298
1299 // TODO(aluebs): Investigate if the transient suppression placement should
1300 // be before or after the AGC.
1301 if (submodules_.transient_suppressor) {
1302 float voice_probability =
1303 submodules_.agc_manager.get()
1304 ? submodules_.agc_manager->voice_probability()
1305 : 1.f;
1306
1307 submodules_.transient_suppressor->Suppress(
1308 capture_buffer->channels()[0], capture_buffer->num_frames(),
1309 capture_buffer->num_channels(),
1310 capture_buffer->split_bands_const(0)[kBand0To8kHz],
1311 capture_buffer->num_frames_per_band(),
1312 capture_.keyboard_info.keyboard_data,
1313 capture_.keyboard_info.num_keyboard_frames, voice_probability,
1314 capture_.key_pressed);
1315 }
1316
Artem Titov0b489302021-07-28 20:50:03 +02001317 // Experimental APM sub-module that analyzes `capture_buffer`.
Per Åhgren19775cb2021-03-12 23:08:09 +00001318 if (submodules_.capture_analyzer) {
1319 submodules_.capture_analyzer->Analyze(capture_buffer);
1320 }
1321
1322 if (submodules_.gain_controller2) {
1323 submodules_.gain_controller2->NotifyAnalogLevel(
1324 recommended_stream_analog_level_locked());
1325 submodules_.gain_controller2->Process(capture_buffer);
1326 }
1327
1328 if (submodules_.capture_post_processor) {
1329 submodules_.capture_post_processor->Process(capture_buffer);
1330 }
1331
Per Åhgren19775cb2021-03-12 23:08:09 +00001332 capture_output_rms_.Analyze(rtc::ArrayView<const float>(
1333 capture_buffer->channels_const()[0],
1334 capture_nonlocked_.capture_processing_format.num_frames()));
1335 if (log_rms) {
1336 RmsLevel::Levels levels = capture_output_rms_.AverageAndPeak();
1337 RTC_HISTOGRAM_COUNTS_LINEAR(
1338 "WebRTC.Audio.ApmCaptureOutputLevelAverageRms", levels.average, 1,
1339 RmsLevel::kMinLevelDb, 64);
1340 RTC_HISTOGRAM_COUNTS_LINEAR("WebRTC.Audio.ApmCaptureOutputLevelPeakRms",
1341 levels.peak, 1, RmsLevel::kMinLevelDb, 64);
1342 }
1343
1344 if (submodules_.agc_manager) {
1345 int level = recommended_stream_analog_level_locked();
1346 data_dumper_->DumpRaw("experimental_gain_control_stream_analog_level", 1,
1347 &level);
1348 }
1349
1350 // Compute echo-detector stats.
1351 if (config_.residual_echo_detector.enabled) {
1352 RTC_DCHECK(submodules_.echo_detector);
1353 auto ed_metrics = submodules_.echo_detector->GetMetrics();
1354 capture_.stats.residual_echo_likelihood = ed_metrics.echo_likelihood;
1355 capture_.stats.residual_echo_likelihood_recent_max =
1356 ed_metrics.echo_likelihood_recent_max;
1357 }
Ilya Nikolaevskiy80979352021-03-11 11:40:46 +00001358 }
1359
Per Åhgren19775cb2021-03-12 23:08:09 +00001360 // Compute echo-controller stats.
Per Åhgrencf4c8722019-12-30 14:32:14 +01001361 if (submodules_.echo_controller) {
1362 auto ec_metrics = submodules_.echo_controller->GetMetrics();
1363 capture_.stats.echo_return_loss = ec_metrics.echo_return_loss;
1364 capture_.stats.echo_return_loss_enhancement =
1365 ec_metrics.echo_return_loss_enhancement;
1366 capture_.stats.delay_ms = ec_metrics.delay_ms;
1367 }
Per Åhgrencf4c8722019-12-30 14:32:14 +01001368
1369 // Pass stats for reporting.
1370 stats_reporter_.UpdateStatistics(capture_.stats);
1371
Per Åhgrendb5d7282021-03-15 16:31:04 +00001372 if (submodules_.capture_levels_adjuster) {
1373 submodules_.capture_levels_adjuster->ApplyPostLevelAdjustment(
1374 *capture_buffer);
1375
1376 // If the analog mic gain emulation is active, retrieve the level from the
1377 // analog gain control and set it to mic gain emulator.
1378 if (config_.capture_level_adjustment.analog_mic_gain_emulation.enabled) {
1379 if (submodules_.agc_manager) {
1380 submodules_.capture_levels_adjuster->SetAnalogMicGainLevel(
1381 submodules_.agc_manager->stream_analog_level());
1382 } else if (submodules_.gain_control) {
1383 submodules_.capture_levels_adjuster->SetAnalogMicGainLevel(
1384 submodules_.gain_control->stream_analog_level());
1385 }
1386 }
1387 }
1388
Per Åhgren55bc0772021-03-12 14:18:36 +00001389 // Temporarily set the output to zero after the stream has been unmuted
1390 // (capture output is again used). The purpose of this is to avoid clicks and
1391 // artefacts in the audio that results when the processing again is
1392 // reactivated after unmuting.
1393 if (!capture_.capture_output_used_last_frame &&
1394 capture_.capture_output_used) {
1395 for (size_t ch = 0; ch < capture_buffer->num_channels(); ++ch) {
1396 rtc::ArrayView<float> channel_view(capture_buffer->channels()[ch],
1397 capture_buffer->num_frames());
1398 std::fill(channel_view.begin(), channel_view.end(), 0.f);
1399 }
1400 }
1401 capture_.capture_output_used_last_frame = capture_.capture_output_used;
1402
peahdf3efa82015-11-28 12:35:15 -08001403 capture_.was_stream_delay_set = false;
niklase@google.com470e71d2011-07-07 08:21:25 +00001404 return kNoError;
1405}
1406
Gustaf Ullberg8c51f2e2019-10-22 15:21:31 +02001407int AudioProcessingImpl::AnalyzeReverseStream(
1408 const float* const* data,
1409 const StreamConfig& reverse_config) {
1410 TRACE_EVENT0("webrtc", "AudioProcessing::AnalyzeReverseStream_StreamConfig");
Markus Handell0df0fae2020-07-07 15:53:34 +02001411 MutexLock lock(&mutex_render_);
Gustaf Ullberg8c51f2e2019-10-22 15:21:31 +02001412 return AnalyzeReverseStreamLocked(data, reverse_config, reverse_config);
1413}
1414
peahde65ddc2016-09-16 15:02:15 -07001415int AudioProcessingImpl::ProcessReverseStream(const float* const* src,
1416 const StreamConfig& input_config,
1417 const StreamConfig& output_config,
1418 float* const* dest) {
peah369f8282015-12-17 06:42:29 -08001419 TRACE_EVENT0("webrtc", "AudioProcessing::ProcessReverseStream_StreamConfig");
Markus Handell0df0fae2020-07-07 15:53:34 +02001420 MutexLock lock(&mutex_render_);
Alessio Bazzica0441bb62021-08-10 15:23:23 +02001421 DenormalDisabler denormal_disabler(use_denormal_disabler_);
1422
peahde65ddc2016-09-16 15:02:15 -07001423 RETURN_ON_ERR(AnalyzeReverseStreamLocked(src, input_config, output_config));
Alex Loiko5825aa62017-12-18 16:02:40 +01001424 if (submodule_states_.RenderMultiBandProcessingActive() ||
1425 submodule_states_.RenderFullBandProcessingActive()) {
peahdf3efa82015-11-28 12:35:15 -08001426 render_.render_audio->CopyTo(formats_.api_format.reverse_output_stream(),
1427 dest);
peah2ace3f92016-09-10 04:42:27 -07001428 } else if (formats_.api_format.reverse_input_stream() !=
1429 formats_.api_format.reverse_output_stream()) {
peahde65ddc2016-09-16 15:02:15 -07001430 render_.render_converter->Convert(src, input_config.num_samples(), dest,
1431 output_config.num_samples());
ekmeyerson60d9b332015-08-14 10:35:55 -07001432 } else {
peahde65ddc2016-09-16 15:02:15 -07001433 CopyAudioIfNeeded(src, input_config.num_frames(),
1434 input_config.num_channels(), dest);
ekmeyerson60d9b332015-08-14 10:35:55 -07001435 }
1436
1437 return kNoError;
Michael Graczyk86c6d332015-07-23 11:41:39 -07001438}
1439
peahdf3efa82015-11-28 12:35:15 -08001440int AudioProcessingImpl::AnalyzeReverseStreamLocked(
ekmeyerson60d9b332015-08-14 10:35:55 -07001441 const float* const* src,
peahde65ddc2016-09-16 15:02:15 -07001442 const StreamConfig& input_config,
1443 const StreamConfig& output_config) {
peahdf3efa82015-11-28 12:35:15 -08001444 if (src == nullptr) {
andrew@webrtc.org17e40642014-03-04 20:58:13 +00001445 return kNullPointerError;
1446 }
andrew@webrtc.org17e40642014-03-04 20:58:13 +00001447
peahde65ddc2016-09-16 15:02:15 -07001448 if (input_config.num_channels() == 0) {
Michael Graczyk86c6d332015-07-23 11:41:39 -07001449 return kBadNumberChannelsError;
andrew@webrtc.org17e40642014-03-04 20:58:13 +00001450 }
1451
peahdf3efa82015-11-28 12:35:15 -08001452 ProcessingConfig processing_config = formats_.api_format;
peahde65ddc2016-09-16 15:02:15 -07001453 processing_config.reverse_input_stream() = input_config;
1454 processing_config.reverse_output_stream() = output_config;
Michael Graczyk86c6d332015-07-23 11:41:39 -07001455
peahdf3efa82015-11-28 12:35:15 -08001456 RETURN_ON_ERR(MaybeInitializeRender(processing_config));
Fredrik Solenbergbbf21a32018-04-12 22:44:09 +02001457 RTC_DCHECK_EQ(input_config.num_frames(),
1458 formats_.api_format.reverse_input_stream().num_frames());
Michael Graczyk86c6d332015-07-23 11:41:39 -07001459
Alessio Bazzica0441bb62021-08-10 15:23:23 +02001460 DenormalDisabler denormal_disabler(use_denormal_disabler_);
1461
aleloi868f32f2017-05-23 07:20:05 -07001462 if (aec_dump_) {
1463 const size_t channel_size =
1464 formats_.api_format.reverse_input_stream().num_frames();
1465 const size_t num_channels =
1466 formats_.api_format.reverse_input_stream().num_channels();
1467 aec_dump_->WriteRenderStreamMessage(
Alex Loikoe36e8bb2018-02-16 11:54:07 +01001468 AudioFrameView<const float>(src, num_channels, channel_size));
aleloi868f32f2017-05-23 07:20:05 -07001469 }
peahdf3efa82015-11-28 12:35:15 -08001470 render_.render_audio->CopyFrom(src,
1471 formats_.api_format.reverse_input_stream());
peahde65ddc2016-09-16 15:02:15 -07001472 return ProcessRenderStreamLocked();
ekmeyerson60d9b332015-08-14 10:35:55 -07001473}
1474
Per Åhgren645f24c2020-03-16 12:06:02 +01001475int AudioProcessingImpl::ProcessReverseStream(const int16_t* const src,
1476 const StreamConfig& input_config,
1477 const StreamConfig& output_config,
1478 int16_t* const dest) {
Per Åhgren71652f42020-03-17 13:23:58 +01001479 TRACE_EVENT0("webrtc", "AudioProcessing::ProcessReverseStream_AudioFrame");
Per Åhgren2507f8c2020-03-19 12:33:29 +01001480
1481 if (input_config.num_channels() <= 0) {
1482 return AudioProcessing::Error::kBadNumberChannelsError;
1483 }
1484
Markus Handell0df0fae2020-07-07 15:53:34 +02001485 MutexLock lock(&mutex_render_);
Alessio Bazzica0441bb62021-08-10 15:23:23 +02001486 DenormalDisabler denormal_disabler(use_denormal_disabler_);
1487
peahdf3efa82015-11-28 12:35:15 -08001488 ProcessingConfig processing_config = formats_.api_format;
ekmeyerson60d9b332015-08-14 10:35:55 -07001489 processing_config.reverse_input_stream().set_sample_rate_hz(
Per Åhgren645f24c2020-03-16 12:06:02 +01001490 input_config.sample_rate_hz());
ekmeyerson60d9b332015-08-14 10:35:55 -07001491 processing_config.reverse_input_stream().set_num_channels(
Per Åhgren645f24c2020-03-16 12:06:02 +01001492 input_config.num_channels());
ekmeyerson60d9b332015-08-14 10:35:55 -07001493 processing_config.reverse_output_stream().set_sample_rate_hz(
Per Åhgren645f24c2020-03-16 12:06:02 +01001494 output_config.sample_rate_hz());
ekmeyerson60d9b332015-08-14 10:35:55 -07001495 processing_config.reverse_output_stream().set_num_channels(
Per Åhgren645f24c2020-03-16 12:06:02 +01001496 output_config.num_channels());
Michael Graczyk86c6d332015-07-23 11:41:39 -07001497
peahdf3efa82015-11-28 12:35:15 -08001498 RETURN_ON_ERR(MaybeInitializeRender(processing_config));
Per Åhgren645f24c2020-03-16 12:06:02 +01001499 if (input_config.num_frames() !=
peahdf3efa82015-11-28 12:35:15 -08001500 formats_.api_format.reverse_input_stream().num_frames()) {
andrew@webrtc.org17e40642014-03-04 20:58:13 +00001501 return kBadDataLengthError;
1502 }
niklase@google.com470e71d2011-07-07 08:21:25 +00001503
aleloi868f32f2017-05-23 07:20:05 -07001504 if (aec_dump_) {
Per Åhgren645f24c2020-03-16 12:06:02 +01001505 aec_dump_->WriteRenderStreamMessage(src, input_config.num_frames(),
1506 input_config.num_channels());
aleloi868f32f2017-05-23 07:20:05 -07001507 }
1508
Per Åhgren645f24c2020-03-16 12:06:02 +01001509 render_.render_audio->CopyFrom(src, input_config);
peahde65ddc2016-09-16 15:02:15 -07001510 RETURN_ON_ERR(ProcessRenderStreamLocked());
Per Åhgrena1351272019-08-15 12:15:46 +02001511 if (submodule_states_.RenderMultiBandProcessingActive() ||
1512 submodule_states_.RenderFullBandProcessingActive()) {
Per Åhgren645f24c2020-03-16 12:06:02 +01001513 render_.render_audio->CopyTo(output_config, dest);
Per Åhgrena1351272019-08-15 12:15:46 +02001514 }
aluebsb0319552016-03-17 20:39:53 -07001515 return kNoError;
andrew@webrtc.org17e40642014-03-04 20:58:13 +00001516}
niklase@google.com470e71d2011-07-07 08:21:25 +00001517
peahde65ddc2016-09-16 15:02:15 -07001518int AudioProcessingImpl::ProcessRenderStreamLocked() {
1519 AudioBuffer* render_buffer = render_.render_audio.get(); // For brevity.
peah9e6a2902017-05-15 07:19:21 -07001520
Alex Loiko73ec0192018-05-15 10:52:28 +02001521 HandleRenderRuntimeSettings();
Alessio Bazzica0441bb62021-08-10 15:23:23 +02001522 DenormalDisabler denormal_disabler(use_denormal_disabler_);
Alex Loiko73ec0192018-05-15 10:52:28 +02001523
saza1d600522019-10-18 13:29:43 +02001524 if (submodules_.render_pre_processor) {
1525 submodules_.render_pre_processor->Process(render_buffer);
Alex Loiko5825aa62017-12-18 16:02:40 +01001526 }
1527
Alessio Bazzicad2b97402018-08-09 14:23:11 +02001528 QueueNonbandedRenderAudio(render_buffer);
1529
peah2ace3f92016-09-10 04:42:27 -07001530 if (submodule_states_.RenderMultiBandSubModulesActive() &&
peahde65ddc2016-09-16 15:02:15 -07001531 SampleRateSupportsMultiBand(
1532 formats_.render_processing_format.sample_rate_hz())) {
1533 render_buffer->SplitIntoFrequencyBands();
niklase@google.com470e71d2011-07-07 08:21:25 +00001534 }
1535
peahce4d9152017-05-19 01:28:05 -07001536 if (submodule_states_.RenderMultiBandSubModulesActive()) {
1537 QueueBandedRenderAudio(render_buffer);
1538 }
1539
Alessio Bazzicad2b97402018-08-09 14:23:11 +02001540 // TODO(peah): Perform the queuing inside QueueRenderAudiuo().
saza1d600522019-10-18 13:29:43 +02001541 if (submodules_.echo_controller) {
1542 submodules_.echo_controller->AnalyzeRender(render_buffer);
peahe0eae3c2016-12-14 01:16:23 -08001543 }
niklase@google.com470e71d2011-07-07 08:21:25 +00001544
peah2ace3f92016-09-10 04:42:27 -07001545 if (submodule_states_.RenderMultiBandProcessingActive() &&
peahde65ddc2016-09-16 15:02:15 -07001546 SampleRateSupportsMultiBand(
1547 formats_.render_processing_format.sample_rate_hz())) {
1548 render_buffer->MergeFrequencyBands();
ekmeyerson60d9b332015-08-14 10:35:55 -07001549 }
1550
andrew@webrtc.org17e40642014-03-04 20:58:13 +00001551 return kNoError;
niklase@google.com470e71d2011-07-07 08:21:25 +00001552}
1553
1554int AudioProcessingImpl::set_stream_delay_ms(int delay) {
Markus Handell0df0fae2020-07-07 15:53:34 +02001555 MutexLock lock(&mutex_capture_);
andrew@webrtc.org5f23d642012-05-29 21:14:06 +00001556 Error retval = kNoError;
peahdf3efa82015-11-28 12:35:15 -08001557 capture_.was_stream_delay_set = true;
andrew@webrtc.org6f9f8172012-03-06 19:03:39 +00001558
niklase@google.com470e71d2011-07-07 08:21:25 +00001559 if (delay < 0) {
andrew@webrtc.org5f23d642012-05-29 21:14:06 +00001560 delay = 0;
1561 retval = kBadStreamParameterWarning;
niklase@google.com470e71d2011-07-07 08:21:25 +00001562 }
1563
1564 // TODO(ajm): the max is rather arbitrarily chosen; investigate.
1565 if (delay > 500) {
andrew@webrtc.org5f23d642012-05-29 21:14:06 +00001566 delay = 500;
1567 retval = kBadStreamParameterWarning;
niklase@google.com470e71d2011-07-07 08:21:25 +00001568 }
1569
peahdf3efa82015-11-28 12:35:15 -08001570 capture_nonlocked_.stream_delay_ms = delay;
andrew@webrtc.org5f23d642012-05-29 21:14:06 +00001571 return retval;
niklase@google.com470e71d2011-07-07 08:21:25 +00001572}
1573
Per Åhgrenc20a19c2019-11-13 11:12:29 +01001574bool AudioProcessingImpl::GetLinearAecOutput(
1575 rtc::ArrayView<std::array<float, 160>> linear_output) const {
Markus Handell0df0fae2020-07-07 15:53:34 +02001576 MutexLock lock(&mutex_capture_);
Per Åhgrenc20a19c2019-11-13 11:12:29 +01001577 AudioBuffer* linear_aec_buffer = capture_.linear_aec_output.get();
1578
1579 RTC_DCHECK(linear_aec_buffer);
1580 if (linear_aec_buffer) {
1581 RTC_DCHECK_EQ(1, linear_aec_buffer->num_bands());
1582 RTC_DCHECK_EQ(linear_output.size(), linear_aec_buffer->num_channels());
1583
1584 for (size_t ch = 0; ch < linear_aec_buffer->num_channels(); ++ch) {
1585 RTC_DCHECK_EQ(linear_output[ch].size(), linear_aec_buffer->num_frames());
1586 rtc::ArrayView<const float> channel_view =
1587 rtc::ArrayView<const float>(linear_aec_buffer->channels_const()[ch],
1588 linear_aec_buffer->num_frames());
Gustaf Ullberg45436972020-11-13 14:30:30 +01001589 FloatS16ToFloat(channel_view.data(), channel_view.size(),
1590 linear_output[ch].data());
Per Åhgrenc20a19c2019-11-13 11:12:29 +01001591 }
1592 return true;
1593 }
1594 RTC_LOG(LS_ERROR) << "No linear AEC output available";
Artem Titovd3251962021-11-15 16:57:07 +01001595 RTC_DCHECK_NOTREACHED();
Per Åhgrenc20a19c2019-11-13 11:12:29 +01001596 return false;
1597}
1598
niklase@google.com470e71d2011-07-07 08:21:25 +00001599int AudioProcessingImpl::stream_delay_ms() const {
peahdf3efa82015-11-28 12:35:15 -08001600 // Used as callback from submodules, hence locking is not allowed.
1601 return capture_nonlocked_.stream_delay_ms;
niklase@google.com470e71d2011-07-07 08:21:25 +00001602}
1603
andrew@webrtc.org17e40642014-03-04 20:58:13 +00001604void AudioProcessingImpl::set_stream_key_pressed(bool key_pressed) {
Markus Handell0df0fae2020-07-07 15:53:34 +02001605 MutexLock lock(&mutex_capture_);
peahdf3efa82015-11-28 12:35:15 -08001606 capture_.key_pressed = key_pressed;
andrew@webrtc.org17e40642014-03-04 20:58:13 +00001607}
1608
Sam Zackrissonf0d1c032019-03-27 13:28:08 +01001609void AudioProcessingImpl::set_stream_analog_level(int level) {
Markus Handell0df0fae2020-07-07 15:53:34 +02001610 MutexLock lock_capture(&mutex_capture_);
Per Åhgren0e3198e2019-11-18 08:52:22 +01001611
Per Åhgrendb5d7282021-03-15 16:31:04 +00001612 if (config_.capture_level_adjustment.analog_mic_gain_emulation.enabled) {
1613 // If the analog mic gain is emulated internally, simply cache the level for
1614 // later reporting back as the recommended stream analog level to use.
1615 capture_.cached_stream_analog_level_ = level;
1616 return;
1617 }
1618
Per Åhgren0e3198e2019-11-18 08:52:22 +01001619 if (submodules_.agc_manager) {
1620 submodules_.agc_manager->set_stream_analog_level(level);
1621 data_dumper_->DumpRaw("experimental_gain_control_set_stream_analog_level",
1622 1, &level);
Per Åhgrendb5d7282021-03-15 16:31:04 +00001623 return;
1624 }
1625
1626 if (submodules_.gain_control) {
Per Åhgren0e3198e2019-11-18 08:52:22 +01001627 int error = submodules_.gain_control->set_stream_analog_level(level);
1628 RTC_DCHECK_EQ(kNoError, error);
Per Åhgrendb5d7282021-03-15 16:31:04 +00001629 return;
Per Åhgren0e3198e2019-11-18 08:52:22 +01001630 }
Per Åhgrendb5d7282021-03-15 16:31:04 +00001631
1632 // If no analog mic gain control functionality is in place, cache the level
1633 // for later reporting back as the recommended stream analog level to use.
1634 capture_.cached_stream_analog_level_ = level;
Sam Zackrissonf0d1c032019-03-27 13:28:08 +01001635}
1636
1637int AudioProcessingImpl::recommended_stream_analog_level() const {
Markus Handell0df0fae2020-07-07 15:53:34 +02001638 MutexLock lock_capture(&mutex_capture_);
Markus Handell02ba1d22020-05-14 14:31:18 +02001639 return recommended_stream_analog_level_locked();
1640}
1641
1642int AudioProcessingImpl::recommended_stream_analog_level_locked() const {
Per Åhgrendb5d7282021-03-15 16:31:04 +00001643 if (config_.capture_level_adjustment.analog_mic_gain_emulation.enabled) {
Per Åhgren0695df12020-01-13 14:43:13 +01001644 return capture_.cached_stream_analog_level_;
Per Åhgren0e3198e2019-11-18 08:52:22 +01001645 }
Per Åhgrendb5d7282021-03-15 16:31:04 +00001646
1647 if (submodules_.agc_manager) {
1648 return submodules_.agc_manager->stream_analog_level();
1649 }
1650
1651 if (submodules_.gain_control) {
1652 return submodules_.gain_control->stream_analog_level();
1653 }
1654
1655 return capture_.cached_stream_analog_level_;
Sam Zackrissonf0d1c032019-03-27 13:28:08 +01001656}
1657
Per Åhgren09e9a832020-05-11 11:03:47 +02001658bool AudioProcessingImpl::CreateAndAttachAecDump(const std::string& file_name,
1659 int64_t max_log_size_bytes,
1660 rtc::TaskQueue* worker_queue) {
1661 std::unique_ptr<AecDump> aec_dump =
1662 AecDumpFactory::Create(file_name, max_log_size_bytes, worker_queue);
1663 if (!aec_dump) {
1664 return false;
1665 }
1666
1667 AttachAecDump(std::move(aec_dump));
1668 return true;
1669}
1670
1671bool AudioProcessingImpl::CreateAndAttachAecDump(FILE* handle,
1672 int64_t max_log_size_bytes,
1673 rtc::TaskQueue* worker_queue) {
1674 std::unique_ptr<AecDump> aec_dump =
1675 AecDumpFactory::Create(handle, max_log_size_bytes, worker_queue);
1676 if (!aec_dump) {
1677 return false;
1678 }
1679
1680 AttachAecDump(std::move(aec_dump));
1681 return true;
1682}
1683
aleloi868f32f2017-05-23 07:20:05 -07001684void AudioProcessingImpl::AttachAecDump(std::unique_ptr<AecDump> aec_dump) {
1685 RTC_DCHECK(aec_dump);
Markus Handell0df0fae2020-07-07 15:53:34 +02001686 MutexLock lock_render(&mutex_render_);
1687 MutexLock lock_capture(&mutex_capture_);
aleloi868f32f2017-05-23 07:20:05 -07001688
1689 // The previously attached AecDump will be destroyed with the
1690 // 'aec_dump' parameter, which is after locks are released.
1691 aec_dump_.swap(aec_dump);
1692 WriteAecDumpConfigMessage(true);
Minyue Li656d6092018-08-10 15:38:52 +02001693 aec_dump_->WriteInitMessage(formats_.api_format, rtc::TimeUTCMillis());
aleloi868f32f2017-05-23 07:20:05 -07001694}
1695
1696void AudioProcessingImpl::DetachAecDump() {
1697 // The d-tor of a task-queue based AecDump blocks until all pending
1698 // tasks are done. This construction avoids blocking while holding
1699 // the render and capture locks.
1700 std::unique_ptr<AecDump> aec_dump = nullptr;
1701 {
Markus Handell0df0fae2020-07-07 15:53:34 +02001702 MutexLock lock_render(&mutex_render_);
1703 MutexLock lock_capture(&mutex_capture_);
aleloi868f32f2017-05-23 07:20:05 -07001704 aec_dump = std::move(aec_dump_);
1705 }
1706}
1707
peah8271d042016-11-22 07:24:52 -08001708AudioProcessing::Config AudioProcessingImpl::GetConfig() const {
Markus Handell0df0fae2020-07-07 15:53:34 +02001709 MutexLock lock_render(&mutex_render_);
1710 MutexLock lock_capture(&mutex_capture_);
peah8271d042016-11-22 07:24:52 -08001711 return config_;
1712}
1713
peah2ace3f92016-09-10 04:42:27 -07001714bool AudioProcessingImpl::UpdateActiveSubmoduleStates() {
1715 return submodule_states_.Update(
Per Åhgren62ea0aa2019-12-09 10:18:44 +01001716 config_.high_pass_filter.enabled, !!submodules_.echo_control_mobile,
Per Åhgren8ad9e742020-01-30 07:40:58 +01001717 config_.residual_echo_detector.enabled, !!submodules_.noise_suppressor,
Per Åhgren0695df12020-01-13 14:43:13 +01001718 !!submodules_.gain_control, !!submodules_.gain_controller2,
Per Åhgrendb5d7282021-03-15 16:31:04 +00001719 config_.pre_amplifier.enabled || config_.capture_level_adjustment.enabled,
1720 capture_nonlocked_.echo_controller_enabled,
Per Åhgrenc0734712020-01-02 15:15:36 +01001721 config_.voice_detection.enabled, !!submodules_.transient_suppressor);
ekmeyerson60d9b332015-08-14 10:35:55 -07001722}
1723
Per Åhgrenc0734712020-01-02 15:15:36 +01001724void AudioProcessingImpl::InitializeTransientSuppressor() {
Gustaf Ullberga399c822021-05-18 12:17:56 +02001725 if (config_.transient_suppression.enabled &&
1726 !constants_.transient_suppressor_forced_off) {
sazaaa42ecd2020-04-01 15:24:40 +02001727 // Attempt to create a transient suppressor, if one is not already created.
Per Åhgrenc0734712020-01-02 15:15:36 +01001728 if (!submodules_.transient_suppressor) {
Sam Zackrissonb37e59d2020-04-27 08:39:33 +02001729 submodules_.transient_suppressor =
1730 CreateTransientSuppressor(submodule_creation_overrides_);
pbos@webrtc.org788acd12014-12-15 09:41:24 +00001731 }
sazaaa42ecd2020-04-01 15:24:40 +02001732 if (submodules_.transient_suppressor) {
1733 submodules_.transient_suppressor->Initialize(
1734 proc_fullband_sample_rate_hz(), capture_nonlocked_.split_rate,
1735 num_proc_channels());
1736 } else {
1737 RTC_LOG(LS_WARNING)
1738 << "No transient suppressor created (probably disabled)";
1739 }
Per Åhgrenc0734712020-01-02 15:15:36 +01001740 } else {
1741 submodules_.transient_suppressor.reset();
pbos@webrtc.org788acd12014-12-15 09:41:24 +00001742 }
pbos@webrtc.org788acd12014-12-15 09:41:24 +00001743}
1744
Per Åhgren0f14db22020-01-03 14:27:14 +01001745void AudioProcessingImpl::InitializeHighPassFilter(bool forced_reset) {
Per Åhgrenb8106462019-12-04 08:34:12 +01001746 bool high_pass_filter_needed_by_aec =
1747 config_.echo_canceller.enabled &&
1748 config_.echo_canceller.enforce_high_pass_filtering &&
1749 !config_.echo_canceller.mobile_mode;
1750 if (submodule_states_.HighPassFilteringRequired() ||
1751 high_pass_filter_needed_by_aec) {
Per Åhgrenc0424252019-12-10 13:04:15 +01001752 bool use_full_band = config_.high_pass_filter.apply_in_full_band &&
1753 !constants_.enforce_split_band_hpf;
1754 int rate = use_full_band ? proc_fullband_sample_rate_hz()
1755 : proc_split_sample_rate_hz();
1756 size_t num_channels =
1757 use_full_band ? num_output_channels() : num_proc_channels();
1758
Per Åhgren0f14db22020-01-03 14:27:14 +01001759 if (!submodules_.high_pass_filter ||
1760 rate != submodules_.high_pass_filter->sample_rate_hz() ||
1761 forced_reset ||
1762 num_channels != submodules_.high_pass_filter->num_channels()) {
1763 submodules_.high_pass_filter.reset(
1764 new HighPassFilter(rate, num_channels));
1765 }
peah8271d042016-11-22 07:24:52 -08001766 } else {
saza1d600522019-10-18 13:29:43 +02001767 submodules_.high_pass_filter.reset();
peah8271d042016-11-22 07:24:52 -08001768 }
1769}
alessiob3ec96df2017-05-22 06:57:06 -07001770
Sam Zackrisson0824c6f2019-10-07 14:03:56 +02001771void AudioProcessingImpl::InitializeVoiceDetector() {
1772 if (config_.voice_detection.enabled) {
saza1d600522019-10-18 13:29:43 +02001773 submodules_.voice_detector = std::make_unique<VoiceDetection>(
Sam Zackrisson0824c6f2019-10-07 14:03:56 +02001774 proc_split_sample_rate_hz(), VoiceDetection::kVeryLowLikelihood);
1775 } else {
saza1d600522019-10-18 13:29:43 +02001776 submodules_.voice_detector.reset();
Sam Zackrisson0824c6f2019-10-07 14:03:56 +02001777 }
1778}
Gustaf Ullberg8eb9c7d2017-10-14 08:28:46 +02001779void AudioProcessingImpl::InitializeEchoController() {
Per Åhgrenb6e24d72019-04-29 12:14:50 +02001780 bool use_echo_controller =
1781 echo_control_factory_ ||
Per Åhgren62ea0aa2019-12-09 10:18:44 +01001782 (config_.echo_canceller.enabled && !config_.echo_canceller.mobile_mode);
Per Åhgrenb6e24d72019-04-29 12:14:50 +02001783
1784 if (use_echo_controller) {
1785 // Create and activate the echo controller.
Per Åhgren200feba2019-03-06 04:16:46 +01001786 if (echo_control_factory_) {
Per Åhgren4e5c7092019-11-01 20:44:11 +01001787 submodules_.echo_controller = echo_control_factory_->Create(
1788 proc_sample_rate_hz(), num_reverse_channels(), num_proc_channels());
Gustaf Ullberg2c6f3732019-11-07 17:15:12 +01001789 RTC_DCHECK(submodules_.echo_controller);
Per Åhgren200feba2019-03-06 04:16:46 +01001790 } else {
Per Åhgrenb2b58d82019-12-02 14:59:40 +01001791 EchoCanceller3Config config =
1792 use_setup_specific_default_aec3_config_
1793 ? EchoCanceller3::CreateDefaultConfig(num_reverse_channels(),
1794 num_proc_channels())
1795 : EchoCanceller3Config();
saza1d600522019-10-18 13:29:43 +02001796 submodules_.echo_controller = std::make_unique<EchoCanceller3>(
Per Åhgrenb2b58d82019-12-02 14:59:40 +01001797 config, proc_sample_rate_hz(), num_reverse_channels(),
Sam Zackrissonfeee1e42019-09-20 07:50:35 +02001798 num_proc_channels());
Per Åhgren200feba2019-03-06 04:16:46 +01001799 }
1800
Per Åhgrenc20a19c2019-11-13 11:12:29 +01001801 // Setup the storage for returning the linear AEC output.
1802 if (config_.echo_canceller.export_linear_aec_output) {
1803 constexpr int kLinearOutputRateHz = 16000;
1804 capture_.linear_aec_output = std::make_unique<AudioBuffer>(
1805 kLinearOutputRateHz, num_proc_channels(), kLinearOutputRateHz,
1806 num_proc_channels(), kLinearOutputRateHz, num_proc_channels());
1807 } else {
1808 capture_.linear_aec_output.reset();
1809 }
1810
Per Åhgren200feba2019-03-06 04:16:46 +01001811 capture_nonlocked_.echo_controller_enabled = true;
Per Åhgren200feba2019-03-06 04:16:46 +01001812
saza1d600522019-10-18 13:29:43 +02001813 submodules_.echo_control_mobile.reset();
Per Åhgrenb6e24d72019-04-29 12:14:50 +02001814 aecm_render_signal_queue_.reset();
Per Åhgrenf204faf2019-04-25 15:18:06 +02001815 return;
peahe0eae3c2016-12-14 01:16:23 -08001816 }
Per Åhgrenf204faf2019-04-25 15:18:06 +02001817
saza1d600522019-10-18 13:29:43 +02001818 submodules_.echo_controller.reset();
Per Åhgrenf204faf2019-04-25 15:18:06 +02001819 capture_nonlocked_.echo_controller_enabled = false;
Per Åhgrenc20a19c2019-11-13 11:12:29 +01001820 capture_.linear_aec_output.reset();
Per Åhgrenf204faf2019-04-25 15:18:06 +02001821
1822 if (!config_.echo_canceller.enabled) {
saza1d600522019-10-18 13:29:43 +02001823 submodules_.echo_control_mobile.reset();
Per Åhgrenb6e24d72019-04-29 12:14:50 +02001824 aecm_render_signal_queue_.reset();
Per Åhgrenf204faf2019-04-25 15:18:06 +02001825 return;
1826 }
1827
1828 if (config_.echo_canceller.mobile_mode) {
1829 // Create and activate AECM.
Per Åhgrenb6e24d72019-04-29 12:14:50 +02001830 size_t max_element_size =
1831 std::max(static_cast<size_t>(1),
1832 kMaxAllowedValuesOfSamplesPerBand *
1833 EchoControlMobileImpl::NumCancellersRequired(
1834 num_output_channels(), num_reverse_channels()));
1835
1836 std::vector<int16_t> template_queue_element(max_element_size);
1837
1838 aecm_render_signal_queue_.reset(
1839 new SwapQueue<std::vector<int16_t>, RenderQueueItemVerifier<int16_t>>(
1840 kMaxNumFramesToBuffer, template_queue_element,
1841 RenderQueueItemVerifier<int16_t>(max_element_size)));
1842
1843 aecm_render_queue_buffer_.resize(max_element_size);
1844 aecm_capture_queue_buffer_.resize(max_element_size);
1845
saza1d600522019-10-18 13:29:43 +02001846 submodules_.echo_control_mobile.reset(new EchoControlMobileImpl());
Per Åhgrenb6e24d72019-04-29 12:14:50 +02001847
saza1d600522019-10-18 13:29:43 +02001848 submodules_.echo_control_mobile->Initialize(proc_split_sample_rate_hz(),
1849 num_reverse_channels(),
1850 num_output_channels());
Per Åhgrenf204faf2019-04-25 15:18:06 +02001851 return;
1852 }
1853
saza1d600522019-10-18 13:29:43 +02001854 submodules_.echo_control_mobile.reset();
Per Åhgrenb6e24d72019-04-29 12:14:50 +02001855 aecm_render_signal_queue_.reset();
peahe0eae3c2016-12-14 01:16:23 -08001856}
peah8271d042016-11-22 07:24:52 -08001857
Per Åhgren0695df12020-01-13 14:43:13 +01001858void AudioProcessingImpl::InitializeGainController1() {
1859 if (!config_.gain_controller1.enabled) {
1860 submodules_.agc_manager.reset();
1861 submodules_.gain_control.reset();
1862 return;
1863 }
1864
1865 if (!submodules_.gain_control) {
1866 submodules_.gain_control.reset(new GainControlImpl());
1867 }
1868
1869 submodules_.gain_control->Initialize(num_proc_channels(),
1870 proc_sample_rate_hz());
Per Åhgren0695df12020-01-13 14:43:13 +01001871 if (!config_.gain_controller1.analog_gain_controller.enabled) {
1872 int error = submodules_.gain_control->set_mode(
1873 Agc1ConfigModeToInterfaceMode(config_.gain_controller1.mode));
1874 RTC_DCHECK_EQ(kNoError, error);
1875 error = submodules_.gain_control->set_target_level_dbfs(
1876 config_.gain_controller1.target_level_dbfs);
1877 RTC_DCHECK_EQ(kNoError, error);
1878 error = submodules_.gain_control->set_compression_gain_db(
1879 config_.gain_controller1.compression_gain_db);
1880 RTC_DCHECK_EQ(kNoError, error);
1881 error = submodules_.gain_control->enable_limiter(
1882 config_.gain_controller1.enable_limiter);
1883 RTC_DCHECK_EQ(kNoError, error);
Hanna Silencd597042021-11-02 11:02:48 +01001884 constexpr int kAnalogLevelMinimum = 0;
1885 constexpr int kAnalogLevelMaximum = 255;
Per Åhgren0695df12020-01-13 14:43:13 +01001886 error = submodules_.gain_control->set_analog_level_limits(
Hanna Silencd597042021-11-02 11:02:48 +01001887 kAnalogLevelMinimum, kAnalogLevelMaximum);
Per Åhgren0695df12020-01-13 14:43:13 +01001888 RTC_DCHECK_EQ(kNoError, error);
1889
1890 submodules_.agc_manager.reset();
1891 return;
1892 }
1893
1894 if (!submodules_.agc_manager.get() ||
1895 submodules_.agc_manager->num_channels() !=
1896 static_cast<int>(num_proc_channels()) ||
1897 submodules_.agc_manager->sample_rate_hz() !=
1898 capture_nonlocked_.split_rate) {
1899 int stream_analog_level = -1;
1900 const bool re_creation = !!submodules_.agc_manager;
1901 if (re_creation) {
1902 stream_analog_level = submodules_.agc_manager->stream_analog_level();
1903 }
1904 submodules_.agc_manager.reset(new AgcManagerDirect(
1905 num_proc_channels(),
1906 config_.gain_controller1.analog_gain_controller.startup_min_volume,
1907 config_.gain_controller1.analog_gain_controller.clipped_level_min,
Per Åhgren0695df12020-01-13 14:43:13 +01001908 !config_.gain_controller1.analog_gain_controller
1909 .enable_digital_adaptive,
Hanna Silenb8dc7fa2021-05-20 17:37:56 +02001910 capture_nonlocked_.split_rate,
1911 config_.gain_controller1.analog_gain_controller.clipped_level_step,
1912 config_.gain_controller1.analog_gain_controller.clipped_ratio_threshold,
Hanna Silena0047152021-06-03 03:29:38 +02001913 config_.gain_controller1.analog_gain_controller.clipped_wait_frames,
1914 config_.gain_controller1.analog_gain_controller.clipping_predictor));
Per Åhgren0695df12020-01-13 14:43:13 +01001915 if (re_creation) {
1916 submodules_.agc_manager->set_stream_analog_level(stream_analog_level);
1917 }
1918 }
1919 submodules_.agc_manager->Initialize();
1920 submodules_.agc_manager->SetupDigitalGainControl(
1921 submodules_.gain_control.get());
Per Åhgren0a144a72021-02-09 08:47:51 +01001922 submodules_.agc_manager->HandleCaptureOutputUsedChange(
1923 capture_.capture_output_used);
Per Åhgren0695df12020-01-13 14:43:13 +01001924}
1925
Alessio Bazzica38901042021-10-14 12:14:21 +02001926void AudioProcessingImpl::InitializeGainController2(bool config_has_changed) {
1927 if (!config_has_changed) {
1928 return;
1929 }
1930 if (!config_.gain_controller2.enabled) {
Per Åhgren2bd85ab2020-01-03 10:36:34 +01001931 submodules_.gain_controller2.reset();
Alessio Bazzica38901042021-10-14 12:14:21 +02001932 return;
1933 }
1934 if (!submodules_.gain_controller2 || config_has_changed) {
1935 submodules_.gain_controller2 = std::make_unique<GainController2>(
1936 config_.gain_controller2, proc_fullband_sample_rate_hz(),
1937 num_input_channels());
alessiob3ec96df2017-05-22 06:57:06 -07001938 }
1939}
1940
saza0bad15f2019-10-16 11:46:11 +02001941void AudioProcessingImpl::InitializeNoiseSuppressor() {
Per Åhgren0cbb58e2019-10-29 22:59:44 +01001942 submodules_.noise_suppressor.reset();
1943
saza0bad15f2019-10-16 11:46:11 +02001944 if (config_.noise_suppression.enabled) {
sazaaa42ecd2020-04-01 15:24:40 +02001945 auto map_level =
1946 [](AudioProcessing::Config::NoiseSuppression::Level level) {
1947 using NoiseSuppresionConfig =
1948 AudioProcessing::Config::NoiseSuppression;
1949 switch (level) {
1950 case NoiseSuppresionConfig::kLow:
1951 return NsConfig::SuppressionLevel::k6dB;
1952 case NoiseSuppresionConfig::kModerate:
1953 return NsConfig::SuppressionLevel::k12dB;
1954 case NoiseSuppresionConfig::kHigh:
1955 return NsConfig::SuppressionLevel::k18dB;
1956 case NoiseSuppresionConfig::kVeryHigh:
1957 return NsConfig::SuppressionLevel::k21dB;
sazaaa42ecd2020-04-01 15:24:40 +02001958 }
Karl Wibergc95b9392020-11-08 00:49:37 +01001959 RTC_CHECK_NOTREACHED();
sazaaa42ecd2020-04-01 15:24:40 +02001960 };
Per Åhgren0cbb58e2019-10-29 22:59:44 +01001961
sazaaa42ecd2020-04-01 15:24:40 +02001962 NsConfig cfg;
1963 cfg.target_level = map_level(config_.noise_suppression.level);
1964 submodules_.noise_suppressor = std::make_unique<NoiseSuppressor>(
1965 cfg, proc_sample_rate_hz(), num_proc_channels());
saza0bad15f2019-10-16 11:46:11 +02001966 }
1967}
1968
Per Åhgrendb5d7282021-03-15 16:31:04 +00001969void AudioProcessingImpl::InitializeCaptureLevelsAdjuster() {
1970 if (config_.pre_amplifier.enabled ||
1971 config_.capture_level_adjustment.enabled) {
1972 // Use both the pre-amplifier and the capture level adjustment gains as
1973 // pre-gains.
1974 float pre_gain = 1.f;
1975 if (config_.pre_amplifier.enabled) {
1976 pre_gain *= config_.pre_amplifier.fixed_gain_factor;
1977 }
1978 if (config_.capture_level_adjustment.enabled) {
1979 pre_gain *= config_.capture_level_adjustment.pre_gain_factor;
1980 }
1981
1982 submodules_.capture_levels_adjuster =
1983 std::make_unique<CaptureLevelsAdjuster>(
1984 config_.capture_level_adjustment.analog_mic_gain_emulation.enabled,
1985 config_.capture_level_adjustment.analog_mic_gain_emulation
1986 .initial_level,
1987 pre_gain, config_.capture_level_adjustment.post_gain_factor);
Alex Loikob5c9a792018-04-16 16:31:22 +02001988 } else {
Per Åhgrendb5d7282021-03-15 16:31:04 +00001989 submodules_.capture_levels_adjuster.reset();
Alex Loikob5c9a792018-04-16 16:31:22 +02001990 }
1991}
1992
ivoc9f4a4a02016-10-28 05:39:16 -07001993void AudioProcessingImpl::InitializeResidualEchoDetector() {
saza1d600522019-10-18 13:29:43 +02001994 RTC_DCHECK(submodules_.echo_detector);
1995 submodules_.echo_detector->Initialize(
Gustaf Ullberg422b9e02019-10-09 13:02:14 +02001996 proc_fullband_sample_rate_hz(), 1,
Ivo Creusenb1facc12018-04-12 16:15:58 +02001997 formats_.render_processing_format.sample_rate_hz(), 1);
ivoc9f4a4a02016-10-28 05:39:16 -07001998}
1999
Valeriia Nemychnikovaf06eb572018-08-29 10:37:09 +02002000void AudioProcessingImpl::InitializeAnalyzer() {
saza1d600522019-10-18 13:29:43 +02002001 if (submodules_.capture_analyzer) {
2002 submodules_.capture_analyzer->Initialize(proc_fullband_sample_rate_hz(),
2003 num_proc_channels());
Valeriia Nemychnikovaf06eb572018-08-29 10:37:09 +02002004 }
2005}
2006
Sam Zackrisson0beac582017-09-25 12:04:02 +02002007void AudioProcessingImpl::InitializePostProcessor() {
saza1d600522019-10-18 13:29:43 +02002008 if (submodules_.capture_post_processor) {
2009 submodules_.capture_post_processor->Initialize(
Gustaf Ullberg422b9e02019-10-09 13:02:14 +02002010 proc_fullband_sample_rate_hz(), num_proc_channels());
Sam Zackrisson0beac582017-09-25 12:04:02 +02002011 }
2012}
2013
Alex Loiko5825aa62017-12-18 16:02:40 +01002014void AudioProcessingImpl::InitializePreProcessor() {
saza1d600522019-10-18 13:29:43 +02002015 if (submodules_.render_pre_processor) {
2016 submodules_.render_pre_processor->Initialize(
Alex Loiko5825aa62017-12-18 16:02:40 +01002017 formats_.render_processing_format.sample_rate_hz(),
2018 formats_.render_processing_format.num_channels());
2019 }
2020}
2021
aleloi868f32f2017-05-23 07:20:05 -07002022void AudioProcessingImpl::WriteAecDumpConfigMessage(bool forced) {
2023 if (!aec_dump_) {
2024 return;
2025 }
Per Åhgrenf204faf2019-04-25 15:18:06 +02002026
2027 std::string experiments_description = "";
aleloi868f32f2017-05-23 07:20:05 -07002028 // TODO(peah): Add semicolon-separated concatenations of experiment
2029 // descriptions for other submodules.
Per Åhgren0695df12020-01-13 14:43:13 +01002030 if (config_.gain_controller1.analog_gain_controller.clipped_level_min !=
2031 kClippedLevelMin) {
aleloi868f32f2017-05-23 07:20:05 -07002032 experiments_description += "AgcClippingLevelExperiment;";
2033 }
Sam Zackrisson701bd172020-02-18 14:50:28 +01002034 if (!!submodules_.capture_post_processor) {
2035 experiments_description += "CapturePostProcessor;";
2036 }
2037 if (!!submodules_.render_pre_processor) {
2038 experiments_description += "RenderPreProcessor;";
2039 }
Gustaf Ullbergce045ac2017-10-16 13:49:04 +02002040 if (capture_nonlocked_.echo_controller_enabled) {
2041 experiments_description += "EchoController;";
aleloi868f32f2017-05-23 07:20:05 -07002042 }
Alessio Bazzica270f7b52017-10-13 11:05:17 +02002043 if (config_.gain_controller2.enabled) {
2044 experiments_description += "GainController2;";
2045 }
aleloi868f32f2017-05-23 07:20:05 -07002046
2047 InternalAPMConfig apm_config;
2048
Per Åhgren200feba2019-03-06 04:16:46 +01002049 apm_config.aec_enabled = config_.echo_canceller.enabled;
Per Åhgren62ea0aa2019-12-09 10:18:44 +01002050 apm_config.aec_delay_agnostic_enabled = false;
2051 apm_config.aec_extended_filter_enabled = false;
2052 apm_config.aec_suppression_level = 0;
aleloi868f32f2017-05-23 07:20:05 -07002053
saza1d600522019-10-18 13:29:43 +02002054 apm_config.aecm_enabled = !!submodules_.echo_control_mobile;
aleloi868f32f2017-05-23 07:20:05 -07002055 apm_config.aecm_comfort_noise_enabled =
saza1d600522019-10-18 13:29:43 +02002056 submodules_.echo_control_mobile &&
2057 submodules_.echo_control_mobile->is_comfort_noise_enabled();
Per Åhgrenb6e24d72019-04-29 12:14:50 +02002058 apm_config.aecm_routing_mode =
saza1d600522019-10-18 13:29:43 +02002059 submodules_.echo_control_mobile
2060 ? static_cast<int>(submodules_.echo_control_mobile->routing_mode())
Per Åhgrenb6e24d72019-04-29 12:14:50 +02002061 : 0;
aleloi868f32f2017-05-23 07:20:05 -07002062
Per Åhgren0695df12020-01-13 14:43:13 +01002063 apm_config.agc_enabled = !!submodules_.gain_control;
2064
2065 apm_config.agc_mode = submodules_.gain_control
2066 ? static_cast<int>(submodules_.gain_control->mode())
2067 : GainControl::kAdaptiveAnalog;
aleloi868f32f2017-05-23 07:20:05 -07002068 apm_config.agc_limiter_enabled =
Per Åhgren0695df12020-01-13 14:43:13 +01002069 submodules_.gain_control ? submodules_.gain_control->is_limiter_enabled()
2070 : false;
Per Åhgren0e3198e2019-11-18 08:52:22 +01002071 apm_config.noise_robust_agc_enabled = !!submodules_.agc_manager;
aleloi868f32f2017-05-23 07:20:05 -07002072
2073 apm_config.hpf_enabled = config_.high_pass_filter.enabled;
2074
saza0bad15f2019-10-16 11:46:11 +02002075 apm_config.ns_enabled = config_.noise_suppression.enabled;
2076 apm_config.ns_level = static_cast<int>(config_.noise_suppression.level);
aleloi868f32f2017-05-23 07:20:05 -07002077
2078 apm_config.transient_suppression_enabled =
Per Åhgrenc0734712020-01-02 15:15:36 +01002079 config_.transient_suppression.enabled;
aleloi868f32f2017-05-23 07:20:05 -07002080 apm_config.experiments_description = experiments_description;
Alex Loiko5feb30e2018-04-16 13:52:32 +02002081 apm_config.pre_amplifier_enabled = config_.pre_amplifier.enabled;
2082 apm_config.pre_amplifier_fixed_gain_factor =
2083 config_.pre_amplifier.fixed_gain_factor;
aleloi868f32f2017-05-23 07:20:05 -07002084
2085 if (!forced && apm_config == apm_config_for_aec_dump_) {
2086 return;
2087 }
2088 aec_dump_->WriteConfig(apm_config);
2089 apm_config_for_aec_dump_ = apm_config;
2090}
2091
2092void AudioProcessingImpl::RecordUnprocessedCaptureStream(
2093 const float* const* src) {
2094 RTC_DCHECK(aec_dump_);
2095 WriteAecDumpConfigMessage(false);
2096
2097 const size_t channel_size = formats_.api_format.input_stream().num_frames();
2098 const size_t num_channels = formats_.api_format.input_stream().num_channels();
2099 aec_dump_->AddCaptureStreamInput(
Alex Loikoe36e8bb2018-02-16 11:54:07 +01002100 AudioFrameView<const float>(src, num_channels, channel_size));
aleloi868f32f2017-05-23 07:20:05 -07002101 RecordAudioProcessingState();
2102}
2103
2104void AudioProcessingImpl::RecordUnprocessedCaptureStream(
Per Åhgren645f24c2020-03-16 12:06:02 +01002105 const int16_t* const data,
2106 const StreamConfig& config) {
aleloi868f32f2017-05-23 07:20:05 -07002107 RTC_DCHECK(aec_dump_);
2108 WriteAecDumpConfigMessage(false);
2109
Per Åhgren645f24c2020-03-16 12:06:02 +01002110 aec_dump_->AddCaptureStreamInput(data, config.num_channels(),
2111 config.num_frames());
aleloi868f32f2017-05-23 07:20:05 -07002112 RecordAudioProcessingState();
2113}
2114
2115void AudioProcessingImpl::RecordProcessedCaptureStream(
2116 const float* const* processed_capture_stream) {
2117 RTC_DCHECK(aec_dump_);
2118
2119 const size_t channel_size = formats_.api_format.output_stream().num_frames();
2120 const size_t num_channels =
2121 formats_.api_format.output_stream().num_channels();
Alex Loikoe36e8bb2018-02-16 11:54:07 +01002122 aec_dump_->AddCaptureStreamOutput(AudioFrameView<const float>(
2123 processed_capture_stream, num_channels, channel_size));
aleloi868f32f2017-05-23 07:20:05 -07002124 aec_dump_->WriteCaptureStreamMessage();
2125}
2126
2127void AudioProcessingImpl::RecordProcessedCaptureStream(
Per Åhgren645f24c2020-03-16 12:06:02 +01002128 const int16_t* const data,
2129 const StreamConfig& config) {
aleloi868f32f2017-05-23 07:20:05 -07002130 RTC_DCHECK(aec_dump_);
2131
Per Åhgren645f24c2020-03-16 12:06:02 +01002132 aec_dump_->AddCaptureStreamOutput(data, config.num_channels(),
Per Åhgren088329f2020-03-18 21:59:52 +01002133 config.num_frames());
aleloi868f32f2017-05-23 07:20:05 -07002134 aec_dump_->WriteCaptureStreamMessage();
2135}
2136
2137void AudioProcessingImpl::RecordAudioProcessingState() {
2138 RTC_DCHECK(aec_dump_);
2139 AecDump::AudioProcessingState audio_proc_state;
2140 audio_proc_state.delay = capture_nonlocked_.stream_delay_ms;
Per Åhgren62ea0aa2019-12-09 10:18:44 +01002141 audio_proc_state.drift = 0;
Markus Handell02ba1d22020-05-14 14:31:18 +02002142 audio_proc_state.level = recommended_stream_analog_level_locked();
aleloi868f32f2017-05-23 07:20:05 -07002143 audio_proc_state.keypress = capture_.key_pressed;
2144 aec_dump_->AddAudioProcessingState(audio_proc_state);
2145}
2146
Per Åhgrenc0734712020-01-02 15:15:36 +01002147AudioProcessingImpl::ApmCaptureState::ApmCaptureState()
Sam Zackrisson2d31aea2020-01-17 10:55:09 +01002148 : was_stream_delay_set(false),
Per Åhgren0a144a72021-02-09 08:47:51 +01002149 capture_output_used(true),
Per Åhgren55bc0772021-03-12 14:18:36 +00002150 capture_output_used_last_frame(true),
kwiberg83ffe452016-08-29 14:46:07 -07002151 key_pressed(false),
peahde65ddc2016-09-16 15:02:15 -07002152 capture_processing_format(kSampleRate16kHz),
peah67995532017-04-10 14:12:41 -07002153 split_rate(kSampleRate16kHz),
Per Åhgren88cf0502018-07-16 17:08:41 +02002154 echo_path_gain_change(false),
Per Åhgrend2650d12018-10-02 17:00:59 +02002155 prev_analog_mic_level(-1),
Per Åhgrendb5d7282021-03-15 16:31:04 +00002156 prev_pre_adjustment_gain(-1.f),
Fredrik Hernqvistca362852019-05-10 15:50:02 +02002157 playout_volume(-1),
2158 prev_playout_volume(-1) {}
kwiberg83ffe452016-08-29 14:46:07 -07002159
2160AudioProcessingImpl::ApmCaptureState::~ApmCaptureState() = default;
2161
Per Åhgrena1351272019-08-15 12:15:46 +02002162void AudioProcessingImpl::ApmCaptureState::KeyboardInfo::Extract(
2163 const float* const* data,
2164 const StreamConfig& stream_config) {
2165 if (stream_config.has_keyboard()) {
2166 keyboard_data = data[stream_config.num_channels()];
2167 } else {
2168 keyboard_data = NULL;
2169 }
2170 num_keyboard_frames = stream_config.num_frames();
2171}
2172
kwiberg83ffe452016-08-29 14:46:07 -07002173AudioProcessingImpl::ApmRenderState::ApmRenderState() = default;
2174
2175AudioProcessingImpl::ApmRenderState::~ApmRenderState() = default;
2176
Per Åhgrencf4c8722019-12-30 14:32:14 +01002177AudioProcessingImpl::ApmStatsReporter::ApmStatsReporter()
2178 : stats_message_queue_(1) {}
2179
2180AudioProcessingImpl::ApmStatsReporter::~ApmStatsReporter() = default;
2181
2182AudioProcessingStats AudioProcessingImpl::ApmStatsReporter::GetStatistics() {
Markus Handell0df0fae2020-07-07 15:53:34 +02002183 MutexLock lock_stats(&mutex_stats_);
Per Åhgrencf4c8722019-12-30 14:32:14 +01002184 bool new_stats_available = stats_message_queue_.Remove(&cached_stats_);
2185 // If the message queue is full, return the cached stats.
2186 static_cast<void>(new_stats_available);
2187
2188 return cached_stats_;
2189}
2190
2191void AudioProcessingImpl::ApmStatsReporter::UpdateStatistics(
2192 const AudioProcessingStats& new_stats) {
2193 AudioProcessingStats stats_to_queue = new_stats;
2194 bool stats_message_passed = stats_message_queue_.Insert(&stats_to_queue);
2195 // If the message queue is full, discard the new stats.
2196 static_cast<void>(stats_message_passed);
2197}
2198
niklase@google.com470e71d2011-07-07 08:21:25 +00002199} // namespace webrtc