blob: 31a6a14d6a760688e313e3eebfb47cff66055ea6 [file] [log] [blame]
niklase@google.com470e71d2011-07-07 08:21:25 +00001/*
andrew@webrtc.org40654032012-01-30 20:51:15 +00002 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
niklase@google.com470e71d2011-07-07 08:21:25 +00003 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020011#include "modules/audio_processing/audio_processing_impl.h"
niklase@google.com470e71d2011-07-07 08:21:25 +000012
Michael Graczyk86c6d332015-07-23 11:41:39 -070013#include <algorithm>
Yves Gerey988cc082018-10-23 12:03:01 +020014#include <cstdint>
Mirko Bonadei317a1f02019-09-17 17:06:18 +020015#include <memory>
alessiob3ec96df2017-05-22 06:57:06 -070016#include <string>
Yves Gerey988cc082018-10-23 12:03:01 +020017#include <type_traits>
18#include <utility>
niklase@google.com470e71d2011-07-07 08:21:25 +000019
Alessio Bazzicaefbe3af2022-03-18 12:39:00 +010020#include "absl/strings/match.h"
Yves Gerey988cc082018-10-23 12:03:01 +020021#include "absl/types/optional.h"
22#include "api/array_view.h"
Per Åhgren645f24c2020-03-16 12:06:02 +010023#include "api/audio/audio_frame.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020024#include "common_audio/audio_converter.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020025#include "common_audio/include/audio_util.h"
Per Åhgren09e9a832020-05-11 11:03:47 +020026#include "modules/audio_processing/aec_dump/aec_dump_factory.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020027#include "modules/audio_processing/audio_buffer.h"
Yves Gerey988cc082018-10-23 12:03:01 +020028#include "modules/audio_processing/include/audio_frame_view.h"
Per Åhgren13735822018-02-12 21:42:56 +010029#include "modules/audio_processing/logging/apm_data_dumper.h"
Sam Zackrissonb37e59d2020-04-27 08:39:33 +020030#include "modules/audio_processing/optionally_built_submodule_creators.h"
Steve Anton10542f22019-01-11 09:11:00 -080031#include "rtc_base/atomic_ops.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020032#include "rtc_base/checks.h"
33#include "rtc_base/logging.h"
Steve Anton10542f22019-01-11 09:11:00 -080034#include "rtc_base/time_utils.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020035#include "rtc_base/trace_event.h"
Alessio Bazzica0441bb62021-08-10 15:23:23 +020036#include "system_wrappers/include/denormal_disabler.h"
Sam Zackrissonfeee1e42019-09-20 07:50:35 +020037#include "system_wrappers/include/field_trial.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020038#include "system_wrappers/include/metrics.h"
andrew@webrtc.org7bf26462011-12-03 00:03:31 +000039
Michael Graczyk86c6d332015-07-23 11:41:39 -070040#define RETURN_ON_ERR(expr) \
41 do { \
42 int err = (expr); \
43 if (err != kNoError) { \
44 return err; \
45 } \
andrew@webrtc.org60730cf2014-01-07 17:45:09 +000046 } while (0)
47
niklase@google.com470e71d2011-07-07 08:21:25 +000048namespace webrtc {
aluebsdf6416a2016-03-16 18:26:35 -070049
Michael Graczyk86c6d332015-07-23 11:41:39 -070050namespace {
51
peah2ace3f92016-09-10 04:42:27 -070052bool SampleRateSupportsMultiBand(int sample_rate_hz) {
aluebsdf6416a2016-03-16 18:26:35 -070053 return sample_rate_hz == AudioProcessing::kSampleRate32kHz ||
54 sample_rate_hz == AudioProcessing::kSampleRate48kHz;
55}
56
Per Åhgrenc0424252019-12-10 13:04:15 +010057// Checks whether the high-pass filter should be done in the full-band.
58bool EnforceSplitBandHpf() {
59 return field_trial::IsEnabled("WebRTC-FullBandHpfKillSwitch");
60}
61
Per Åhgrenb2b58d82019-12-02 14:59:40 +010062// Checks whether AEC3 should be allowed to decide what the default
63// configuration should be based on the render and capture channel configuration
64// at hand.
65bool UseSetupSpecificDefaultAec3Congfig() {
66 return !field_trial::IsEnabled(
67 "WebRTC-Aec3SetupSpecificDefaultConfigDefaultsKillSwitch");
68}
69
Alessio Bazzicaefbe3af2022-03-18 12:39:00 +010070// If the "WebRTC-Audio-TransientSuppressorVadMode" field trial is unspecified,
71// returns `TransientSuppressor::VadMode::kDefault`, otherwise parses the field
72// trial and returns the specified mode:
73// - WebRTC-Audio-TransientSuppressorVadMode/Enabled-Default returns `kDefault`;
74// - WebRTC-Audio-TransientSuppressorVadMode/Enabled-RnnVad returns `kRnnVad`;
75// - WebRTC-Audio-TransientSuppressorVadMode/Enabled-NoVad returns `kNoVad`.
76TransientSuppressor::VadMode GetTransientSuppressorVadMode() {
77 constexpr char kFieldTrial[] = "WebRTC-Audio-TransientSuppressorVadMode";
78 std::string full_name = webrtc::field_trial::FindFullName(kFieldTrial);
79 if (full_name.empty() || absl::EndsWith(full_name, "-Default")) {
80 return TransientSuppressor::VadMode::kDefault;
81 }
82 if (absl::EndsWith(full_name, "-RnnVad")) {
83 return TransientSuppressor::VadMode::kRnnVad;
84 }
85 if (absl::EndsWith(full_name, "-NoVad")) {
86 return TransientSuppressor::VadMode::kNoVad;
87 }
88 // Fallback to default.
89 RTC_LOG(LS_WARNING) << "Invalid parameter for " << kFieldTrial;
90 return TransientSuppressor::VadMode::kDefault;
91}
92
Per Åhgrenc8626b62019-08-23 15:49:51 +020093// Identify the native processing rate that best handles a sample rate.
Per Åhgrenfcbe4072019-09-15 00:27:58 +020094int SuitableProcessRate(int minimum_rate,
95 int max_splitting_rate,
96 bool band_splitting_required) {
Per Åhgrenc8626b62019-08-23 15:49:51 +020097 const int uppermost_native_rate =
Per Åhgrenfcbe4072019-09-15 00:27:58 +020098 band_splitting_required ? max_splitting_rate : 48000;
Per Åhgrenc8626b62019-08-23 15:49:51 +020099 for (auto rate : {16000, 32000, 48000}) {
peah2ace3f92016-09-10 04:42:27 -0700100 if (rate >= uppermost_native_rate) {
101 return uppermost_native_rate;
102 }
103 if (rate >= minimum_rate) {
aluebsdf6416a2016-03-16 18:26:35 -0700104 return rate;
105 }
106 }
Artem Titovd3251962021-11-15 16:57:07 +0100107 RTC_DCHECK_NOTREACHED();
peah2ace3f92016-09-10 04:42:27 -0700108 return uppermost_native_rate;
aluebsdf6416a2016-03-16 18:26:35 -0700109}
110
Sam Zackrissonf0d1c032019-03-27 13:28:08 +0100111GainControl::Mode Agc1ConfigModeToInterfaceMode(
112 AudioProcessing::Config::GainController1::Mode mode) {
113 using Agc1Config = AudioProcessing::Config::GainController1;
114 switch (mode) {
115 case Agc1Config::kAdaptiveAnalog:
116 return GainControl::kAdaptiveAnalog;
117 case Agc1Config::kAdaptiveDigital:
118 return GainControl::kAdaptiveDigital;
119 case Agc1Config::kFixedDigital:
120 return GainControl::kFixedDigital;
121 }
Karl Wibergc95b9392020-11-08 00:49:37 +0100122 RTC_CHECK_NOTREACHED();
Sam Zackrissonf0d1c032019-03-27 13:28:08 +0100123}
124
Per Åhgren19775cb2021-03-12 23:08:09 +0000125bool MinimizeProcessingForUnusedOutput() {
126 return !field_trial::IsEnabled("WebRTC-MutedStateKillSwitch");
127}
128
peah9e6a2902017-05-15 07:19:21 -0700129// Maximum lengths that frame of samples being passed from the render side to
130// the capture side can have (does not apply to AEC3).
131static const size_t kMaxAllowedValuesOfSamplesPerBand = 160;
132static const size_t kMaxAllowedValuesOfSamplesPerFrame = 480;
133
peah764e3642016-10-22 05:04:30 -0700134// Maximum number of frames to buffer in the render queue.
135// TODO(peah): Decrease this once we properly handle hugely unbalanced
136// reverse and forward call numbers.
137static const size_t kMaxNumFramesToBuffer = 100;
Alessio Bazzica3438a932020-10-14 12:47:50 +0200138
Sam Zackrisson03cb7e52021-12-06 15:40:04 +0100139void PackRenderAudioBufferForEchoDetector(const AudioBuffer& audio,
140 std::vector<float>& packed_buffer) {
141 packed_buffer.clear();
142 packed_buffer.insert(packed_buffer.end(), audio.channels_const()[0],
143 audio.channels_const()[0] + audio.num_frames());
144}
145
Michael Graczyk86c6d332015-07-23 11:41:39 -0700146} // namespace
andrew@webrtc.org60730cf2014-01-07 17:45:09 +0000147
148// Throughout webrtc, it's assumed that success is represented by zero.
kwiberg@webrtc.org2ebfac52015-01-14 10:51:54 +0000149static_assert(AudioProcessing::kNoError == 0, "kNoError must be zero");
andrew@webrtc.org60730cf2014-01-07 17:45:09 +0000150
saza1d600522019-10-18 13:29:43 +0200151AudioProcessingImpl::SubmoduleStates::SubmoduleStates(
Alex Loiko5825aa62017-12-18 16:02:40 +0100152 bool capture_post_processor_enabled,
Valeriia Nemychnikovaf06eb572018-08-29 10:37:09 +0200153 bool render_pre_processor_enabled,
154 bool capture_analyzer_enabled)
Alex Loiko5825aa62017-12-18 16:02:40 +0100155 : capture_post_processor_enabled_(capture_post_processor_enabled),
Valeriia Nemychnikovaf06eb572018-08-29 10:37:09 +0200156 render_pre_processor_enabled_(render_pre_processor_enabled),
157 capture_analyzer_enabled_(capture_analyzer_enabled) {}
peah2ace3f92016-09-10 04:42:27 -0700158
saza1d600522019-10-18 13:29:43 +0200159bool AudioProcessingImpl::SubmoduleStates::Update(
Sam Zackrissoncb1b5562018-09-28 14:15:09 +0200160 bool high_pass_filter_enabled,
peah2ace3f92016-09-10 04:42:27 -0700161 bool mobile_echo_controller_enabled,
162 bool noise_suppressor_enabled,
peah2ace3f92016-09-10 04:42:27 -0700163 bool adaptive_gain_controller_enabled,
alessiob3ec96df2017-05-22 06:57:06 -0700164 bool gain_controller2_enabled,
Hanna Silen0c1ad292022-06-16 16:35:45 +0200165 bool voice_activity_detector_enabled,
Per Åhgrendb5d7282021-03-15 16:31:04 +0000166 bool gain_adjustment_enabled,
Gustaf Ullbergce045ac2017-10-16 13:49:04 +0200167 bool echo_controller_enabled,
peah2ace3f92016-09-10 04:42:27 -0700168 bool transient_suppressor_enabled) {
169 bool changed = false;
Sam Zackrissoncb1b5562018-09-28 14:15:09 +0200170 changed |= (high_pass_filter_enabled != high_pass_filter_enabled_);
peah2ace3f92016-09-10 04:42:27 -0700171 changed |=
172 (mobile_echo_controller_enabled != mobile_echo_controller_enabled_);
173 changed |= (noise_suppressor_enabled != noise_suppressor_enabled_);
174 changed |=
peah2ace3f92016-09-10 04:42:27 -0700175 (adaptive_gain_controller_enabled != adaptive_gain_controller_enabled_);
Per Åhgren6ee75fd2019-04-26 11:33:37 +0200176 changed |= (gain_controller2_enabled != gain_controller2_enabled_);
Hanna Silen0c1ad292022-06-16 16:35:45 +0200177 changed |=
178 (voice_activity_detector_enabled != voice_activity_detector_enabled_);
Per Åhgrendb5d7282021-03-15 16:31:04 +0000179 changed |= (gain_adjustment_enabled != gain_adjustment_enabled_);
Gustaf Ullbergce045ac2017-10-16 13:49:04 +0200180 changed |= (echo_controller_enabled != echo_controller_enabled_);
peah2ace3f92016-09-10 04:42:27 -0700181 changed |= (transient_suppressor_enabled != transient_suppressor_enabled_);
182 if (changed) {
Sam Zackrissoncb1b5562018-09-28 14:15:09 +0200183 high_pass_filter_enabled_ = high_pass_filter_enabled;
peah2ace3f92016-09-10 04:42:27 -0700184 mobile_echo_controller_enabled_ = mobile_echo_controller_enabled;
185 noise_suppressor_enabled_ = noise_suppressor_enabled;
peah2ace3f92016-09-10 04:42:27 -0700186 adaptive_gain_controller_enabled_ = adaptive_gain_controller_enabled;
alessiob3ec96df2017-05-22 06:57:06 -0700187 gain_controller2_enabled_ = gain_controller2_enabled;
Hanna Silen0c1ad292022-06-16 16:35:45 +0200188 voice_activity_detector_enabled_ = voice_activity_detector_enabled;
Per Åhgrendb5d7282021-03-15 16:31:04 +0000189 gain_adjustment_enabled_ = gain_adjustment_enabled;
Gustaf Ullbergce045ac2017-10-16 13:49:04 +0200190 echo_controller_enabled_ = echo_controller_enabled;
peah2ace3f92016-09-10 04:42:27 -0700191 transient_suppressor_enabled_ = transient_suppressor_enabled;
192 }
193
194 changed |= first_update_;
195 first_update_ = false;
196 return changed;
197}
198
saza1d600522019-10-18 13:29:43 +0200199bool AudioProcessingImpl::SubmoduleStates::CaptureMultiBandSubModulesActive()
peah2ace3f92016-09-10 04:42:27 -0700200 const {
Alessio Bazzica1db0a262022-02-15 14:18:09 +0000201 return CaptureMultiBandProcessingPresent();
peah2ace3f92016-09-10 04:42:27 -0700202}
203
saza1d600522019-10-18 13:29:43 +0200204bool AudioProcessingImpl::SubmoduleStates::CaptureMultiBandProcessingPresent()
205 const {
Gustaf Ullberg8675eee2019-10-09 13:34:36 +0200206 // If echo controller is present, assume it performs active processing.
207 return CaptureMultiBandProcessingActive(/*ec_processing_active=*/true);
208}
209
saza1d600522019-10-18 13:29:43 +0200210bool AudioProcessingImpl::SubmoduleStates::CaptureMultiBandProcessingActive(
Gustaf Ullberg8675eee2019-10-09 13:34:36 +0200211 bool ec_processing_active) const {
Per Åhgren62ea0aa2019-12-09 10:18:44 +0100212 return high_pass_filter_enabled_ || mobile_echo_controller_enabled_ ||
213 noise_suppressor_enabled_ || adaptive_gain_controller_enabled_ ||
Gustaf Ullberg8675eee2019-10-09 13:34:36 +0200214 (echo_controller_enabled_ && ec_processing_active);
peah2ace3f92016-09-10 04:42:27 -0700215}
216
saza1d600522019-10-18 13:29:43 +0200217bool AudioProcessingImpl::SubmoduleStates::CaptureFullBandProcessingActive()
peah23ac8b42017-05-23 05:33:56 -0700218 const {
Alex Loikob5c9a792018-04-16 16:31:22 +0200219 return gain_controller2_enabled_ || capture_post_processor_enabled_ ||
Per Åhgrendb5d7282021-03-15 16:31:04 +0000220 gain_adjustment_enabled_;
peah23ac8b42017-05-23 05:33:56 -0700221}
222
saza1d600522019-10-18 13:29:43 +0200223bool AudioProcessingImpl::SubmoduleStates::CaptureAnalyzerActive() const {
Valeriia Nemychnikovaf06eb572018-08-29 10:37:09 +0200224 return capture_analyzer_enabled_;
225}
226
saza1d600522019-10-18 13:29:43 +0200227bool AudioProcessingImpl::SubmoduleStates::RenderMultiBandSubModulesActive()
peah2ace3f92016-09-10 04:42:27 -0700228 const {
Per Åhgren62ea0aa2019-12-09 10:18:44 +0100229 return RenderMultiBandProcessingActive() || mobile_echo_controller_enabled_ ||
230 adaptive_gain_controller_enabled_ || echo_controller_enabled_;
peah2ace3f92016-09-10 04:42:27 -0700231}
232
saza1d600522019-10-18 13:29:43 +0200233bool AudioProcessingImpl::SubmoduleStates::RenderFullBandProcessingActive()
Alex Loiko5825aa62017-12-18 16:02:40 +0100234 const {
235 return render_pre_processor_enabled_;
236}
237
saza1d600522019-10-18 13:29:43 +0200238bool AudioProcessingImpl::SubmoduleStates::RenderMultiBandProcessingActive()
peah2ace3f92016-09-10 04:42:27 -0700239 const {
peah2ace3f92016-09-10 04:42:27 -0700240 return false;
peah2ace3f92016-09-10 04:42:27 -0700241}
242
saza1d600522019-10-18 13:29:43 +0200243bool AudioProcessingImpl::SubmoduleStates::HighPassFilteringRequired() const {
Per Åhgren62ea0aa2019-12-09 10:18:44 +0100244 return high_pass_filter_enabled_ || mobile_echo_controller_enabled_ ||
245 noise_suppressor_enabled_;
Sam Zackrissoncb1b5562018-09-28 14:15:09 +0200246}
247
Alessio Bazzicabe1b8982021-09-17 08:26:10 +0200248AudioProcessingImpl::AudioProcessingImpl()
Alessio Bazzica20a9ac62021-10-14 10:55:08 +0200249 : AudioProcessingImpl(/*config=*/{},
250 /*capture_post_processor=*/nullptr,
Sam Zackrissonfeee1e42019-09-20 07:50:35 +0200251 /*render_pre_processor=*/nullptr,
252 /*echo_control_factory=*/nullptr,
253 /*echo_detector=*/nullptr,
254 /*capture_analyzer=*/nullptr) {}
aluebs@webrtc.orgd82f55d2015-01-15 18:07:21 +0000255
Per Åhgren13735822018-02-12 21:42:56 +0100256int AudioProcessingImpl::instance_count_ = 0;
257
Sam Zackrisson0beac582017-09-25 12:04:02 +0200258AudioProcessingImpl::AudioProcessingImpl(
Alessio Bazzica20a9ac62021-10-14 10:55:08 +0200259 const AudioProcessing::Config& config,
Alex Loiko5825aa62017-12-18 16:02:40 +0100260 std::unique_ptr<CustomProcessing> capture_post_processor,
261 std::unique_ptr<CustomProcessing> render_pre_processor,
Gustaf Ullberg002ef282017-10-12 15:13:17 +0200262 std::unique_ptr<EchoControlFactory> echo_control_factory,
Valeriia Nemychnikovaf06eb572018-08-29 10:37:09 +0200263 rtc::scoped_refptr<EchoDetector> echo_detector,
264 std::unique_ptr<CustomAudioAnalyzer> capture_analyzer)
Per Åhgren13735822018-02-12 21:42:56 +0100265 : data_dumper_(
266 new ApmDataDumper(rtc::AtomicOps::Increment(&instance_count_))),
Per Åhgrenb2b58d82019-12-02 14:59:40 +0100267 use_setup_specific_default_aec3_config_(
268 UseSetupSpecificDefaultAec3Congfig()),
Alessio Bazzica0441bb62021-08-10 15:23:23 +0200269 use_denormal_disabler_(
270 !field_trial::IsEnabled("WebRTC-ApmDenormalDisablerKillSwitch")),
Alessio Bazzicaefbe3af2022-03-18 12:39:00 +0100271 transient_suppressor_vad_mode_(GetTransientSuppressorVadMode()),
Per Åhgren652ada52021-03-03 10:52:44 +0000272 capture_runtime_settings_(RuntimeSettingQueueSize()),
273 render_runtime_settings_(RuntimeSettingQueueSize()),
Alex Loiko73ec0192018-05-15 10:52:28 +0200274 capture_runtime_settings_enqueuer_(&capture_runtime_settings_),
275 render_runtime_settings_enqueuer_(&render_runtime_settings_),
Gustaf Ullberg002ef282017-10-12 15:13:17 +0200276 echo_control_factory_(std::move(echo_control_factory)),
Alessio Bazzica20a9ac62021-10-14 10:55:08 +0200277 config_(config),
Valeriia Nemychnikovaf06eb572018-08-29 10:37:09 +0200278 submodule_states_(!!capture_post_processor,
279 !!render_pre_processor,
280 !!capture_analyzer),
saza1d600522019-10-18 13:29:43 +0200281 submodules_(std::move(capture_post_processor),
282 std::move(render_pre_processor),
283 std::move(echo_detector),
Per Åhgren3daedb62019-11-22 12:11:40 +0100284 std::move(capture_analyzer)),
Per Åhgren0695df12020-01-13 14:43:13 +0100285 constants_(!field_trial::IsEnabled(
Sam Zackrissonfeee1e42019-09-20 07:50:35 +0200286 "WebRTC-ApmExperimentalMultiChannelRenderKillSwitch"),
287 !field_trial::IsEnabled(
Per Åhgrenc0424252019-12-10 13:04:15 +0100288 "WebRTC-ApmExperimentalMultiChannelCaptureKillSwitch"),
Per Åhgren19775cb2021-03-12 23:08:09 +0000289 EnforceSplitBandHpf(),
Gustaf Ullberga399c822021-05-18 12:17:56 +0200290 MinimizeProcessingForUnusedOutput(),
291 field_trial::IsEnabled("WebRTC-TransientSuppressorForcedOff")),
Per Åhgren19775cb2021-03-12 23:08:09 +0000292 capture_(),
Alessio Bazzicacc22f512018-08-30 13:01:34 +0200293 capture_nonlocked_() {
Sam Zackrisson72cc71c2019-10-21 12:54:02 +0200294 RTC_LOG(LS_INFO) << "Injected APM submodules:"
Jonas Olssonb2b20312020-01-14 12:11:31 +0100295 "\nEcho control factory: "
296 << !!echo_control_factory_
Sam Zackrisson72cc71c2019-10-21 12:54:02 +0200297 << "\nEcho detector: " << !!submodules_.echo_detector
298 << "\nCapture analyzer: " << !!submodules_.capture_analyzer
299 << "\nCapture post processor: "
300 << !!submodules_.capture_post_processor
301 << "\nRender pre processor: "
302 << !!submodules_.render_pre_processor;
Alessio Bazzica0441bb62021-08-10 15:23:23 +0200303 RTC_LOG(LS_INFO) << "Denormal disabler: "
304 << (DenormalDisabler::IsSupported() ? "supported"
305 : "unsupported");
Sam Zackrisson72cc71c2019-10-21 12:54:02 +0200306
Sam Zackrisson421c8592019-02-11 13:39:46 +0100307 // Mark Echo Controller enabled if a factory is injected.
308 capture_nonlocked_.echo_controller_enabled =
309 static_cast<bool>(echo_control_factory_);
niklase@google.com470e71d2011-07-07 08:21:25 +0000310
Per Åhgren0ade9832020-09-01 23:57:20 +0200311 Initialize();
niklase@google.com470e71d2011-07-07 08:21:25 +0000312}
313
Per Åhgren0e3198e2019-11-18 08:52:22 +0100314AudioProcessingImpl::~AudioProcessingImpl() = default;
niklase@google.com470e71d2011-07-07 08:21:25 +0000315
niklase@google.com470e71d2011-07-07 08:21:25 +0000316int AudioProcessingImpl::Initialize() {
peahdf3efa82015-11-28 12:35:15 -0800317 // Run in a single-threaded manner during initialization.
Markus Handell0df0fae2020-07-07 15:53:34 +0200318 MutexLock lock_render(&mutex_render_);
319 MutexLock lock_capture(&mutex_capture_);
Per Åhgren0ade9832020-09-01 23:57:20 +0200320 InitializeLocked();
321 return kNoError;
niklase@google.com470e71d2011-07-07 08:21:25 +0000322}
323
Michael Graczyk86c6d332015-07-23 11:41:39 -0700324int AudioProcessingImpl::Initialize(const ProcessingConfig& processing_config) {
peahdf3efa82015-11-28 12:35:15 -0800325 // Run in a single-threaded manner during initialization.
Markus Handell0df0fae2020-07-07 15:53:34 +0200326 MutexLock lock_render(&mutex_render_);
327 MutexLock lock_capture(&mutex_capture_);
Michael Graczyk86c6d332015-07-23 11:41:39 -0700328 return InitializeLocked(processing_config);
andrew@webrtc.orga8b97372014-03-10 22:26:12 +0000329}
330
peahdf3efa82015-11-28 12:35:15 -0800331int AudioProcessingImpl::MaybeInitializeRender(
peah81b9bfe2015-11-27 02:47:28 -0800332 const ProcessingConfig& processing_config) {
peahdf3efa82015-11-28 12:35:15 -0800333 // Called from both threads. Thread check is therefore not possible.
Oskar Sundbom4b276482019-05-23 14:28:00 +0200334 if (processing_config == formats_.api_format) {
peah192164e2015-11-17 02:16:45 -0800335 return kNoError;
336 }
peahdf3efa82015-11-28 12:35:15 -0800337
Markus Handell0df0fae2020-07-07 15:53:34 +0200338 MutexLock lock_capture(&mutex_capture_);
peah192164e2015-11-17 02:16:45 -0800339 return InitializeLocked(processing_config);
340}
341
Per Åhgren0ade9832020-09-01 23:57:20 +0200342void AudioProcessingImpl::InitializeLocked() {
Per Åhgren4bdced52017-06-27 16:00:38 +0200343 UpdateActiveSubmoduleStates();
344
Per Åhgrend47941e2019-08-22 11:51:13 +0200345 const int render_audiobuffer_sample_rate_hz =
peahdf3efa82015-11-28 12:35:15 -0800346 formats_.api_format.reverse_output_stream().num_frames() == 0
Per Åhgrend47941e2019-08-22 11:51:13 +0200347 ? formats_.render_processing_format.sample_rate_hz()
348 : formats_.api_format.reverse_output_stream().sample_rate_hz();
peahdf3efa82015-11-28 12:35:15 -0800349 if (formats_.api_format.reverse_input_stream().num_channels() > 0) {
350 render_.render_audio.reset(new AudioBuffer(
Per Åhgrend47941e2019-08-22 11:51:13 +0200351 formats_.api_format.reverse_input_stream().sample_rate_hz(),
peahdf3efa82015-11-28 12:35:15 -0800352 formats_.api_format.reverse_input_stream().num_channels(),
Per Åhgrend47941e2019-08-22 11:51:13 +0200353 formats_.render_processing_format.sample_rate_hz(),
peahde65ddc2016-09-16 15:02:15 -0700354 formats_.render_processing_format.num_channels(),
Per Åhgrend47941e2019-08-22 11:51:13 +0200355 render_audiobuffer_sample_rate_hz,
356 formats_.render_processing_format.num_channels()));
peah2ace3f92016-09-10 04:42:27 -0700357 if (formats_.api_format.reverse_input_stream() !=
358 formats_.api_format.reverse_output_stream()) {
kwibergc2b785d2016-02-24 05:22:32 -0800359 render_.render_converter = AudioConverter::Create(
peahdf3efa82015-11-28 12:35:15 -0800360 formats_.api_format.reverse_input_stream().num_channels(),
361 formats_.api_format.reverse_input_stream().num_frames(),
362 formats_.api_format.reverse_output_stream().num_channels(),
kwibergc2b785d2016-02-24 05:22:32 -0800363 formats_.api_format.reverse_output_stream().num_frames());
ekmeyerson60d9b332015-08-14 10:35:55 -0700364 } else {
peahdf3efa82015-11-28 12:35:15 -0800365 render_.render_converter.reset(nullptr);
ekmeyerson60d9b332015-08-14 10:35:55 -0700366 }
Michael Graczyk86c6d332015-07-23 11:41:39 -0700367 } else {
peahdf3efa82015-11-28 12:35:15 -0800368 render_.render_audio.reset(nullptr);
369 render_.render_converter.reset(nullptr);
Michael Graczyk86c6d332015-07-23 11:41:39 -0700370 }
peahce4d9152017-05-19 01:28:05 -0700371
Per Åhgrend47941e2019-08-22 11:51:13 +0200372 capture_.capture_audio.reset(new AudioBuffer(
373 formats_.api_format.input_stream().sample_rate_hz(),
374 formats_.api_format.input_stream().num_channels(),
375 capture_nonlocked_.capture_processing_format.sample_rate_hz(),
376 formats_.api_format.output_stream().num_channels(),
377 formats_.api_format.output_stream().sample_rate_hz(),
378 formats_.api_format.output_stream().num_channels()));
niklase@google.com470e71d2011-07-07 08:21:25 +0000379
Gustaf Ullberg422b9e02019-10-09 13:02:14 +0200380 if (capture_nonlocked_.capture_processing_format.sample_rate_hz() <
381 formats_.api_format.output_stream().sample_rate_hz() &&
382 formats_.api_format.output_stream().sample_rate_hz() == 48000) {
383 capture_.capture_fullband_audio.reset(
384 new AudioBuffer(formats_.api_format.input_stream().sample_rate_hz(),
385 formats_.api_format.input_stream().num_channels(),
386 formats_.api_format.output_stream().sample_rate_hz(),
387 formats_.api_format.output_stream().num_channels(),
388 formats_.api_format.output_stream().sample_rate_hz(),
389 formats_.api_format.output_stream().num_channels()));
390 } else {
391 capture_.capture_fullband_audio.reset();
392 }
393
peah764e3642016-10-22 05:04:30 -0700394 AllocateRenderQueue();
395
Per Åhgren0695df12020-01-13 14:43:13 +0100396 InitializeGainController1();
Per Åhgrenc0734712020-01-02 15:15:36 +0100397 InitializeTransientSuppressor();
Per Åhgren0f14db22020-01-03 14:27:14 +0100398 InitializeHighPassFilter(true);
ivoc9f4a4a02016-10-28 05:39:16 -0700399 InitializeResidualEchoDetector();
Gustaf Ullberg8eb9c7d2017-10-14 08:28:46 +0200400 InitializeEchoController();
Alessio Bazzica38901042021-10-14 12:14:21 +0200401 InitializeGainController2(/*config_has_changed=*/true);
Hanna Silen0c1ad292022-06-16 16:35:45 +0200402 InitializeVoiceActivityDetector(/*config_has_changed=*/true);
saza0bad15f2019-10-16 11:46:11 +0200403 InitializeNoiseSuppressor();
Valeriia Nemychnikovaf06eb572018-08-29 10:37:09 +0200404 InitializeAnalyzer();
Sam Zackrisson0beac582017-09-25 12:04:02 +0200405 InitializePostProcessor();
Alex Loiko5825aa62017-12-18 16:02:40 +0100406 InitializePreProcessor();
Per Åhgrendb5d7282021-03-15 16:31:04 +0000407 InitializeCaptureLevelsAdjuster();
solenberg70f99032015-12-08 11:07:32 -0800408
aleloi868f32f2017-05-23 07:20:05 -0700409 if (aec_dump_) {
Minyue Li656d6092018-08-10 15:38:52 +0200410 aec_dump_->WriteInitMessage(formats_.api_format, rtc::TimeUTCMillis());
aleloi868f32f2017-05-23 07:20:05 -0700411 }
niklase@google.com470e71d2011-07-07 08:21:25 +0000412}
413
Michael Graczyk86c6d332015-07-23 11:41:39 -0700414int AudioProcessingImpl::InitializeLocked(const ProcessingConfig& config) {
Per Åhgren4bdced52017-06-27 16:00:38 +0200415 UpdateActiveSubmoduleStates();
416
Michael Graczyk86c6d332015-07-23 11:41:39 -0700417 for (const auto& stream : config.streams) {
Michael Graczyk86c6d332015-07-23 11:41:39 -0700418 if (stream.num_channels() > 0 && stream.sample_rate_hz() <= 0) {
419 return kBadSampleRateError;
420 }
andrew@webrtc.orga8b97372014-03-10 22:26:12 +0000421 }
Michael Graczyk86c6d332015-07-23 11:41:39 -0700422
Peter Kasting69558702016-01-12 16:26:35 -0800423 const size_t num_in_channels = config.input_stream().num_channels();
424 const size_t num_out_channels = config.output_stream().num_channels();
Michael Graczyk86c6d332015-07-23 11:41:39 -0700425
426 // Need at least one input channel.
427 // Need either one output channel or as many outputs as there are inputs.
428 if (num_in_channels == 0 ||
429 !(num_out_channels == 1 || num_out_channels == num_in_channels)) {
Michael Graczykc2047542015-07-22 21:06:11 -0700430 return kBadNumberChannelsError;
431 }
432
peahdf3efa82015-11-28 12:35:15 -0800433 formats_.api_format = config;
andrew@webrtc.orgddbb8a22014-04-22 21:00:04 +0000434
Per Åhgrenfcbe4072019-09-15 00:27:58 +0200435 // Choose maximum rate to use for the split filtering.
436 RTC_DCHECK(config_.pipeline.maximum_internal_processing_rate == 48000 ||
437 config_.pipeline.maximum_internal_processing_rate == 32000);
438 int max_splitting_rate = 48000;
439 if (config_.pipeline.maximum_internal_processing_rate == 32000) {
440 max_splitting_rate = config_.pipeline.maximum_internal_processing_rate;
441 }
442
Per Åhgrenc8626b62019-08-23 15:49:51 +0200443 int capture_processing_rate = SuitableProcessRate(
peah423d2362016-04-09 16:06:52 -0700444 std::min(formats_.api_format.input_stream().sample_rate_hz(),
peah2ace3f92016-09-10 04:42:27 -0700445 formats_.api_format.output_stream().sample_rate_hz()),
Per Åhgrenfcbe4072019-09-15 00:27:58 +0200446 max_splitting_rate,
peah2ace3f92016-09-10 04:42:27 -0700447 submodule_states_.CaptureMultiBandSubModulesActive() ||
448 submodule_states_.RenderMultiBandSubModulesActive());
Per Åhgrenc8626b62019-08-23 15:49:51 +0200449 RTC_DCHECK_NE(8000, capture_processing_rate);
andrew@webrtc.orga8b97372014-03-10 22:26:12 +0000450
peahde65ddc2016-09-16 15:02:15 -0700451 capture_nonlocked_.capture_processing_format =
452 StreamConfig(capture_processing_rate);
peah2ace3f92016-09-10 04:42:27 -0700453
peah2ce640f2017-04-07 03:57:48 -0700454 int render_processing_rate;
Gustaf Ullbergbd83b912017-10-18 12:32:42 +0200455 if (!capture_nonlocked_.echo_controller_enabled) {
Per Åhgrenc8626b62019-08-23 15:49:51 +0200456 render_processing_rate = SuitableProcessRate(
peah2ce640f2017-04-07 03:57:48 -0700457 std::min(formats_.api_format.reverse_input_stream().sample_rate_hz(),
458 formats_.api_format.reverse_output_stream().sample_rate_hz()),
Per Åhgrenfcbe4072019-09-15 00:27:58 +0200459 max_splitting_rate,
peah2ce640f2017-04-07 03:57:48 -0700460 submodule_states_.CaptureMultiBandSubModulesActive() ||
461 submodule_states_.RenderMultiBandSubModulesActive());
462 } else {
463 render_processing_rate = capture_processing_rate;
464 }
465
peahde65ddc2016-09-16 15:02:15 -0700466 // If the forward sample rate is 8 kHz, the render stream is also processed
aluebseb3603b2016-04-20 15:27:58 -0700467 // at this rate.
peahde65ddc2016-09-16 15:02:15 -0700468 if (capture_nonlocked_.capture_processing_format.sample_rate_hz() ==
469 kSampleRate8kHz) {
470 render_processing_rate = kSampleRate8kHz;
andrew@webrtc.orga8b97372014-03-10 22:26:12 +0000471 } else {
peahde65ddc2016-09-16 15:02:15 -0700472 render_processing_rate =
473 std::max(render_processing_rate, static_cast<int>(kSampleRate16kHz));
andrew@webrtc.orgddbb8a22014-04-22 21:00:04 +0000474 }
475
Per Åhgrenc8626b62019-08-23 15:49:51 +0200476 RTC_DCHECK_NE(8000, render_processing_rate);
477
peahce4d9152017-05-19 01:28:05 -0700478 if (submodule_states_.RenderMultiBandSubModulesActive()) {
Sam Zackrissonfeee1e42019-09-20 07:50:35 +0200479 // By default, downmix the render stream to mono for analysis. This has been
480 // demonstrated to work well for AEC in most practical scenarios.
Per Åhgrene14cb992019-11-27 09:34:22 +0100481 const bool multi_channel_render = config_.pipeline.multi_channel_render &&
482 constants_.multi_channel_render_support;
Sam Zackrissonfeee1e42019-09-20 07:50:35 +0200483 int render_processing_num_channels =
Per Åhgrene14cb992019-11-27 09:34:22 +0100484 multi_channel_render
Sam Zackrissonfeee1e42019-09-20 07:50:35 +0200485 ? formats_.api_format.reverse_input_stream().num_channels()
486 : 1;
487 formats_.render_processing_format =
488 StreamConfig(render_processing_rate, render_processing_num_channels);
peahce4d9152017-05-19 01:28:05 -0700489 } else {
490 formats_.render_processing_format = StreamConfig(
491 formats_.api_format.reverse_input_stream().sample_rate_hz(),
492 formats_.api_format.reverse_input_stream().num_channels());
493 }
andrew@webrtc.orgddbb8a22014-04-22 21:00:04 +0000494
peahde65ddc2016-09-16 15:02:15 -0700495 if (capture_nonlocked_.capture_processing_format.sample_rate_hz() ==
496 kSampleRate32kHz ||
497 capture_nonlocked_.capture_processing_format.sample_rate_hz() ==
498 kSampleRate48kHz) {
peahdf3efa82015-11-28 12:35:15 -0800499 capture_nonlocked_.split_rate = kSampleRate16kHz;
andrew@webrtc.orgddbb8a22014-04-22 21:00:04 +0000500 } else {
peahdf3efa82015-11-28 12:35:15 -0800501 capture_nonlocked_.split_rate =
peahde65ddc2016-09-16 15:02:15 -0700502 capture_nonlocked_.capture_processing_format.sample_rate_hz();
andrew@webrtc.orga8b97372014-03-10 22:26:12 +0000503 }
504
Per Åhgren0ade9832020-09-01 23:57:20 +0200505 InitializeLocked();
506 return kNoError;
andrew@webrtc.orga8b97372014-03-10 22:26:12 +0000507}
508
peah88ac8532016-09-12 16:47:25 -0700509void AudioProcessingImpl::ApplyConfig(const AudioProcessing::Config& config) {
Sam Zackrisson72cc71c2019-10-21 12:54:02 +0200510 RTC_LOG(LS_INFO) << "AudioProcessing::ApplyConfig: " << config.ToString();
511
peah88ac8532016-09-12 16:47:25 -0700512 // Run in a single-threaded manner when applying the settings.
Markus Handell0df0fae2020-07-07 15:53:34 +0200513 MutexLock lock_render(&mutex_render_);
514 MutexLock lock_capture(&mutex_capture_);
peah88ac8532016-09-12 16:47:25 -0700515
Sam Zackrissonfeee1e42019-09-20 07:50:35 +0200516 const bool pipeline_config_changed =
Per Åhgrene14cb992019-11-27 09:34:22 +0100517 config_.pipeline.multi_channel_render !=
518 config.pipeline.multi_channel_render ||
519 config_.pipeline.multi_channel_capture !=
Per Åhgrenc0424252019-12-10 13:04:15 +0100520 config.pipeline.multi_channel_capture ||
521 config_.pipeline.maximum_internal_processing_rate !=
522 config.pipeline.maximum_internal_processing_rate;
Sam Zackrissonfeee1e42019-09-20 07:50:35 +0200523
Per Åhgren200feba2019-03-06 04:16:46 +0100524 const bool aec_config_changed =
525 config_.echo_canceller.enabled != config.echo_canceller.enabled ||
Per Åhgren62ea0aa2019-12-09 10:18:44 +0100526 config_.echo_canceller.mobile_mode != config.echo_canceller.mobile_mode;
Per Åhgren200feba2019-03-06 04:16:46 +0100527
Sam Zackrissonf0d1c032019-03-27 13:28:08 +0100528 const bool agc1_config_changed =
Alessio Bazzica3438a932020-10-14 12:47:50 +0200529 config_.gain_controller1 != config.gain_controller1;
Sam Zackrissonf0d1c032019-03-27 13:28:08 +0100530
Per Åhgren2bd85ab2020-01-03 10:36:34 +0100531 const bool agc2_config_changed =
Alessio Bazzica3438a932020-10-14 12:47:50 +0200532 config_.gain_controller2 != config.gain_controller2;
Per Åhgren2bd85ab2020-01-03 10:36:34 +0100533
saza0bad15f2019-10-16 11:46:11 +0200534 const bool ns_config_changed =
535 config_.noise_suppression.enabled != config.noise_suppression.enabled ||
536 config_.noise_suppression.level != config.noise_suppression.level;
537
Per Åhgrenc0734712020-01-02 15:15:36 +0100538 const bool ts_config_changed = config_.transient_suppression.enabled !=
539 config.transient_suppression.enabled;
540
Per Åhgren0f14db22020-01-03 14:27:14 +0100541 const bool pre_amplifier_config_changed =
542 config_.pre_amplifier.enabled != config.pre_amplifier.enabled ||
543 config_.pre_amplifier.fixed_gain_factor !=
544 config.pre_amplifier.fixed_gain_factor;
545
Per Åhgrendb5d7282021-03-15 16:31:04 +0000546 const bool gain_adjustment_config_changed =
547 config_.capture_level_adjustment != config.capture_level_adjustment;
548
Yves Gerey499bc6c2018-10-10 18:29:07 +0200549 config_ = config;
550
Per Åhgren200feba2019-03-06 04:16:46 +0100551 if (aec_config_changed) {
552 InitializeEchoController();
553 }
Sam Zackrissoncdf0e6d2018-09-17 11:05:17 +0200554
saza0bad15f2019-10-16 11:46:11 +0200555 if (ns_config_changed) {
556 InitializeNoiseSuppressor();
557 }
Sam Zackrisson23513132019-01-11 15:10:32 +0100558
Per Åhgrenc0734712020-01-02 15:15:36 +0100559 if (ts_config_changed) {
560 InitializeTransientSuppressor();
561 }
562
Per Åhgren0f14db22020-01-03 14:27:14 +0100563 InitializeHighPassFilter(false);
peah8271d042016-11-22 07:24:52 -0800564
Sam Zackrissonf0d1c032019-03-27 13:28:08 +0100565 if (agc1_config_changed) {
Per Åhgren0695df12020-01-13 14:43:13 +0100566 InitializeGainController1();
Sam Zackrissonf0d1c032019-03-27 13:28:08 +0100567 }
568
Sam Zackrissonab1aee02018-03-05 15:59:06 +0100569 const bool config_ok = GainController2::Validate(config_.gain_controller2);
alessiob3ec96df2017-05-22 06:57:06 -0700570 if (!config_ok) {
Alessio Bazzica0c83e152020-10-14 12:49:54 +0200571 RTC_LOG(LS_ERROR)
572 << "Invalid Gain Controller 2 config; using the default config.";
alessiob3ec96df2017-05-22 06:57:06 -0700573 config_.gain_controller2 = AudioProcessing::Config::GainController2();
574 }
Per Åhgren0f14db22020-01-03 14:27:14 +0100575
Alessio Bazzica38901042021-10-14 12:14:21 +0200576 InitializeGainController2(agc2_config_changed);
Hanna Silen0c1ad292022-06-16 16:35:45 +0200577 InitializeVoiceActivityDetector(agc2_config_changed);
Per Åhgren0f14db22020-01-03 14:27:14 +0100578
Per Åhgrendb5d7282021-03-15 16:31:04 +0000579 if (pre_amplifier_config_changed || gain_adjustment_config_changed) {
580 InitializeCaptureLevelsAdjuster();
Per Åhgren0f14db22020-01-03 14:27:14 +0100581 }
Sam Zackrissonb24c00f2018-11-26 16:18:25 +0100582
Sam Zackrissonfeee1e42019-09-20 07:50:35 +0200583 // Reinitialization must happen after all submodule configuration to avoid
584 // additional reinitializations on the next capture / render processing call.
585 if (pipeline_config_changed) {
586 InitializeLocked(formats_.api_format);
587 }
peah88ac8532016-09-12 16:47:25 -0700588}
589
Sam Zackrissonb37e59d2020-04-27 08:39:33 +0200590void AudioProcessingImpl::OverrideSubmoduleCreationForTesting(
591 const ApmSubmoduleCreationOverrides& overrides) {
Markus Handell0df0fae2020-07-07 15:53:34 +0200592 MutexLock lock(&mutex_capture_);
Sam Zackrissonb37e59d2020-04-27 08:39:33 +0200593 submodule_creation_overrides_ = overrides;
594}
595
andrew@webrtc.orgddbb8a22014-04-22 21:00:04 +0000596int AudioProcessingImpl::proc_sample_rate_hz() const {
peahdf3efa82015-11-28 12:35:15 -0800597 // Used as callback from submodules, hence locking is not allowed.
peahde65ddc2016-09-16 15:02:15 -0700598 return capture_nonlocked_.capture_processing_format.sample_rate_hz();
niklase@google.com470e71d2011-07-07 08:21:25 +0000599}
600
Gustaf Ullberg422b9e02019-10-09 13:02:14 +0200601int AudioProcessingImpl::proc_fullband_sample_rate_hz() const {
602 return capture_.capture_fullband_audio
603 ? capture_.capture_fullband_audio->num_frames() * 100
604 : capture_nonlocked_.capture_processing_format.sample_rate_hz();
605}
606
andrew@webrtc.orgddbb8a22014-04-22 21:00:04 +0000607int AudioProcessingImpl::proc_split_sample_rate_hz() const {
peahdf3efa82015-11-28 12:35:15 -0800608 // Used as callback from submodules, hence locking is not allowed.
609 return capture_nonlocked_.split_rate;
niklase@google.com470e71d2011-07-07 08:21:25 +0000610}
611
Peter Kasting69558702016-01-12 16:26:35 -0800612size_t AudioProcessingImpl::num_reverse_channels() const {
peahdf3efa82015-11-28 12:35:15 -0800613 // Used as callback from submodules, hence locking is not allowed.
peahde65ddc2016-09-16 15:02:15 -0700614 return formats_.render_processing_format.num_channels();
niklase@google.com470e71d2011-07-07 08:21:25 +0000615}
616
Peter Kasting69558702016-01-12 16:26:35 -0800617size_t AudioProcessingImpl::num_input_channels() const {
peahdf3efa82015-11-28 12:35:15 -0800618 // Used as callback from submodules, hence locking is not allowed.
619 return formats_.api_format.input_stream().num_channels();
niklase@google.com470e71d2011-07-07 08:21:25 +0000620}
621
Peter Kasting69558702016-01-12 16:26:35 -0800622size_t AudioProcessingImpl::num_proc_channels() const {
aluebsb2328d12016-01-11 20:32:29 -0800623 // Used as callback from submodules, hence locking is not allowed.
Per Åhgrene14cb992019-11-27 09:34:22 +0100624 const bool multi_channel_capture = config_.pipeline.multi_channel_capture &&
625 constants_.multi_channel_capture_support;
626 if (capture_nonlocked_.echo_controller_enabled && !multi_channel_capture) {
Sam Zackrissonfeee1e42019-09-20 07:50:35 +0200627 return 1;
628 }
629 return num_output_channels();
aluebsb2328d12016-01-11 20:32:29 -0800630}
631
Peter Kasting69558702016-01-12 16:26:35 -0800632size_t AudioProcessingImpl::num_output_channels() const {
peahdf3efa82015-11-28 12:35:15 -0800633 // Used as callback from submodules, hence locking is not allowed.
634 return formats_.api_format.output_stream().num_channels();
niklase@google.com470e71d2011-07-07 08:21:25 +0000635}
636
andrew@webrtc.org17342e52014-02-12 22:28:31 +0000637void AudioProcessingImpl::set_output_will_be_muted(bool muted) {
Markus Handell0df0fae2020-07-07 15:53:34 +0200638 MutexLock lock(&mutex_capture_);
Per Åhgren0a144a72021-02-09 08:47:51 +0100639 HandleCaptureOutputUsedSetting(!muted);
640}
641
642void AudioProcessingImpl::HandleCaptureOutputUsedSetting(
643 bool capture_output_used) {
Per Åhgren19775cb2021-03-12 23:08:09 +0000644 capture_.capture_output_used =
645 capture_output_used || !constants_.minimize_processing_for_unused_output;
646
saza1d600522019-10-18 13:29:43 +0200647 if (submodules_.agc_manager.get()) {
Per Åhgren0a144a72021-02-09 08:47:51 +0100648 submodules_.agc_manager->HandleCaptureOutputUsedChange(
649 capture_.capture_output_used);
pbos@webrtc.org788acd12014-12-15 09:41:24 +0000650 }
Per Åhgren652ada52021-03-03 10:52:44 +0000651 if (submodules_.echo_controller) {
652 submodules_.echo_controller->SetCaptureOutputUsage(
653 capture_.capture_output_used);
654 }
Per Åhgren15179a92021-03-12 14:12:44 +0000655 if (submodules_.noise_suppressor) {
656 submodules_.noise_suppressor->SetCaptureOutputUsage(
657 capture_.capture_output_used);
658 }
andrew@webrtc.org17342e52014-02-12 22:28:31 +0000659}
660
Alessio Bazzicac054e782018-04-16 12:10:09 +0200661void AudioProcessingImpl::SetRuntimeSetting(RuntimeSetting setting) {
Per Åhgren0a144a72021-02-09 08:47:51 +0100662 PostRuntimeSetting(setting);
663}
664
665bool AudioProcessingImpl::PostRuntimeSetting(RuntimeSetting setting) {
Alex Loiko73ec0192018-05-15 10:52:28 +0200666 switch (setting.type()) {
667 case RuntimeSetting::Type::kCustomRenderProcessingRuntimeSetting:
Alessio Bazzica7c19a702019-11-07 13:22:00 +0100668 case RuntimeSetting::Type::kPlayoutAudioDeviceChange:
Per Åhgren0a144a72021-02-09 08:47:51 +0100669 return render_runtime_settings_enqueuer_.Enqueue(setting);
Alex Loiko73ec0192018-05-15 10:52:28 +0200670 case RuntimeSetting::Type::kCapturePreGain:
Per Åhgrendb5d7282021-03-15 16:31:04 +0000671 case RuntimeSetting::Type::kCapturePostGain:
Sam Zackrissonf0d1c032019-03-27 13:28:08 +0100672 case RuntimeSetting::Type::kCaptureCompressionGain:
Per Åhgren6ee75fd2019-04-26 11:33:37 +0200673 case RuntimeSetting::Type::kCaptureFixedPostGain:
Per Åhgren552d3e32020-08-12 08:46:47 +0200674 case RuntimeSetting::Type::kCaptureOutputUsed:
Per Åhgren0a144a72021-02-09 08:47:51 +0100675 return capture_runtime_settings_enqueuer_.Enqueue(setting);
676 case RuntimeSetting::Type::kPlayoutVolumeChange: {
677 bool enqueueing_successful;
678 enqueueing_successful =
679 capture_runtime_settings_enqueuer_.Enqueue(setting);
680 enqueueing_successful =
681 render_runtime_settings_enqueuer_.Enqueue(setting) &&
682 enqueueing_successful;
683 return enqueueing_successful;
684 }
Alessio Bazzica7c19a702019-11-07 13:22:00 +0100685 case RuntimeSetting::Type::kNotSpecified:
Artem Titovd3251962021-11-15 16:57:07 +0100686 RTC_DCHECK_NOTREACHED();
Per Åhgren0a144a72021-02-09 08:47:51 +0100687 return true;
Alex Loiko73ec0192018-05-15 10:52:28 +0200688 }
689 // The language allows the enum to have a non-enumerator
690 // value. Check that this doesn't happen.
Artem Titovd3251962021-11-15 16:57:07 +0100691 RTC_DCHECK_NOTREACHED();
Per Åhgren0a144a72021-02-09 08:47:51 +0100692 return true;
Alessio Bazzicac054e782018-04-16 12:10:09 +0200693}
694
695AudioProcessingImpl::RuntimeSettingEnqueuer::RuntimeSettingEnqueuer(
696 SwapQueue<RuntimeSetting>* runtime_settings)
Alessio Bazzica33444dc2018-04-20 13:16:55 +0200697 : runtime_settings_(*runtime_settings) {
698 RTC_DCHECK(runtime_settings);
Alessio Bazzicac054e782018-04-16 12:10:09 +0200699}
700
701AudioProcessingImpl::RuntimeSettingEnqueuer::~RuntimeSettingEnqueuer() =
702 default;
703
Per Åhgren0a144a72021-02-09 08:47:51 +0100704bool AudioProcessingImpl::RuntimeSettingEnqueuer::Enqueue(
Alessio Bazzicac054e782018-04-16 12:10:09 +0200705 RuntimeSetting setting) {
Per Åhgren652ada52021-03-03 10:52:44 +0000706 const bool successful_insert = runtime_settings_.Insert(&setting);
707
708 if (!successful_insert) {
Alessio Bazzica0b10f462020-10-23 12:40:30 +0200709 RTC_HISTOGRAM_BOOLEAN("WebRTC.Audio.ApmRuntimeSettingCannotEnqueue", 1);
Alessio Bazzicac054e782018-04-16 12:10:09 +0200710 RTC_LOG(LS_ERROR) << "Cannot enqueue a new runtime setting.";
Alessio Bazzica0b10f462020-10-23 12:40:30 +0200711 }
Per Åhgren652ada52021-03-03 10:52:44 +0000712 return successful_insert;
Alessio Bazzicac054e782018-04-16 12:10:09 +0200713}
andrew@webrtc.org17342e52014-02-12 22:28:31 +0000714
Sam Zackrisson12e319a2020-01-03 14:54:20 +0100715int AudioProcessingImpl::MaybeInitializeCapture(
716 const StreamConfig& input_config,
717 const StreamConfig& output_config) {
peahdf3efa82015-11-28 12:35:15 -0800718 ProcessingConfig processing_config;
peah2ace3f92016-09-10 04:42:27 -0700719 bool reinitialization_required = false;
peahdf3efa82015-11-28 12:35:15 -0800720 {
Sam Zackrisson12e319a2020-01-03 14:54:20 +0100721 // Acquire the capture lock in order to access api_format. The lock is
722 // released immediately, as we may need to acquire the render lock as part
723 // of the conditional reinitialization.
Markus Handell0df0fae2020-07-07 15:53:34 +0200724 MutexLock lock_capture(&mutex_capture_);
peahdf3efa82015-11-28 12:35:15 -0800725 processing_config = formats_.api_format;
peah2ace3f92016-09-10 04:42:27 -0700726 reinitialization_required = UpdateActiveSubmoduleStates();
niklase@google.com470e71d2011-07-07 08:21:25 +0000727 }
andrew@webrtc.org17e40642014-03-04 20:58:13 +0000728
Oskar Sundbom4b276482019-05-23 14:28:00 +0200729 if (processing_config.input_stream() != input_config) {
730 processing_config.input_stream() = input_config;
731 reinitialization_required = true;
peahdf3efa82015-11-28 12:35:15 -0800732 }
Oskar Sundbom4b276482019-05-23 14:28:00 +0200733
734 if (processing_config.output_stream() != output_config) {
735 processing_config.output_stream() = output_config;
736 reinitialization_required = true;
737 }
738
739 if (reinitialization_required) {
Markus Handell0df0fae2020-07-07 15:53:34 +0200740 MutexLock lock_render(&mutex_render_);
741 MutexLock lock_capture(&mutex_capture_);
Oskar Sundbom4b276482019-05-23 14:28:00 +0200742 RETURN_ON_ERR(InitializeLocked(processing_config));
743 }
Sam Zackrisson12e319a2020-01-03 14:54:20 +0100744 return kNoError;
745}
746
747int AudioProcessingImpl::ProcessStream(const float* const* src,
748 const StreamConfig& input_config,
749 const StreamConfig& output_config,
750 float* const* dest) {
751 TRACE_EVENT0("webrtc", "AudioProcessing::ProcessStream_StreamConfig");
752 if (!src || !dest) {
753 return kNullPointerError;
754 }
755
756 RETURN_ON_ERR(MaybeInitializeCapture(input_config, output_config));
Oskar Sundbom4b276482019-05-23 14:28:00 +0200757
Markus Handell0df0fae2020-07-07 15:53:34 +0200758 MutexLock lock_capture(&mutex_capture_);
Alessio Bazzica0441bb62021-08-10 15:23:23 +0200759 DenormalDisabler denormal_disabler(use_denormal_disabler_);
andrew@webrtc.org17e40642014-03-04 20:58:13 +0000760
aleloi868f32f2017-05-23 07:20:05 -0700761 if (aec_dump_) {
762 RecordUnprocessedCaptureStream(src);
763 }
764
peahdf3efa82015-11-28 12:35:15 -0800765 capture_.capture_audio->CopyFrom(src, formats_.api_format.input_stream());
Gustaf Ullberg8675eee2019-10-09 13:34:36 +0200766 if (capture_.capture_fullband_audio) {
767 capture_.capture_fullband_audio->CopyFrom(
768 src, formats_.api_format.input_stream());
769 }
peahde65ddc2016-09-16 15:02:15 -0700770 RETURN_ON_ERR(ProcessCaptureStreamLocked());
Gustaf Ullberg422b9e02019-10-09 13:02:14 +0200771 if (capture_.capture_fullband_audio) {
772 capture_.capture_fullband_audio->CopyTo(formats_.api_format.output_stream(),
773 dest);
774 } else {
775 capture_.capture_audio->CopyTo(formats_.api_format.output_stream(), dest);
776 }
andrew@webrtc.org17e40642014-03-04 20:58:13 +0000777
aleloi868f32f2017-05-23 07:20:05 -0700778 if (aec_dump_) {
779 RecordProcessedCaptureStream(dest);
780 }
andrew@webrtc.org17e40642014-03-04 20:58:13 +0000781 return kNoError;
782}
783
Alex Loiko73ec0192018-05-15 10:52:28 +0200784void AudioProcessingImpl::HandleCaptureRuntimeSettings() {
Alessio Bazzicac054e782018-04-16 12:10:09 +0200785 RuntimeSetting setting;
Per Åhgren652ada52021-03-03 10:52:44 +0000786 int num_settings_processed = 0;
Alex Loiko73ec0192018-05-15 10:52:28 +0200787 while (capture_runtime_settings_.Remove(&setting)) {
Alex Loiko62347222018-09-10 10:18:07 +0200788 if (aec_dump_) {
789 aec_dump_->WriteRuntimeSetting(setting);
790 }
Alessio Bazzicac054e782018-04-16 12:10:09 +0200791 switch (setting.type()) {
792 case RuntimeSetting::Type::kCapturePreGain:
Per Åhgrendb5d7282021-03-15 16:31:04 +0000793 if (config_.pre_amplifier.enabled ||
794 config_.capture_level_adjustment.enabled) {
Alex Loikob5c9a792018-04-16 16:31:22 +0200795 float value;
796 setting.GetFloat(&value);
Per Åhgrendb5d7282021-03-15 16:31:04 +0000797 // If the pre-amplifier is used, apply the new gain to the
798 // pre-amplifier regardless if the capture level adjustment is
799 // activated. This approach allows both functionalities to coexist
800 // until they have been properly merged.
801 if (config_.pre_amplifier.enabled) {
802 config_.pre_amplifier.fixed_gain_factor = value;
803 } else {
804 config_.capture_level_adjustment.pre_gain_factor = value;
805 }
806
807 // Use both the pre-amplifier and the capture level adjustment gains
808 // as pre-gains.
809 float gain = 1.f;
810 if (config_.pre_amplifier.enabled) {
811 gain *= config_.pre_amplifier.fixed_gain_factor;
812 }
813 if (config_.capture_level_adjustment.enabled) {
814 gain *= config_.capture_level_adjustment.pre_gain_factor;
815 }
816
817 submodules_.capture_levels_adjuster->SetPreGain(gain);
818 }
819 // TODO(bugs.chromium.org/9138): Log setting handling by Aec Dump.
820 break;
821 case RuntimeSetting::Type::kCapturePostGain:
822 if (config_.capture_level_adjustment.enabled) {
823 float value;
824 setting.GetFloat(&value);
825 config_.capture_level_adjustment.post_gain_factor = value;
826 submodules_.capture_levels_adjuster->SetPostGain(
827 config_.capture_level_adjustment.post_gain_factor);
Alex Loikob5c9a792018-04-16 16:31:22 +0200828 }
829 // TODO(bugs.chromium.org/9138): Log setting handling by Aec Dump.
Alessio Bazzicac054e782018-04-16 12:10:09 +0200830 break;
Sam Zackrissonf0d1c032019-03-27 13:28:08 +0100831 case RuntimeSetting::Type::kCaptureCompressionGain: {
Per Åhgren0e3198e2019-11-18 08:52:22 +0100832 if (!submodules_.agc_manager) {
833 float value;
834 setting.GetFloat(&value);
835 int int_value = static_cast<int>(value + .5f);
836 config_.gain_controller1.compression_gain_db = int_value;
Per Åhgren0695df12020-01-13 14:43:13 +0100837 if (submodules_.gain_control) {
838 int error =
839 submodules_.gain_control->set_compression_gain_db(int_value);
840 RTC_DCHECK_EQ(kNoError, error);
841 }
Per Åhgren0e3198e2019-11-18 08:52:22 +0100842 }
Sam Zackrissonf0d1c032019-03-27 13:28:08 +0100843 break;
844 }
Per Åhgren6ee75fd2019-04-26 11:33:37 +0200845 case RuntimeSetting::Type::kCaptureFixedPostGain: {
Per Åhgren2bd85ab2020-01-03 10:36:34 +0100846 if (submodules_.gain_controller2) {
Per Åhgren6ee75fd2019-04-26 11:33:37 +0200847 float value;
848 setting.GetFloat(&value);
849 config_.gain_controller2.fixed_digital.gain_db = value;
Alessio Bazzica38901042021-10-14 12:14:21 +0200850 submodules_.gain_controller2->SetFixedGainDb(value);
Per Åhgren6ee75fd2019-04-26 11:33:37 +0200851 }
852 break;
853 }
Fredrik Hernqvistca362852019-05-10 15:50:02 +0200854 case RuntimeSetting::Type::kPlayoutVolumeChange: {
855 int value;
856 setting.GetInt(&value);
857 capture_.playout_volume = value;
858 break;
859 }
Alessio Bazzica7c19a702019-11-07 13:22:00 +0100860 case RuntimeSetting::Type::kPlayoutAudioDeviceChange:
Artem Titovd3251962021-11-15 16:57:07 +0100861 RTC_DCHECK_NOTREACHED();
Alessio Bazzica7c19a702019-11-07 13:22:00 +0100862 break;
Alex Loiko73ec0192018-05-15 10:52:28 +0200863 case RuntimeSetting::Type::kCustomRenderProcessingRuntimeSetting:
Artem Titovd3251962021-11-15 16:57:07 +0100864 RTC_DCHECK_NOTREACHED();
Alex Loiko73ec0192018-05-15 10:52:28 +0200865 break;
866 case RuntimeSetting::Type::kNotSpecified:
Artem Titovd3251962021-11-15 16:57:07 +0100867 RTC_DCHECK_NOTREACHED();
Alex Loiko73ec0192018-05-15 10:52:28 +0200868 break;
Per Åhgren552d3e32020-08-12 08:46:47 +0200869 case RuntimeSetting::Type::kCaptureOutputUsed:
Per Åhgren0a144a72021-02-09 08:47:51 +0100870 bool value;
871 setting.GetBool(&value);
872 HandleCaptureOutputUsedSetting(value);
Per Åhgren552d3e32020-08-12 08:46:47 +0200873 break;
Alex Loiko73ec0192018-05-15 10:52:28 +0200874 }
Per Åhgren652ada52021-03-03 10:52:44 +0000875 ++num_settings_processed;
876 }
877
878 if (num_settings_processed >= RuntimeSettingQueueSize()) {
879 // Handle overrun of the runtime settings queue, which likely will has
880 // caused settings to be discarded.
881 HandleOverrunInCaptureRuntimeSettingsQueue();
882 }
883}
884
885void AudioProcessingImpl::HandleOverrunInCaptureRuntimeSettingsQueue() {
886 // Fall back to a safe state for the case when a setting for capture output
887 // usage setting has been missed.
Per Åhgren19775cb2021-03-12 23:08:09 +0000888 HandleCaptureOutputUsedSetting(/*capture_output_used=*/true);
Alex Loiko73ec0192018-05-15 10:52:28 +0200889}
890
891void AudioProcessingImpl::HandleRenderRuntimeSettings() {
892 RuntimeSetting setting;
893 while (render_runtime_settings_.Remove(&setting)) {
Alex Loiko62347222018-09-10 10:18:07 +0200894 if (aec_dump_) {
895 aec_dump_->WriteRuntimeSetting(setting);
896 }
Alex Loiko73ec0192018-05-15 10:52:28 +0200897 switch (setting.type()) {
Alessio Bazzica7c19a702019-11-07 13:22:00 +0100898 case RuntimeSetting::Type::kPlayoutAudioDeviceChange: // fall-through
Alessio Bazzica7587de42019-11-11 13:32:20 +0100899 case RuntimeSetting::Type::kPlayoutVolumeChange: // fall-through
Alex Loiko73ec0192018-05-15 10:52:28 +0200900 case RuntimeSetting::Type::kCustomRenderProcessingRuntimeSetting:
saza1d600522019-10-18 13:29:43 +0200901 if (submodules_.render_pre_processor) {
902 submodules_.render_pre_processor->SetRuntimeSetting(setting);
Alex Loiko73ec0192018-05-15 10:52:28 +0200903 }
904 break;
Sam Zackrissonf0d1c032019-03-27 13:28:08 +0100905 case RuntimeSetting::Type::kCapturePreGain: // fall-through
Per Åhgrendb5d7282021-03-15 16:31:04 +0000906 case RuntimeSetting::Type::kCapturePostGain: // fall-through
Sam Zackrissonf0d1c032019-03-27 13:28:08 +0100907 case RuntimeSetting::Type::kCaptureCompressionGain: // fall-through
Per Åhgren6ee75fd2019-04-26 11:33:37 +0200908 case RuntimeSetting::Type::kCaptureFixedPostGain: // fall-through
Per Åhgren552d3e32020-08-12 08:46:47 +0200909 case RuntimeSetting::Type::kCaptureOutputUsed: // fall-through
Alessio Bazzica33444dc2018-04-20 13:16:55 +0200910 case RuntimeSetting::Type::kNotSpecified:
Artem Titovd3251962021-11-15 16:57:07 +0100911 RTC_DCHECK_NOTREACHED();
Alessio Bazzicac054e782018-04-16 12:10:09 +0200912 break;
913 }
914 }
915}
916
peah9e6a2902017-05-15 07:19:21 -0700917void AudioProcessingImpl::QueueBandedRenderAudio(AudioBuffer* audio) {
kwibergaf476c72016-11-28 15:21:39 -0800918 RTC_DCHECK_GE(160, audio->num_frames_per_band());
peah764e3642016-10-22 05:04:30 -0700919
saza1d600522019-10-18 13:29:43 +0200920 if (submodules_.echo_control_mobile) {
Per Åhgrenb6e24d72019-04-29 12:14:50 +0200921 EchoControlMobileImpl::PackRenderAudioBuffer(audio, num_output_channels(),
922 num_reverse_channels(),
923 &aecm_render_queue_buffer_);
924 RTC_DCHECK(aecm_render_signal_queue_);
925 // Insert the samples into the queue.
926 if (!aecm_render_signal_queue_->Insert(&aecm_render_queue_buffer_)) {
927 // The data queue is full and needs to be emptied.
928 EmptyQueuedRenderAudio();
peaha0624602016-10-25 04:45:24 -0700929
Per Åhgrenb6e24d72019-04-29 12:14:50 +0200930 // Retry the insert (should always work).
931 bool result =
932 aecm_render_signal_queue_->Insert(&aecm_render_queue_buffer_);
933 RTC_DCHECK(result);
934 }
peah764e3642016-10-22 05:04:30 -0700935 }
peah701d6282016-10-25 05:42:20 -0700936
Per Åhgren0695df12020-01-13 14:43:13 +0100937 if (!submodules_.agc_manager && submodules_.gain_control) {
Per Åhgrene35b32c2019-11-22 18:22:04 +0100938 GainControlImpl::PackRenderAudioBuffer(*audio, &agc_render_queue_buffer_);
peah701d6282016-10-25 05:42:20 -0700939 // Insert the samples into the queue.
940 if (!agc_render_signal_queue_->Insert(&agc_render_queue_buffer_)) {
941 // The data queue is full and needs to be emptied.
942 EmptyQueuedRenderAudio();
943
944 // Retry the insert (should always work).
945 bool result = agc_render_signal_queue_->Insert(&agc_render_queue_buffer_);
946 RTC_DCHECK(result);
947 }
948 }
peah9e6a2902017-05-15 07:19:21 -0700949}
ivoc9f4a4a02016-10-28 05:39:16 -0700950
peah9e6a2902017-05-15 07:19:21 -0700951void AudioProcessingImpl::QueueNonbandedRenderAudio(AudioBuffer* audio) {
Sam Zackrisson03cb7e52021-12-06 15:40:04 +0100952 if (submodules_.echo_detector) {
953 PackRenderAudioBufferForEchoDetector(*audio, red_render_queue_buffer_);
954 RTC_DCHECK(red_render_signal_queue_);
955 // Insert the samples into the queue.
956 if (!red_render_signal_queue_->Insert(&red_render_queue_buffer_)) {
957 // The data queue is full and needs to be emptied.
958 EmptyQueuedRenderAudio();
ivoc9f4a4a02016-10-28 05:39:16 -0700959
Sam Zackrisson03cb7e52021-12-06 15:40:04 +0100960 // Retry the insert (should always work).
961 bool result = red_render_signal_queue_->Insert(&red_render_queue_buffer_);
962 RTC_DCHECK(result);
963 }
ivoc9f4a4a02016-10-28 05:39:16 -0700964 }
peah764e3642016-10-22 05:04:30 -0700965}
966
967void AudioProcessingImpl::AllocateRenderQueue() {
peah701d6282016-10-25 05:42:20 -0700968 const size_t new_agc_render_queue_element_max_size =
peah9e6a2902017-05-15 07:19:21 -0700969 std::max(static_cast<size_t>(1), kMaxAllowedValuesOfSamplesPerBand);
peah701d6282016-10-25 05:42:20 -0700970
ivoc9f4a4a02016-10-28 05:39:16 -0700971 const size_t new_red_render_queue_element_max_size =
972 std::max(static_cast<size_t>(1), kMaxAllowedValuesOfSamplesPerFrame);
973
peaha0624602016-10-25 04:45:24 -0700974 // Reallocate the queues if the queue item sizes are too small to fit the
975 // data to put in the queues.
peah701d6282016-10-25 05:42:20 -0700976
977 if (agc_render_queue_element_max_size_ <
978 new_agc_render_queue_element_max_size) {
979 agc_render_queue_element_max_size_ = new_agc_render_queue_element_max_size;
980
981 std::vector<int16_t> template_queue_element(
982 agc_render_queue_element_max_size_);
983
984 agc_render_signal_queue_.reset(
985 new SwapQueue<std::vector<int16_t>, RenderQueueItemVerifier<int16_t>>(
986 kMaxNumFramesToBuffer, template_queue_element,
987 RenderQueueItemVerifier<int16_t>(
988 agc_render_queue_element_max_size_)));
989
990 agc_render_queue_buffer_.resize(agc_render_queue_element_max_size_);
991 agc_capture_queue_buffer_.resize(agc_render_queue_element_max_size_);
992 } else {
993 agc_render_signal_queue_->Clear();
peah764e3642016-10-22 05:04:30 -0700994 }
ivoc9f4a4a02016-10-28 05:39:16 -0700995
Sam Zackrisson03cb7e52021-12-06 15:40:04 +0100996 if (submodules_.echo_detector) {
997 if (red_render_queue_element_max_size_ <
998 new_red_render_queue_element_max_size) {
999 red_render_queue_element_max_size_ =
1000 new_red_render_queue_element_max_size;
ivoc9f4a4a02016-10-28 05:39:16 -07001001
Sam Zackrisson03cb7e52021-12-06 15:40:04 +01001002 std::vector<float> template_queue_element(
1003 red_render_queue_element_max_size_);
ivoc9f4a4a02016-10-28 05:39:16 -07001004
Sam Zackrisson03cb7e52021-12-06 15:40:04 +01001005 red_render_signal_queue_.reset(
1006 new SwapQueue<std::vector<float>, RenderQueueItemVerifier<float>>(
1007 kMaxNumFramesToBuffer, template_queue_element,
1008 RenderQueueItemVerifier<float>(
1009 red_render_queue_element_max_size_)));
ivoc9f4a4a02016-10-28 05:39:16 -07001010
Sam Zackrisson03cb7e52021-12-06 15:40:04 +01001011 red_render_queue_buffer_.resize(red_render_queue_element_max_size_);
1012 red_capture_queue_buffer_.resize(red_render_queue_element_max_size_);
1013 } else {
1014 red_render_signal_queue_->Clear();
1015 }
ivoc9f4a4a02016-10-28 05:39:16 -07001016 }
peah764e3642016-10-22 05:04:30 -07001017}
1018
1019void AudioProcessingImpl::EmptyQueuedRenderAudio() {
Markus Handell0df0fae2020-07-07 15:53:34 +02001020 MutexLock lock_capture(&mutex_capture_);
Markus Handell02ba1d22020-05-14 14:31:18 +02001021 EmptyQueuedRenderAudioLocked();
1022}
1023
1024void AudioProcessingImpl::EmptyQueuedRenderAudioLocked() {
saza1d600522019-10-18 13:29:43 +02001025 if (submodules_.echo_control_mobile) {
Per Åhgrenb6e24d72019-04-29 12:14:50 +02001026 RTC_DCHECK(aecm_render_signal_queue_);
1027 while (aecm_render_signal_queue_->Remove(&aecm_capture_queue_buffer_)) {
saza1d600522019-10-18 13:29:43 +02001028 submodules_.echo_control_mobile->ProcessRenderAudio(
Per Åhgrenb6e24d72019-04-29 12:14:50 +02001029 aecm_capture_queue_buffer_);
1030 }
peah701d6282016-10-25 05:42:20 -07001031 }
1032
Per Åhgren0695df12020-01-13 14:43:13 +01001033 if (submodules_.gain_control) {
1034 while (agc_render_signal_queue_->Remove(&agc_capture_queue_buffer_)) {
1035 submodules_.gain_control->ProcessRenderAudio(agc_capture_queue_buffer_);
1036 }
peah764e3642016-10-22 05:04:30 -07001037 }
ivoc9f4a4a02016-10-28 05:39:16 -07001038
Sam Zackrisson03cb7e52021-12-06 15:40:04 +01001039 if (submodules_.echo_detector) {
1040 while (red_render_signal_queue_->Remove(&red_capture_queue_buffer_)) {
1041 submodules_.echo_detector->AnalyzeRenderAudio(red_capture_queue_buffer_);
1042 }
ivoc9f4a4a02016-10-28 05:39:16 -07001043 }
peah764e3642016-10-22 05:04:30 -07001044}
1045
Per Åhgren645f24c2020-03-16 12:06:02 +01001046int AudioProcessingImpl::ProcessStream(const int16_t* const src,
1047 const StreamConfig& input_config,
1048 const StreamConfig& output_config,
Per Åhgrendc5522b2020-03-19 14:55:58 +01001049 int16_t* const dest) {
Per Åhgren71652f42020-03-17 13:23:58 +01001050 TRACE_EVENT0("webrtc", "AudioProcessing::ProcessStream_AudioFrame");
Sam Zackrisson12e319a2020-01-03 14:54:20 +01001051 RETURN_ON_ERR(MaybeInitializeCapture(input_config, output_config));
Oskar Sundbom4b276482019-05-23 14:28:00 +02001052
Markus Handell0df0fae2020-07-07 15:53:34 +02001053 MutexLock lock_capture(&mutex_capture_);
Alessio Bazzica0441bb62021-08-10 15:23:23 +02001054 DenormalDisabler denormal_disabler(use_denormal_disabler_);
niklase@google.com470e71d2011-07-07 08:21:25 +00001055
aleloi868f32f2017-05-23 07:20:05 -07001056 if (aec_dump_) {
Per Åhgren645f24c2020-03-16 12:06:02 +01001057 RecordUnprocessedCaptureStream(src, input_config);
aleloi868f32f2017-05-23 07:20:05 -07001058 }
1059
Per Åhgren645f24c2020-03-16 12:06:02 +01001060 capture_.capture_audio->CopyFrom(src, input_config);
Gustaf Ullberg3c918b12019-10-11 13:14:44 +02001061 if (capture_.capture_fullband_audio) {
Per Åhgren645f24c2020-03-16 12:06:02 +01001062 capture_.capture_fullband_audio->CopyFrom(src, input_config);
Gustaf Ullberg3c918b12019-10-11 13:14:44 +02001063 }
peahde65ddc2016-09-16 15:02:15 -07001064 RETURN_ON_ERR(ProcessCaptureStreamLocked());
Gustaf Ullberg8675eee2019-10-09 13:34:36 +02001065 if (submodule_states_.CaptureMultiBandProcessingPresent() ||
Per Åhgrena1351272019-08-15 12:15:46 +02001066 submodule_states_.CaptureFullBandProcessingActive()) {
Gustaf Ullberg422b9e02019-10-09 13:02:14 +02001067 if (capture_.capture_fullband_audio) {
Per Åhgren645f24c2020-03-16 12:06:02 +01001068 capture_.capture_fullband_audio->CopyTo(output_config, dest);
Gustaf Ullberg422b9e02019-10-09 13:02:14 +02001069 } else {
Per Åhgren645f24c2020-03-16 12:06:02 +01001070 capture_.capture_audio->CopyTo(output_config, dest);
Gustaf Ullberg422b9e02019-10-09 13:02:14 +02001071 }
Per Åhgrena1351272019-08-15 12:15:46 +02001072 }
Per Åhgren645f24c2020-03-16 12:06:02 +01001073
aleloi868f32f2017-05-23 07:20:05 -07001074 if (aec_dump_) {
Per Åhgren645f24c2020-03-16 12:06:02 +01001075 RecordProcessedCaptureStream(dest, output_config);
aleloi868f32f2017-05-23 07:20:05 -07001076 }
andrew@webrtc.org17e40642014-03-04 20:58:13 +00001077
1078 return kNoError;
1079}
1080
peahde65ddc2016-09-16 15:02:15 -07001081int AudioProcessingImpl::ProcessCaptureStreamLocked() {
Markus Handell02ba1d22020-05-14 14:31:18 +02001082 EmptyQueuedRenderAudioLocked();
Alex Loiko73ec0192018-05-15 10:52:28 +02001083 HandleCaptureRuntimeSettings();
Alessio Bazzica0441bb62021-08-10 15:23:23 +02001084 DenormalDisabler denormal_disabler(use_denormal_disabler_);
Alessio Bazzicac054e782018-04-16 12:10:09 +02001085
peahb58a1582016-03-15 09:34:24 -07001086 // Ensure that not both the AEC and AECM are active at the same time.
Sam Zackrisson2a959d92018-07-23 14:48:07 +00001087 // TODO(peah): Simplify once the public API Enable functions for these
1088 // are moved to APM.
Sam Zackrisson308bc642019-12-23 10:22:08 +01001089 RTC_DCHECK_LE(
1090 !!submodules_.echo_controller + !!submodules_.echo_control_mobile, 1);
peahb58a1582016-03-15 09:34:24 -07001091
peahde65ddc2016-09-16 15:02:15 -07001092 AudioBuffer* capture_buffer = capture_.capture_audio.get(); // For brevity.
Per Åhgren2e8e1c62019-12-20 00:42:22 +01001093 AudioBuffer* linear_aec_buffer = capture_.linear_aec_output.get();
ekmeyerson60d9b332015-08-14 10:35:55 -07001094
Per Åhgrenc0424252019-12-10 13:04:15 +01001095 if (submodules_.high_pass_filter &&
1096 config_.high_pass_filter.apply_in_full_band &&
1097 !constants_.enforce_split_band_hpf) {
1098 submodules_.high_pass_filter->Process(capture_buffer,
1099 /*use_split_band_data=*/false);
1100 }
1101
Per Åhgrendb5d7282021-03-15 16:31:04 +00001102 if (submodules_.capture_levels_adjuster) {
1103 // If the analog mic gain emulation is active, get the emulated analog mic
1104 // gain and pass it to the analog gain control functionality.
1105 if (config_.capture_level_adjustment.analog_mic_gain_emulation.enabled) {
1106 int level = submodules_.capture_levels_adjuster->GetAnalogMicGainLevel();
1107 if (submodules_.agc_manager) {
1108 submodules_.agc_manager->set_stream_analog_level(level);
1109 } else if (submodules_.gain_control) {
1110 int error = submodules_.gain_control->set_stream_analog_level(level);
1111 RTC_DCHECK_EQ(kNoError, error);
1112 }
1113 }
1114
1115 submodules_.capture_levels_adjuster->ApplyPreLevelAdjustment(
1116 *capture_buffer);
Alex Loikob5c9a792018-04-16 16:31:22 +02001117 }
1118
Per Åhgren928146f2019-08-20 09:19:21 +02001119 capture_input_rms_.Analyze(rtc::ArrayView<const float>(
Per Åhgrend47941e2019-08-22 11:51:13 +02001120 capture_buffer->channels_const()[0],
henrik.lundin290d43a2016-11-29 08:09:09 -08001121 capture_nonlocked_.capture_processing_format.num_frames()));
peah1b08dc32016-12-20 13:45:58 -08001122 const bool log_rms = ++capture_rms_interval_counter_ >= 1000;
1123 if (log_rms) {
1124 capture_rms_interval_counter_ = 0;
1125 RmsLevel::Levels levels = capture_input_rms_.AverageAndPeak();
henrik.lundin45bb5132016-12-06 04:28:04 -08001126 RTC_HISTOGRAM_COUNTS_LINEAR("WebRTC.Audio.ApmCaptureInputLevelAverageRms",
1127 levels.average, 1, RmsLevel::kMinLevelDb, 64);
1128 RTC_HISTOGRAM_COUNTS_LINEAR("WebRTC.Audio.ApmCaptureInputLevelPeakRms",
1129 levels.peak, 1, RmsLevel::kMinLevelDb, 64);
henrik.lundin290d43a2016-11-29 08:09:09 -08001130 }
1131
Hanna Silen5c7d5c92021-10-06 17:32:17 +02001132 // Detect an analog gain change.
1133 int analog_mic_level = recommended_stream_analog_level_locked();
1134 const bool analog_mic_level_changed =
1135 capture_.prev_analog_mic_level != analog_mic_level &&
1136 capture_.prev_analog_mic_level != -1;
1137 capture_.prev_analog_mic_level = analog_mic_level;
Hanna Silen529131d2021-10-29 14:55:45 +02001138 analog_gain_stats_reporter_.UpdateStatistics(analog_mic_level);
Hanna Silen5c7d5c92021-10-06 17:32:17 +02001139
saza1d600522019-10-18 13:29:43 +02001140 if (submodules_.echo_controller) {
Hanna Silen5c7d5c92021-10-06 17:32:17 +02001141 capture_.echo_path_gain_change = analog_mic_level_changed;
Per Åhgren88cf0502018-07-16 17:08:41 +02001142
Per Åhgrendb5d7282021-03-15 16:31:04 +00001143 // Detect and flag any change in the capture level adjustment pre-gain.
1144 if (submodules_.capture_levels_adjuster) {
1145 float pre_adjustment_gain =
1146 submodules_.capture_levels_adjuster->GetPreAdjustmentGain();
Per Åhgrend2650d12018-10-02 17:00:59 +02001147 capture_.echo_path_gain_change =
1148 capture_.echo_path_gain_change ||
Per Åhgrendb5d7282021-03-15 16:31:04 +00001149 (capture_.prev_pre_adjustment_gain != pre_adjustment_gain &&
1150 capture_.prev_pre_adjustment_gain >= 0.f);
1151 capture_.prev_pre_adjustment_gain = pre_adjustment_gain;
Per Åhgrend2650d12018-10-02 17:00:59 +02001152 }
Fredrik Hernqvistca362852019-05-10 15:50:02 +02001153
1154 // Detect volume change.
1155 capture_.echo_path_gain_change =
1156 capture_.echo_path_gain_change ||
1157 (capture_.prev_playout_volume != capture_.playout_volume &&
1158 capture_.prev_playout_volume >= 0);
1159 capture_.prev_playout_volume = capture_.playout_volume;
1160
saza1d600522019-10-18 13:29:43 +02001161 submodules_.echo_controller->AnalyzeCapture(capture_buffer);
peahe0eae3c2016-12-14 01:16:23 -08001162 }
1163
Per Åhgren0695df12020-01-13 14:43:13 +01001164 if (submodules_.agc_manager) {
Per Åhgren3daedb62019-11-22 12:11:40 +01001165 submodules_.agc_manager->AnalyzePreProcess(capture_buffer);
pbos@webrtc.org788acd12014-12-15 09:41:24 +00001166 }
1167
peah2ace3f92016-09-10 04:42:27 -07001168 if (submodule_states_.CaptureMultiBandSubModulesActive() &&
1169 SampleRateSupportsMultiBand(
peahde65ddc2016-09-16 15:02:15 -07001170 capture_nonlocked_.capture_processing_format.sample_rate_hz())) {
1171 capture_buffer->SplitIntoFrequencyBands();
niklase@google.com470e71d2011-07-07 08:21:25 +00001172 }
1173
Per Åhgrene14cb992019-11-27 09:34:22 +01001174 const bool multi_channel_capture = config_.pipeline.multi_channel_capture &&
1175 constants_.multi_channel_capture_support;
1176 if (submodules_.echo_controller && !multi_channel_capture) {
peah522d71b2017-02-23 05:16:26 -08001177 // Force down-mixing of the number of channels after the detection of
1178 // capture signal saturation.
1179 // TODO(peah): Look into ensuring that this kind of tampering with the
1180 // AudioBuffer functionality should not be needed.
1181 capture_buffer->set_num_channels(1);
1182 }
1183
Per Åhgrenc0424252019-12-10 13:04:15 +01001184 if (submodules_.high_pass_filter &&
1185 (!config_.high_pass_filter.apply_in_full_band ||
1186 constants_.enforce_split_band_hpf)) {
1187 submodules_.high_pass_filter->Process(capture_buffer,
1188 /*use_split_band_data=*/true);
peah8271d042016-11-22 07:24:52 -08001189 }
Per Åhgren0cbb58e2019-10-29 22:59:44 +01001190
Per Åhgren0695df12020-01-13 14:43:13 +01001191 if (submodules_.gain_control) {
1192 RETURN_ON_ERR(
1193 submodules_.gain_control->AnalyzeCaptureAudio(*capture_buffer));
1194 }
Per Åhgren2e8e1c62019-12-20 00:42:22 +01001195
Per Åhgren8ad9e742020-01-30 07:40:58 +01001196 if ((!config_.noise_suppression.analyze_linear_aec_output_when_available ||
1197 !linear_aec_buffer || submodules_.echo_control_mobile) &&
1198 submodules_.noise_suppressor) {
1199 submodules_.noise_suppressor->Analyze(*capture_buffer);
saza0bad15f2019-10-16 11:46:11 +02001200 }
peahb58a1582016-03-15 09:34:24 -07001201
saza1d600522019-10-18 13:29:43 +02001202 if (submodules_.echo_control_mobile) {
Per Åhgrenb6e24d72019-04-29 12:14:50 +02001203 // Ensure that the stream delay was set before the call to the
1204 // AECM ProcessCaptureAudio function.
Sam Zackrisson2d31aea2020-01-17 10:55:09 +01001205 if (!capture_.was_stream_delay_set) {
Per Åhgrenb6e24d72019-04-29 12:14:50 +02001206 return AudioProcessing::kStreamParameterNotSetError;
Per Åhgrend0fa8202018-04-18 09:35:13 +02001207 }
1208
saza1d600522019-10-18 13:29:43 +02001209 if (submodules_.noise_suppressor) {
Per Åhgren0cbb58e2019-10-29 22:59:44 +01001210 submodules_.noise_suppressor->Process(capture_buffer);
Per Åhgrenb6e24d72019-04-29 12:14:50 +02001211 }
peahe0eae3c2016-12-14 01:16:23 -08001212
saza1d600522019-10-18 13:29:43 +02001213 RETURN_ON_ERR(submodules_.echo_control_mobile->ProcessCaptureAudio(
Per Åhgren46537a32017-06-07 10:08:10 +02001214 capture_buffer, stream_delay_ms()));
Per Åhgrenb6e24d72019-04-29 12:14:50 +02001215 } else {
saza1d600522019-10-18 13:29:43 +02001216 if (submodules_.echo_controller) {
Per Åhgrenb6e24d72019-04-29 12:14:50 +02001217 data_dumper_->DumpRaw("stream_delay", stream_delay_ms());
1218
Sam Zackrisson2d31aea2020-01-17 10:55:09 +01001219 if (capture_.was_stream_delay_set) {
saza1d600522019-10-18 13:29:43 +02001220 submodules_.echo_controller->SetAudioBufferDelay(stream_delay_ms());
Per Åhgrenb6e24d72019-04-29 12:14:50 +02001221 }
1222
saza1d600522019-10-18 13:29:43 +02001223 submodules_.echo_controller->ProcessCapture(
Per Åhgrenc20a19c2019-11-13 11:12:29 +01001224 capture_buffer, linear_aec_buffer, capture_.echo_path_gain_change);
Per Åhgrenb6e24d72019-04-29 12:14:50 +02001225 }
1226
Per Åhgren2e8e1c62019-12-20 00:42:22 +01001227 if (config_.noise_suppression.analyze_linear_aec_output_when_available &&
Per Åhgren8ad9e742020-01-30 07:40:58 +01001228 linear_aec_buffer && submodules_.noise_suppressor) {
1229 submodules_.noise_suppressor->Analyze(*linear_aec_buffer);
Per Åhgren2e8e1c62019-12-20 00:42:22 +01001230 }
1231
saza1d600522019-10-18 13:29:43 +02001232 if (submodules_.noise_suppressor) {
Per Åhgren0cbb58e2019-10-29 22:59:44 +01001233 submodules_.noise_suppressor->Process(capture_buffer);
saza0bad15f2019-10-16 11:46:11 +02001234 }
Per Åhgren46537a32017-06-07 10:08:10 +02001235 }
ivoc9f4a4a02016-10-28 05:39:16 -07001236
Per Åhgren0695df12020-01-13 14:43:13 +01001237 if (submodules_.agc_manager) {
Per Åhgren3daedb62019-11-22 12:11:40 +01001238 submodules_.agc_manager->Process(capture_buffer);
1239
1240 absl::optional<int> new_digital_gain =
1241 submodules_.agc_manager->GetDigitalComressionGain();
Per Åhgren0695df12020-01-13 14:43:13 +01001242 if (new_digital_gain && submodules_.gain_control) {
Per Åhgren3daedb62019-11-22 12:11:40 +01001243 submodules_.gain_control->set_compression_gain_db(*new_digital_gain);
1244 }
pbos@webrtc.org788acd12014-12-15 09:41:24 +00001245 }
Per Åhgren0695df12020-01-13 14:43:13 +01001246
1247 if (submodules_.gain_control) {
1248 // TODO(peah): Add reporting from AEC3 whether there is echo.
1249 RETURN_ON_ERR(submodules_.gain_control->ProcessCaptureAudio(
1250 capture_buffer, /*stream_has_echo*/ false));
1251 }
niklase@google.com470e71d2011-07-07 08:21:25 +00001252
Ilya Nikolaevskiy80979352021-03-11 11:40:46 +00001253 if (submodule_states_.CaptureMultiBandProcessingPresent() &&
1254 SampleRateSupportsMultiBand(
1255 capture_nonlocked_.capture_processing_format.sample_rate_hz())) {
1256 capture_buffer->MergeFrequencyBands();
Gustaf Ullberg422b9e02019-10-09 13:02:14 +02001257 }
1258
Per Åhgren19775cb2021-03-12 23:08:09 +00001259 if (capture_.capture_output_used) {
1260 if (capture_.capture_fullband_audio) {
1261 const auto& ec = submodules_.echo_controller;
1262 bool ec_active = ec ? ec->ActiveProcessing() : false;
1263 // Only update the fullband buffer if the multiband processing has changed
1264 // the signal. Keep the original signal otherwise.
1265 if (submodule_states_.CaptureMultiBandProcessingActive(ec_active)) {
1266 capture_buffer->CopyTo(capture_.capture_fullband_audio.get());
1267 }
1268 capture_buffer = capture_.capture_fullband_audio.get();
Ilya Nikolaevskiy80979352021-03-11 11:40:46 +00001269 }
Per Åhgren19775cb2021-03-12 23:08:09 +00001270
Sam Zackrisson03cb7e52021-12-06 15:40:04 +01001271 if (submodules_.echo_detector) {
Per Åhgren19775cb2021-03-12 23:08:09 +00001272 submodules_.echo_detector->AnalyzeCaptureAudio(
1273 rtc::ArrayView<const float>(capture_buffer->channels()[0],
1274 capture_buffer->num_frames()));
1275 }
1276
Alessio Bazzicaac29b9c2022-06-20 17:52:43 +02001277 absl::optional<float> voice_probability;
1278 if (!!submodules_.voice_activity_detector) {
1279 voice_probability = submodules_.voice_activity_detector->Analyze(
1280 AudioFrameView<const float>(capture_buffer->channels(),
1281 capture_buffer->num_channels(),
1282 capture_buffer->num_frames()));
1283 }
1284
Per Åhgren19775cb2021-03-12 23:08:09 +00001285 if (submodules_.transient_suppressor) {
Alessio Bazzicaac29b9c2022-06-20 17:52:43 +02001286 float transient_suppressor_voice_probability = 1.0f;
Alessio Bazzicaefbe3af2022-03-18 12:39:00 +01001287 switch (transient_suppressor_vad_mode_) {
1288 case TransientSuppressor::VadMode::kDefault:
1289 if (submodules_.agc_manager) {
Alessio Bazzicaac29b9c2022-06-20 17:52:43 +02001290 transient_suppressor_voice_probability =
1291 submodules_.agc_manager->voice_probability();
Alessio Bazzicaefbe3af2022-03-18 12:39:00 +01001292 }
1293 break;
1294 case TransientSuppressor::VadMode::kRnnVad:
Alessio Bazzicaac29b9c2022-06-20 17:52:43 +02001295 RTC_DCHECK(voice_probability.has_value());
1296 transient_suppressor_voice_probability = *voice_probability;
Alessio Bazzicaefbe3af2022-03-18 12:39:00 +01001297 break;
1298 case TransientSuppressor::VadMode::kNoVad:
1299 // The transient suppressor will ignore `voice_probability`.
1300 break;
1301 }
Alessio Bazzicaac29b9c2022-06-20 17:52:43 +02001302 float delayed_voice_probability =
1303 submodules_.transient_suppressor->Suppress(
1304 capture_buffer->channels()[0], capture_buffer->num_frames(),
1305 capture_buffer->num_channels(),
1306 capture_buffer->split_bands_const(0)[kBand0To8kHz],
1307 capture_buffer->num_frames_per_band(),
1308 /*reference_data=*/nullptr, /*reference_length=*/0,
1309 transient_suppressor_voice_probability, capture_.key_pressed);
1310 if (voice_probability.has_value()) {
1311 *voice_probability = delayed_voice_probability;
1312 }
Per Åhgren19775cb2021-03-12 23:08:09 +00001313 }
1314
Artem Titov0b489302021-07-28 20:50:03 +02001315 // Experimental APM sub-module that analyzes `capture_buffer`.
Per Åhgren19775cb2021-03-12 23:08:09 +00001316 if (submodules_.capture_analyzer) {
1317 submodules_.capture_analyzer->Analyze(capture_buffer);
1318 }
1319
1320 if (submodules_.gain_controller2) {
1321 submodules_.gain_controller2->NotifyAnalogLevel(
1322 recommended_stream_analog_level_locked());
Alessio Bazzicaac29b9c2022-06-20 17:52:43 +02001323 submodules_.gain_controller2->Process(voice_probability, capture_buffer);
Per Åhgren19775cb2021-03-12 23:08:09 +00001324 }
1325
1326 if (submodules_.capture_post_processor) {
1327 submodules_.capture_post_processor->Process(capture_buffer);
1328 }
1329
Per Åhgren19775cb2021-03-12 23:08:09 +00001330 capture_output_rms_.Analyze(rtc::ArrayView<const float>(
1331 capture_buffer->channels_const()[0],
1332 capture_nonlocked_.capture_processing_format.num_frames()));
1333 if (log_rms) {
1334 RmsLevel::Levels levels = capture_output_rms_.AverageAndPeak();
1335 RTC_HISTOGRAM_COUNTS_LINEAR(
1336 "WebRTC.Audio.ApmCaptureOutputLevelAverageRms", levels.average, 1,
1337 RmsLevel::kMinLevelDb, 64);
1338 RTC_HISTOGRAM_COUNTS_LINEAR("WebRTC.Audio.ApmCaptureOutputLevelPeakRms",
1339 levels.peak, 1, RmsLevel::kMinLevelDb, 64);
1340 }
1341
1342 if (submodules_.agc_manager) {
1343 int level = recommended_stream_analog_level_locked();
1344 data_dumper_->DumpRaw("experimental_gain_control_stream_analog_level", 1,
1345 &level);
1346 }
1347
1348 // Compute echo-detector stats.
Sam Zackrisson03cb7e52021-12-06 15:40:04 +01001349 if (submodules_.echo_detector) {
Per Åhgren19775cb2021-03-12 23:08:09 +00001350 auto ed_metrics = submodules_.echo_detector->GetMetrics();
1351 capture_.stats.residual_echo_likelihood = ed_metrics.echo_likelihood;
1352 capture_.stats.residual_echo_likelihood_recent_max =
1353 ed_metrics.echo_likelihood_recent_max;
1354 }
Ilya Nikolaevskiy80979352021-03-11 11:40:46 +00001355 }
1356
Per Åhgren19775cb2021-03-12 23:08:09 +00001357 // Compute echo-controller stats.
Per Åhgrencf4c8722019-12-30 14:32:14 +01001358 if (submodules_.echo_controller) {
1359 auto ec_metrics = submodules_.echo_controller->GetMetrics();
1360 capture_.stats.echo_return_loss = ec_metrics.echo_return_loss;
1361 capture_.stats.echo_return_loss_enhancement =
1362 ec_metrics.echo_return_loss_enhancement;
1363 capture_.stats.delay_ms = ec_metrics.delay_ms;
1364 }
Per Åhgrencf4c8722019-12-30 14:32:14 +01001365
1366 // Pass stats for reporting.
1367 stats_reporter_.UpdateStatistics(capture_.stats);
1368
Per Åhgrendb5d7282021-03-15 16:31:04 +00001369 if (submodules_.capture_levels_adjuster) {
1370 submodules_.capture_levels_adjuster->ApplyPostLevelAdjustment(
1371 *capture_buffer);
1372
1373 // If the analog mic gain emulation is active, retrieve the level from the
1374 // analog gain control and set it to mic gain emulator.
1375 if (config_.capture_level_adjustment.analog_mic_gain_emulation.enabled) {
1376 if (submodules_.agc_manager) {
1377 submodules_.capture_levels_adjuster->SetAnalogMicGainLevel(
1378 submodules_.agc_manager->stream_analog_level());
1379 } else if (submodules_.gain_control) {
1380 submodules_.capture_levels_adjuster->SetAnalogMicGainLevel(
1381 submodules_.gain_control->stream_analog_level());
1382 }
1383 }
1384 }
1385
Per Åhgren55bc0772021-03-12 14:18:36 +00001386 // Temporarily set the output to zero after the stream has been unmuted
1387 // (capture output is again used). The purpose of this is to avoid clicks and
1388 // artefacts in the audio that results when the processing again is
1389 // reactivated after unmuting.
1390 if (!capture_.capture_output_used_last_frame &&
1391 capture_.capture_output_used) {
1392 for (size_t ch = 0; ch < capture_buffer->num_channels(); ++ch) {
1393 rtc::ArrayView<float> channel_view(capture_buffer->channels()[ch],
1394 capture_buffer->num_frames());
1395 std::fill(channel_view.begin(), channel_view.end(), 0.f);
1396 }
1397 }
1398 capture_.capture_output_used_last_frame = capture_.capture_output_used;
1399
peahdf3efa82015-11-28 12:35:15 -08001400 capture_.was_stream_delay_set = false;
niklase@google.com470e71d2011-07-07 08:21:25 +00001401 return kNoError;
1402}
1403
Gustaf Ullberg8c51f2e2019-10-22 15:21:31 +02001404int AudioProcessingImpl::AnalyzeReverseStream(
1405 const float* const* data,
1406 const StreamConfig& reverse_config) {
1407 TRACE_EVENT0("webrtc", "AudioProcessing::AnalyzeReverseStream_StreamConfig");
Markus Handell0df0fae2020-07-07 15:53:34 +02001408 MutexLock lock(&mutex_render_);
Gustaf Ullberg8c51f2e2019-10-22 15:21:31 +02001409 return AnalyzeReverseStreamLocked(data, reverse_config, reverse_config);
1410}
1411
peahde65ddc2016-09-16 15:02:15 -07001412int AudioProcessingImpl::ProcessReverseStream(const float* const* src,
1413 const StreamConfig& input_config,
1414 const StreamConfig& output_config,
1415 float* const* dest) {
peah369f8282015-12-17 06:42:29 -08001416 TRACE_EVENT0("webrtc", "AudioProcessing::ProcessReverseStream_StreamConfig");
Markus Handell0df0fae2020-07-07 15:53:34 +02001417 MutexLock lock(&mutex_render_);
Alessio Bazzica0441bb62021-08-10 15:23:23 +02001418 DenormalDisabler denormal_disabler(use_denormal_disabler_);
1419
peahde65ddc2016-09-16 15:02:15 -07001420 RETURN_ON_ERR(AnalyzeReverseStreamLocked(src, input_config, output_config));
Alex Loiko5825aa62017-12-18 16:02:40 +01001421 if (submodule_states_.RenderMultiBandProcessingActive() ||
1422 submodule_states_.RenderFullBandProcessingActive()) {
peahdf3efa82015-11-28 12:35:15 -08001423 render_.render_audio->CopyTo(formats_.api_format.reverse_output_stream(),
1424 dest);
peah2ace3f92016-09-10 04:42:27 -07001425 } else if (formats_.api_format.reverse_input_stream() !=
1426 formats_.api_format.reverse_output_stream()) {
peahde65ddc2016-09-16 15:02:15 -07001427 render_.render_converter->Convert(src, input_config.num_samples(), dest,
1428 output_config.num_samples());
ekmeyerson60d9b332015-08-14 10:35:55 -07001429 } else {
peahde65ddc2016-09-16 15:02:15 -07001430 CopyAudioIfNeeded(src, input_config.num_frames(),
1431 input_config.num_channels(), dest);
ekmeyerson60d9b332015-08-14 10:35:55 -07001432 }
1433
1434 return kNoError;
Michael Graczyk86c6d332015-07-23 11:41:39 -07001435}
1436
peahdf3efa82015-11-28 12:35:15 -08001437int AudioProcessingImpl::AnalyzeReverseStreamLocked(
ekmeyerson60d9b332015-08-14 10:35:55 -07001438 const float* const* src,
peahde65ddc2016-09-16 15:02:15 -07001439 const StreamConfig& input_config,
1440 const StreamConfig& output_config) {
peahdf3efa82015-11-28 12:35:15 -08001441 if (src == nullptr) {
andrew@webrtc.org17e40642014-03-04 20:58:13 +00001442 return kNullPointerError;
1443 }
andrew@webrtc.org17e40642014-03-04 20:58:13 +00001444
peahde65ddc2016-09-16 15:02:15 -07001445 if (input_config.num_channels() == 0) {
Michael Graczyk86c6d332015-07-23 11:41:39 -07001446 return kBadNumberChannelsError;
andrew@webrtc.org17e40642014-03-04 20:58:13 +00001447 }
1448
peahdf3efa82015-11-28 12:35:15 -08001449 ProcessingConfig processing_config = formats_.api_format;
peahde65ddc2016-09-16 15:02:15 -07001450 processing_config.reverse_input_stream() = input_config;
1451 processing_config.reverse_output_stream() = output_config;
Michael Graczyk86c6d332015-07-23 11:41:39 -07001452
peahdf3efa82015-11-28 12:35:15 -08001453 RETURN_ON_ERR(MaybeInitializeRender(processing_config));
Fredrik Solenbergbbf21a32018-04-12 22:44:09 +02001454 RTC_DCHECK_EQ(input_config.num_frames(),
1455 formats_.api_format.reverse_input_stream().num_frames());
Michael Graczyk86c6d332015-07-23 11:41:39 -07001456
Alessio Bazzica0441bb62021-08-10 15:23:23 +02001457 DenormalDisabler denormal_disabler(use_denormal_disabler_);
1458
aleloi868f32f2017-05-23 07:20:05 -07001459 if (aec_dump_) {
1460 const size_t channel_size =
1461 formats_.api_format.reverse_input_stream().num_frames();
1462 const size_t num_channels =
1463 formats_.api_format.reverse_input_stream().num_channels();
1464 aec_dump_->WriteRenderStreamMessage(
Alex Loikoe36e8bb2018-02-16 11:54:07 +01001465 AudioFrameView<const float>(src, num_channels, channel_size));
aleloi868f32f2017-05-23 07:20:05 -07001466 }
peahdf3efa82015-11-28 12:35:15 -08001467 render_.render_audio->CopyFrom(src,
1468 formats_.api_format.reverse_input_stream());
peahde65ddc2016-09-16 15:02:15 -07001469 return ProcessRenderStreamLocked();
ekmeyerson60d9b332015-08-14 10:35:55 -07001470}
1471
Per Åhgren645f24c2020-03-16 12:06:02 +01001472int AudioProcessingImpl::ProcessReverseStream(const int16_t* const src,
1473 const StreamConfig& input_config,
1474 const StreamConfig& output_config,
1475 int16_t* const dest) {
Per Åhgren71652f42020-03-17 13:23:58 +01001476 TRACE_EVENT0("webrtc", "AudioProcessing::ProcessReverseStream_AudioFrame");
Per Åhgren2507f8c2020-03-19 12:33:29 +01001477
1478 if (input_config.num_channels() <= 0) {
1479 return AudioProcessing::Error::kBadNumberChannelsError;
1480 }
1481
Markus Handell0df0fae2020-07-07 15:53:34 +02001482 MutexLock lock(&mutex_render_);
Alessio Bazzica0441bb62021-08-10 15:23:23 +02001483 DenormalDisabler denormal_disabler(use_denormal_disabler_);
1484
peahdf3efa82015-11-28 12:35:15 -08001485 ProcessingConfig processing_config = formats_.api_format;
ekmeyerson60d9b332015-08-14 10:35:55 -07001486 processing_config.reverse_input_stream().set_sample_rate_hz(
Per Åhgren645f24c2020-03-16 12:06:02 +01001487 input_config.sample_rate_hz());
ekmeyerson60d9b332015-08-14 10:35:55 -07001488 processing_config.reverse_input_stream().set_num_channels(
Per Åhgren645f24c2020-03-16 12:06:02 +01001489 input_config.num_channels());
ekmeyerson60d9b332015-08-14 10:35:55 -07001490 processing_config.reverse_output_stream().set_sample_rate_hz(
Per Åhgren645f24c2020-03-16 12:06:02 +01001491 output_config.sample_rate_hz());
ekmeyerson60d9b332015-08-14 10:35:55 -07001492 processing_config.reverse_output_stream().set_num_channels(
Per Åhgren645f24c2020-03-16 12:06:02 +01001493 output_config.num_channels());
Michael Graczyk86c6d332015-07-23 11:41:39 -07001494
peahdf3efa82015-11-28 12:35:15 -08001495 RETURN_ON_ERR(MaybeInitializeRender(processing_config));
Per Åhgren645f24c2020-03-16 12:06:02 +01001496 if (input_config.num_frames() !=
peahdf3efa82015-11-28 12:35:15 -08001497 formats_.api_format.reverse_input_stream().num_frames()) {
andrew@webrtc.org17e40642014-03-04 20:58:13 +00001498 return kBadDataLengthError;
1499 }
niklase@google.com470e71d2011-07-07 08:21:25 +00001500
aleloi868f32f2017-05-23 07:20:05 -07001501 if (aec_dump_) {
Per Åhgren645f24c2020-03-16 12:06:02 +01001502 aec_dump_->WriteRenderStreamMessage(src, input_config.num_frames(),
1503 input_config.num_channels());
aleloi868f32f2017-05-23 07:20:05 -07001504 }
1505
Per Åhgren645f24c2020-03-16 12:06:02 +01001506 render_.render_audio->CopyFrom(src, input_config);
peahde65ddc2016-09-16 15:02:15 -07001507 RETURN_ON_ERR(ProcessRenderStreamLocked());
Per Åhgrena1351272019-08-15 12:15:46 +02001508 if (submodule_states_.RenderMultiBandProcessingActive() ||
1509 submodule_states_.RenderFullBandProcessingActive()) {
Per Åhgren645f24c2020-03-16 12:06:02 +01001510 render_.render_audio->CopyTo(output_config, dest);
Per Åhgrena1351272019-08-15 12:15:46 +02001511 }
aluebsb0319552016-03-17 20:39:53 -07001512 return kNoError;
andrew@webrtc.org17e40642014-03-04 20:58:13 +00001513}
niklase@google.com470e71d2011-07-07 08:21:25 +00001514
peahde65ddc2016-09-16 15:02:15 -07001515int AudioProcessingImpl::ProcessRenderStreamLocked() {
1516 AudioBuffer* render_buffer = render_.render_audio.get(); // For brevity.
peah9e6a2902017-05-15 07:19:21 -07001517
Alex Loiko73ec0192018-05-15 10:52:28 +02001518 HandleRenderRuntimeSettings();
Alessio Bazzica0441bb62021-08-10 15:23:23 +02001519 DenormalDisabler denormal_disabler(use_denormal_disabler_);
Alex Loiko73ec0192018-05-15 10:52:28 +02001520
saza1d600522019-10-18 13:29:43 +02001521 if (submodules_.render_pre_processor) {
1522 submodules_.render_pre_processor->Process(render_buffer);
Alex Loiko5825aa62017-12-18 16:02:40 +01001523 }
1524
Alessio Bazzicad2b97402018-08-09 14:23:11 +02001525 QueueNonbandedRenderAudio(render_buffer);
1526
peah2ace3f92016-09-10 04:42:27 -07001527 if (submodule_states_.RenderMultiBandSubModulesActive() &&
peahde65ddc2016-09-16 15:02:15 -07001528 SampleRateSupportsMultiBand(
1529 formats_.render_processing_format.sample_rate_hz())) {
1530 render_buffer->SplitIntoFrequencyBands();
niklase@google.com470e71d2011-07-07 08:21:25 +00001531 }
1532
peahce4d9152017-05-19 01:28:05 -07001533 if (submodule_states_.RenderMultiBandSubModulesActive()) {
1534 QueueBandedRenderAudio(render_buffer);
1535 }
1536
Alessio Bazzicad2b97402018-08-09 14:23:11 +02001537 // TODO(peah): Perform the queuing inside QueueRenderAudiuo().
saza1d600522019-10-18 13:29:43 +02001538 if (submodules_.echo_controller) {
1539 submodules_.echo_controller->AnalyzeRender(render_buffer);
peahe0eae3c2016-12-14 01:16:23 -08001540 }
niklase@google.com470e71d2011-07-07 08:21:25 +00001541
peah2ace3f92016-09-10 04:42:27 -07001542 if (submodule_states_.RenderMultiBandProcessingActive() &&
peahde65ddc2016-09-16 15:02:15 -07001543 SampleRateSupportsMultiBand(
1544 formats_.render_processing_format.sample_rate_hz())) {
1545 render_buffer->MergeFrequencyBands();
ekmeyerson60d9b332015-08-14 10:35:55 -07001546 }
1547
andrew@webrtc.org17e40642014-03-04 20:58:13 +00001548 return kNoError;
niklase@google.com470e71d2011-07-07 08:21:25 +00001549}
1550
1551int AudioProcessingImpl::set_stream_delay_ms(int delay) {
Markus Handell0df0fae2020-07-07 15:53:34 +02001552 MutexLock lock(&mutex_capture_);
andrew@webrtc.org5f23d642012-05-29 21:14:06 +00001553 Error retval = kNoError;
peahdf3efa82015-11-28 12:35:15 -08001554 capture_.was_stream_delay_set = true;
andrew@webrtc.org6f9f8172012-03-06 19:03:39 +00001555
niklase@google.com470e71d2011-07-07 08:21:25 +00001556 if (delay < 0) {
andrew@webrtc.org5f23d642012-05-29 21:14:06 +00001557 delay = 0;
1558 retval = kBadStreamParameterWarning;
niklase@google.com470e71d2011-07-07 08:21:25 +00001559 }
1560
1561 // TODO(ajm): the max is rather arbitrarily chosen; investigate.
1562 if (delay > 500) {
andrew@webrtc.org5f23d642012-05-29 21:14:06 +00001563 delay = 500;
1564 retval = kBadStreamParameterWarning;
niklase@google.com470e71d2011-07-07 08:21:25 +00001565 }
1566
peahdf3efa82015-11-28 12:35:15 -08001567 capture_nonlocked_.stream_delay_ms = delay;
andrew@webrtc.org5f23d642012-05-29 21:14:06 +00001568 return retval;
niklase@google.com470e71d2011-07-07 08:21:25 +00001569}
1570
Per Åhgrenc20a19c2019-11-13 11:12:29 +01001571bool AudioProcessingImpl::GetLinearAecOutput(
1572 rtc::ArrayView<std::array<float, 160>> linear_output) const {
Markus Handell0df0fae2020-07-07 15:53:34 +02001573 MutexLock lock(&mutex_capture_);
Per Åhgrenc20a19c2019-11-13 11:12:29 +01001574 AudioBuffer* linear_aec_buffer = capture_.linear_aec_output.get();
1575
1576 RTC_DCHECK(linear_aec_buffer);
1577 if (linear_aec_buffer) {
1578 RTC_DCHECK_EQ(1, linear_aec_buffer->num_bands());
1579 RTC_DCHECK_EQ(linear_output.size(), linear_aec_buffer->num_channels());
1580
1581 for (size_t ch = 0; ch < linear_aec_buffer->num_channels(); ++ch) {
1582 RTC_DCHECK_EQ(linear_output[ch].size(), linear_aec_buffer->num_frames());
1583 rtc::ArrayView<const float> channel_view =
1584 rtc::ArrayView<const float>(linear_aec_buffer->channels_const()[ch],
1585 linear_aec_buffer->num_frames());
Gustaf Ullberg45436972020-11-13 14:30:30 +01001586 FloatS16ToFloat(channel_view.data(), channel_view.size(),
1587 linear_output[ch].data());
Per Åhgrenc20a19c2019-11-13 11:12:29 +01001588 }
1589 return true;
1590 }
1591 RTC_LOG(LS_ERROR) << "No linear AEC output available";
Artem Titovd3251962021-11-15 16:57:07 +01001592 RTC_DCHECK_NOTREACHED();
Per Åhgrenc20a19c2019-11-13 11:12:29 +01001593 return false;
1594}
1595
niklase@google.com470e71d2011-07-07 08:21:25 +00001596int AudioProcessingImpl::stream_delay_ms() const {
peahdf3efa82015-11-28 12:35:15 -08001597 // Used as callback from submodules, hence locking is not allowed.
1598 return capture_nonlocked_.stream_delay_ms;
niklase@google.com470e71d2011-07-07 08:21:25 +00001599}
1600
andrew@webrtc.org17e40642014-03-04 20:58:13 +00001601void AudioProcessingImpl::set_stream_key_pressed(bool key_pressed) {
Markus Handell0df0fae2020-07-07 15:53:34 +02001602 MutexLock lock(&mutex_capture_);
peahdf3efa82015-11-28 12:35:15 -08001603 capture_.key_pressed = key_pressed;
andrew@webrtc.org17e40642014-03-04 20:58:13 +00001604}
1605
Sam Zackrissonf0d1c032019-03-27 13:28:08 +01001606void AudioProcessingImpl::set_stream_analog_level(int level) {
Markus Handell0df0fae2020-07-07 15:53:34 +02001607 MutexLock lock_capture(&mutex_capture_);
Per Åhgren0e3198e2019-11-18 08:52:22 +01001608
Per Åhgrendb5d7282021-03-15 16:31:04 +00001609 if (config_.capture_level_adjustment.analog_mic_gain_emulation.enabled) {
1610 // If the analog mic gain is emulated internally, simply cache the level for
1611 // later reporting back as the recommended stream analog level to use.
1612 capture_.cached_stream_analog_level_ = level;
1613 return;
1614 }
1615
Per Åhgren0e3198e2019-11-18 08:52:22 +01001616 if (submodules_.agc_manager) {
1617 submodules_.agc_manager->set_stream_analog_level(level);
1618 data_dumper_->DumpRaw("experimental_gain_control_set_stream_analog_level",
1619 1, &level);
Per Åhgrendb5d7282021-03-15 16:31:04 +00001620 return;
1621 }
1622
1623 if (submodules_.gain_control) {
Per Åhgren0e3198e2019-11-18 08:52:22 +01001624 int error = submodules_.gain_control->set_stream_analog_level(level);
1625 RTC_DCHECK_EQ(kNoError, error);
Per Åhgrendb5d7282021-03-15 16:31:04 +00001626 return;
Per Åhgren0e3198e2019-11-18 08:52:22 +01001627 }
Per Åhgrendb5d7282021-03-15 16:31:04 +00001628
1629 // If no analog mic gain control functionality is in place, cache the level
1630 // for later reporting back as the recommended stream analog level to use.
1631 capture_.cached_stream_analog_level_ = level;
Sam Zackrissonf0d1c032019-03-27 13:28:08 +01001632}
1633
1634int AudioProcessingImpl::recommended_stream_analog_level() const {
Markus Handell0df0fae2020-07-07 15:53:34 +02001635 MutexLock lock_capture(&mutex_capture_);
Markus Handell02ba1d22020-05-14 14:31:18 +02001636 return recommended_stream_analog_level_locked();
1637}
1638
1639int AudioProcessingImpl::recommended_stream_analog_level_locked() const {
Per Åhgrendb5d7282021-03-15 16:31:04 +00001640 if (config_.capture_level_adjustment.analog_mic_gain_emulation.enabled) {
Per Åhgren0695df12020-01-13 14:43:13 +01001641 return capture_.cached_stream_analog_level_;
Per Åhgren0e3198e2019-11-18 08:52:22 +01001642 }
Per Åhgrendb5d7282021-03-15 16:31:04 +00001643
1644 if (submodules_.agc_manager) {
1645 return submodules_.agc_manager->stream_analog_level();
1646 }
1647
1648 if (submodules_.gain_control) {
1649 return submodules_.gain_control->stream_analog_level();
1650 }
1651
1652 return capture_.cached_stream_analog_level_;
Sam Zackrissonf0d1c032019-03-27 13:28:08 +01001653}
1654
Per Åhgren09e9a832020-05-11 11:03:47 +02001655bool AudioProcessingImpl::CreateAndAttachAecDump(const std::string& file_name,
1656 int64_t max_log_size_bytes,
1657 rtc::TaskQueue* worker_queue) {
1658 std::unique_ptr<AecDump> aec_dump =
1659 AecDumpFactory::Create(file_name, max_log_size_bytes, worker_queue);
1660 if (!aec_dump) {
1661 return false;
1662 }
1663
1664 AttachAecDump(std::move(aec_dump));
1665 return true;
1666}
1667
1668bool AudioProcessingImpl::CreateAndAttachAecDump(FILE* handle,
1669 int64_t max_log_size_bytes,
1670 rtc::TaskQueue* worker_queue) {
1671 std::unique_ptr<AecDump> aec_dump =
1672 AecDumpFactory::Create(handle, max_log_size_bytes, worker_queue);
1673 if (!aec_dump) {
1674 return false;
1675 }
1676
1677 AttachAecDump(std::move(aec_dump));
1678 return true;
1679}
1680
aleloi868f32f2017-05-23 07:20:05 -07001681void AudioProcessingImpl::AttachAecDump(std::unique_ptr<AecDump> aec_dump) {
1682 RTC_DCHECK(aec_dump);
Markus Handell0df0fae2020-07-07 15:53:34 +02001683 MutexLock lock_render(&mutex_render_);
1684 MutexLock lock_capture(&mutex_capture_);
aleloi868f32f2017-05-23 07:20:05 -07001685
1686 // The previously attached AecDump will be destroyed with the
1687 // 'aec_dump' parameter, which is after locks are released.
1688 aec_dump_.swap(aec_dump);
1689 WriteAecDumpConfigMessage(true);
Minyue Li656d6092018-08-10 15:38:52 +02001690 aec_dump_->WriteInitMessage(formats_.api_format, rtc::TimeUTCMillis());
aleloi868f32f2017-05-23 07:20:05 -07001691}
1692
1693void AudioProcessingImpl::DetachAecDump() {
1694 // The d-tor of a task-queue based AecDump blocks until all pending
1695 // tasks are done. This construction avoids blocking while holding
1696 // the render and capture locks.
1697 std::unique_ptr<AecDump> aec_dump = nullptr;
1698 {
Markus Handell0df0fae2020-07-07 15:53:34 +02001699 MutexLock lock_render(&mutex_render_);
1700 MutexLock lock_capture(&mutex_capture_);
aleloi868f32f2017-05-23 07:20:05 -07001701 aec_dump = std::move(aec_dump_);
1702 }
1703}
1704
peah8271d042016-11-22 07:24:52 -08001705AudioProcessing::Config AudioProcessingImpl::GetConfig() const {
Markus Handell0df0fae2020-07-07 15:53:34 +02001706 MutexLock lock_render(&mutex_render_);
1707 MutexLock lock_capture(&mutex_capture_);
peah8271d042016-11-22 07:24:52 -08001708 return config_;
1709}
1710
peah2ace3f92016-09-10 04:42:27 -07001711bool AudioProcessingImpl::UpdateActiveSubmoduleStates() {
1712 return submodule_states_.Update(
Per Åhgren62ea0aa2019-12-09 10:18:44 +01001713 config_.high_pass_filter.enabled, !!submodules_.echo_control_mobile,
Sam Zackrisson03cb7e52021-12-06 15:40:04 +01001714 !!submodules_.noise_suppressor, !!submodules_.gain_control,
Hanna Silen0c1ad292022-06-16 16:35:45 +02001715 !!submodules_.gain_controller2, !!submodules_.voice_activity_detector,
Per Åhgrendb5d7282021-03-15 16:31:04 +00001716 config_.pre_amplifier.enabled || config_.capture_level_adjustment.enabled,
1717 capture_nonlocked_.echo_controller_enabled,
Alessio Bazzica1db0a262022-02-15 14:18:09 +00001718 !!submodules_.transient_suppressor);
ekmeyerson60d9b332015-08-14 10:35:55 -07001719}
1720
Per Åhgrenc0734712020-01-02 15:15:36 +01001721void AudioProcessingImpl::InitializeTransientSuppressor() {
Gustaf Ullberga399c822021-05-18 12:17:56 +02001722 if (config_.transient_suppression.enabled &&
1723 !constants_.transient_suppressor_forced_off) {
sazaaa42ecd2020-04-01 15:24:40 +02001724 // Attempt to create a transient suppressor, if one is not already created.
Per Åhgrenc0734712020-01-02 15:15:36 +01001725 if (!submodules_.transient_suppressor) {
Alessio Bazzicaefbe3af2022-03-18 12:39:00 +01001726 submodules_.transient_suppressor = CreateTransientSuppressor(
Alessio Bazzica080006b2022-04-08 09:54:27 +02001727 submodule_creation_overrides_, transient_suppressor_vad_mode_,
1728 proc_fullband_sample_rate_hz(), capture_nonlocked_.split_rate,
1729 num_proc_channels());
1730 if (!submodules_.transient_suppressor) {
1731 RTC_LOG(LS_WARNING)
1732 << "No transient suppressor created (probably disabled)";
1733 }
1734 } else {
sazaaa42ecd2020-04-01 15:24:40 +02001735 submodules_.transient_suppressor->Initialize(
1736 proc_fullband_sample_rate_hz(), capture_nonlocked_.split_rate,
1737 num_proc_channels());
sazaaa42ecd2020-04-01 15:24:40 +02001738 }
Per Åhgrenc0734712020-01-02 15:15:36 +01001739 } else {
1740 submodules_.transient_suppressor.reset();
pbos@webrtc.org788acd12014-12-15 09:41:24 +00001741 }
pbos@webrtc.org788acd12014-12-15 09:41:24 +00001742}
1743
Per Åhgren0f14db22020-01-03 14:27:14 +01001744void AudioProcessingImpl::InitializeHighPassFilter(bool forced_reset) {
Per Åhgrenb8106462019-12-04 08:34:12 +01001745 bool high_pass_filter_needed_by_aec =
1746 config_.echo_canceller.enabled &&
1747 config_.echo_canceller.enforce_high_pass_filtering &&
1748 !config_.echo_canceller.mobile_mode;
1749 if (submodule_states_.HighPassFilteringRequired() ||
1750 high_pass_filter_needed_by_aec) {
Per Åhgrenc0424252019-12-10 13:04:15 +01001751 bool use_full_band = config_.high_pass_filter.apply_in_full_band &&
1752 !constants_.enforce_split_band_hpf;
1753 int rate = use_full_band ? proc_fullband_sample_rate_hz()
1754 : proc_split_sample_rate_hz();
1755 size_t num_channels =
1756 use_full_band ? num_output_channels() : num_proc_channels();
1757
Per Åhgren0f14db22020-01-03 14:27:14 +01001758 if (!submodules_.high_pass_filter ||
1759 rate != submodules_.high_pass_filter->sample_rate_hz() ||
1760 forced_reset ||
1761 num_channels != submodules_.high_pass_filter->num_channels()) {
1762 submodules_.high_pass_filter.reset(
1763 new HighPassFilter(rate, num_channels));
1764 }
peah8271d042016-11-22 07:24:52 -08001765 } else {
saza1d600522019-10-18 13:29:43 +02001766 submodules_.high_pass_filter.reset();
peah8271d042016-11-22 07:24:52 -08001767 }
1768}
alessiob3ec96df2017-05-22 06:57:06 -07001769
Gustaf Ullberg8eb9c7d2017-10-14 08:28:46 +02001770void AudioProcessingImpl::InitializeEchoController() {
Per Åhgrenb6e24d72019-04-29 12:14:50 +02001771 bool use_echo_controller =
1772 echo_control_factory_ ||
Per Åhgren62ea0aa2019-12-09 10:18:44 +01001773 (config_.echo_canceller.enabled && !config_.echo_canceller.mobile_mode);
Per Åhgrenb6e24d72019-04-29 12:14:50 +02001774
1775 if (use_echo_controller) {
1776 // Create and activate the echo controller.
Per Åhgren200feba2019-03-06 04:16:46 +01001777 if (echo_control_factory_) {
Per Åhgren4e5c7092019-11-01 20:44:11 +01001778 submodules_.echo_controller = echo_control_factory_->Create(
1779 proc_sample_rate_hz(), num_reverse_channels(), num_proc_channels());
Gustaf Ullberg2c6f3732019-11-07 17:15:12 +01001780 RTC_DCHECK(submodules_.echo_controller);
Per Åhgren200feba2019-03-06 04:16:46 +01001781 } else {
Sam Zackrisson64cdcc02022-04-07 15:28:14 +02001782 EchoCanceller3Config config;
1783 absl::optional<EchoCanceller3Config> multichannel_config;
1784 if (use_setup_specific_default_aec3_config_) {
1785 multichannel_config = EchoCanceller3::CreateDefaultMultichannelConfig();
1786 }
saza1d600522019-10-18 13:29:43 +02001787 submodules_.echo_controller = std::make_unique<EchoCanceller3>(
Sam Zackrisson64cdcc02022-04-07 15:28:14 +02001788 config, multichannel_config, proc_sample_rate_hz(),
1789 num_reverse_channels(), num_proc_channels());
Per Åhgren200feba2019-03-06 04:16:46 +01001790 }
1791
Per Åhgrenc20a19c2019-11-13 11:12:29 +01001792 // Setup the storage for returning the linear AEC output.
1793 if (config_.echo_canceller.export_linear_aec_output) {
1794 constexpr int kLinearOutputRateHz = 16000;
1795 capture_.linear_aec_output = std::make_unique<AudioBuffer>(
1796 kLinearOutputRateHz, num_proc_channels(), kLinearOutputRateHz,
1797 num_proc_channels(), kLinearOutputRateHz, num_proc_channels());
1798 } else {
1799 capture_.linear_aec_output.reset();
1800 }
1801
Per Åhgren200feba2019-03-06 04:16:46 +01001802 capture_nonlocked_.echo_controller_enabled = true;
Per Åhgren200feba2019-03-06 04:16:46 +01001803
saza1d600522019-10-18 13:29:43 +02001804 submodules_.echo_control_mobile.reset();
Per Åhgrenb6e24d72019-04-29 12:14:50 +02001805 aecm_render_signal_queue_.reset();
Per Åhgrenf204faf2019-04-25 15:18:06 +02001806 return;
peahe0eae3c2016-12-14 01:16:23 -08001807 }
Per Åhgrenf204faf2019-04-25 15:18:06 +02001808
saza1d600522019-10-18 13:29:43 +02001809 submodules_.echo_controller.reset();
Per Åhgrenf204faf2019-04-25 15:18:06 +02001810 capture_nonlocked_.echo_controller_enabled = false;
Per Åhgrenc20a19c2019-11-13 11:12:29 +01001811 capture_.linear_aec_output.reset();
Per Åhgrenf204faf2019-04-25 15:18:06 +02001812
1813 if (!config_.echo_canceller.enabled) {
saza1d600522019-10-18 13:29:43 +02001814 submodules_.echo_control_mobile.reset();
Per Åhgrenb6e24d72019-04-29 12:14:50 +02001815 aecm_render_signal_queue_.reset();
Per Åhgrenf204faf2019-04-25 15:18:06 +02001816 return;
1817 }
1818
1819 if (config_.echo_canceller.mobile_mode) {
1820 // Create and activate AECM.
Per Åhgrenb6e24d72019-04-29 12:14:50 +02001821 size_t max_element_size =
1822 std::max(static_cast<size_t>(1),
1823 kMaxAllowedValuesOfSamplesPerBand *
1824 EchoControlMobileImpl::NumCancellersRequired(
1825 num_output_channels(), num_reverse_channels()));
1826
1827 std::vector<int16_t> template_queue_element(max_element_size);
1828
1829 aecm_render_signal_queue_.reset(
1830 new SwapQueue<std::vector<int16_t>, RenderQueueItemVerifier<int16_t>>(
1831 kMaxNumFramesToBuffer, template_queue_element,
1832 RenderQueueItemVerifier<int16_t>(max_element_size)));
1833
1834 aecm_render_queue_buffer_.resize(max_element_size);
1835 aecm_capture_queue_buffer_.resize(max_element_size);
1836
saza1d600522019-10-18 13:29:43 +02001837 submodules_.echo_control_mobile.reset(new EchoControlMobileImpl());
Per Åhgrenb6e24d72019-04-29 12:14:50 +02001838
saza1d600522019-10-18 13:29:43 +02001839 submodules_.echo_control_mobile->Initialize(proc_split_sample_rate_hz(),
1840 num_reverse_channels(),
1841 num_output_channels());
Per Åhgrenf204faf2019-04-25 15:18:06 +02001842 return;
1843 }
1844
saza1d600522019-10-18 13:29:43 +02001845 submodules_.echo_control_mobile.reset();
Per Åhgrenb6e24d72019-04-29 12:14:50 +02001846 aecm_render_signal_queue_.reset();
peahe0eae3c2016-12-14 01:16:23 -08001847}
peah8271d042016-11-22 07:24:52 -08001848
Per Åhgren0695df12020-01-13 14:43:13 +01001849void AudioProcessingImpl::InitializeGainController1() {
1850 if (!config_.gain_controller1.enabled) {
1851 submodules_.agc_manager.reset();
1852 submodules_.gain_control.reset();
1853 return;
1854 }
1855
1856 if (!submodules_.gain_control) {
1857 submodules_.gain_control.reset(new GainControlImpl());
1858 }
1859
1860 submodules_.gain_control->Initialize(num_proc_channels(),
1861 proc_sample_rate_hz());
Per Åhgren0695df12020-01-13 14:43:13 +01001862 if (!config_.gain_controller1.analog_gain_controller.enabled) {
1863 int error = submodules_.gain_control->set_mode(
1864 Agc1ConfigModeToInterfaceMode(config_.gain_controller1.mode));
1865 RTC_DCHECK_EQ(kNoError, error);
1866 error = submodules_.gain_control->set_target_level_dbfs(
1867 config_.gain_controller1.target_level_dbfs);
1868 RTC_DCHECK_EQ(kNoError, error);
1869 error = submodules_.gain_control->set_compression_gain_db(
1870 config_.gain_controller1.compression_gain_db);
1871 RTC_DCHECK_EQ(kNoError, error);
1872 error = submodules_.gain_control->enable_limiter(
1873 config_.gain_controller1.enable_limiter);
1874 RTC_DCHECK_EQ(kNoError, error);
Hanna Silencd597042021-11-02 11:02:48 +01001875 constexpr int kAnalogLevelMinimum = 0;
1876 constexpr int kAnalogLevelMaximum = 255;
Per Åhgren0695df12020-01-13 14:43:13 +01001877 error = submodules_.gain_control->set_analog_level_limits(
Hanna Silencd597042021-11-02 11:02:48 +01001878 kAnalogLevelMinimum, kAnalogLevelMaximum);
Per Åhgren0695df12020-01-13 14:43:13 +01001879 RTC_DCHECK_EQ(kNoError, error);
1880
1881 submodules_.agc_manager.reset();
1882 return;
1883 }
1884
1885 if (!submodules_.agc_manager.get() ||
1886 submodules_.agc_manager->num_channels() !=
Alessio Bazzicabab12852022-02-03 16:30:25 +01001887 static_cast<int>(num_proc_channels())) {
Per Åhgren0695df12020-01-13 14:43:13 +01001888 int stream_analog_level = -1;
1889 const bool re_creation = !!submodules_.agc_manager;
1890 if (re_creation) {
1891 stream_analog_level = submodules_.agc_manager->stream_analog_level();
1892 }
1893 submodules_.agc_manager.reset(new AgcManagerDirect(
1894 num_proc_channels(),
1895 config_.gain_controller1.analog_gain_controller.startup_min_volume,
1896 config_.gain_controller1.analog_gain_controller.clipped_level_min,
Per Åhgren0695df12020-01-13 14:43:13 +01001897 !config_.gain_controller1.analog_gain_controller
1898 .enable_digital_adaptive,
Hanna Silenb8dc7fa2021-05-20 17:37:56 +02001899 config_.gain_controller1.analog_gain_controller.clipped_level_step,
1900 config_.gain_controller1.analog_gain_controller.clipped_ratio_threshold,
Hanna Silena0047152021-06-03 03:29:38 +02001901 config_.gain_controller1.analog_gain_controller.clipped_wait_frames,
1902 config_.gain_controller1.analog_gain_controller.clipping_predictor));
Per Åhgren0695df12020-01-13 14:43:13 +01001903 if (re_creation) {
1904 submodules_.agc_manager->set_stream_analog_level(stream_analog_level);
1905 }
1906 }
1907 submodules_.agc_manager->Initialize();
1908 submodules_.agc_manager->SetupDigitalGainControl(
1909 submodules_.gain_control.get());
Per Åhgren0a144a72021-02-09 08:47:51 +01001910 submodules_.agc_manager->HandleCaptureOutputUsedChange(
1911 capture_.capture_output_used);
Per Åhgren0695df12020-01-13 14:43:13 +01001912}
1913
Alessio Bazzica38901042021-10-14 12:14:21 +02001914void AudioProcessingImpl::InitializeGainController2(bool config_has_changed) {
1915 if (!config_has_changed) {
1916 return;
1917 }
1918 if (!config_.gain_controller2.enabled) {
Per Åhgren2bd85ab2020-01-03 10:36:34 +01001919 submodules_.gain_controller2.reset();
Alessio Bazzica38901042021-10-14 12:14:21 +02001920 return;
1921 }
1922 if (!submodules_.gain_controller2 || config_has_changed) {
Hanna Silen0c1ad292022-06-16 16:35:45 +02001923 const bool use_internal_vad =
1924 transient_suppressor_vad_mode_ != TransientSuppressor::VadMode::kRnnVad;
Alessio Bazzica38901042021-10-14 12:14:21 +02001925 submodules_.gain_controller2 = std::make_unique<GainController2>(
1926 config_.gain_controller2, proc_fullband_sample_rate_hz(),
Hanna Silen0c1ad292022-06-16 16:35:45 +02001927 num_input_channels(), use_internal_vad);
1928 }
1929}
1930
1931void AudioProcessingImpl::InitializeVoiceActivityDetector(
1932 bool config_has_changed) {
1933 if (!config_has_changed) {
1934 return;
1935 }
1936 const bool use_vad =
1937 transient_suppressor_vad_mode_ == TransientSuppressor::VadMode::kRnnVad &&
1938 config_.gain_controller2.enabled &&
1939 config_.gain_controller2.adaptive_digital.enabled;
1940 if (!use_vad) {
1941 submodules_.voice_activity_detector.reset();
1942 return;
1943 }
1944 if (!submodules_.voice_activity_detector || config_has_changed) {
1945 RTC_DCHECK(!!submodules_.gain_controller2);
1946 // TODO(bugs.webrtc.org/13663): Cache CPU features in APM and use here.
1947 submodules_.voice_activity_detector =
1948 std::make_unique<VoiceActivityDetectorWrapper>(
1949 config_.gain_controller2.adaptive_digital.vad_reset_period_ms,
1950 submodules_.gain_controller2->GetCpuFeatures(),
1951 proc_fullband_sample_rate_hz());
alessiob3ec96df2017-05-22 06:57:06 -07001952 }
1953}
1954
saza0bad15f2019-10-16 11:46:11 +02001955void AudioProcessingImpl::InitializeNoiseSuppressor() {
Per Åhgren0cbb58e2019-10-29 22:59:44 +01001956 submodules_.noise_suppressor.reset();
1957
saza0bad15f2019-10-16 11:46:11 +02001958 if (config_.noise_suppression.enabled) {
sazaaa42ecd2020-04-01 15:24:40 +02001959 auto map_level =
1960 [](AudioProcessing::Config::NoiseSuppression::Level level) {
1961 using NoiseSuppresionConfig =
1962 AudioProcessing::Config::NoiseSuppression;
1963 switch (level) {
1964 case NoiseSuppresionConfig::kLow:
1965 return NsConfig::SuppressionLevel::k6dB;
1966 case NoiseSuppresionConfig::kModerate:
1967 return NsConfig::SuppressionLevel::k12dB;
1968 case NoiseSuppresionConfig::kHigh:
1969 return NsConfig::SuppressionLevel::k18dB;
1970 case NoiseSuppresionConfig::kVeryHigh:
1971 return NsConfig::SuppressionLevel::k21dB;
sazaaa42ecd2020-04-01 15:24:40 +02001972 }
Karl Wibergc95b9392020-11-08 00:49:37 +01001973 RTC_CHECK_NOTREACHED();
sazaaa42ecd2020-04-01 15:24:40 +02001974 };
Per Åhgren0cbb58e2019-10-29 22:59:44 +01001975
sazaaa42ecd2020-04-01 15:24:40 +02001976 NsConfig cfg;
1977 cfg.target_level = map_level(config_.noise_suppression.level);
1978 submodules_.noise_suppressor = std::make_unique<NoiseSuppressor>(
1979 cfg, proc_sample_rate_hz(), num_proc_channels());
saza0bad15f2019-10-16 11:46:11 +02001980 }
1981}
1982
Per Åhgrendb5d7282021-03-15 16:31:04 +00001983void AudioProcessingImpl::InitializeCaptureLevelsAdjuster() {
1984 if (config_.pre_amplifier.enabled ||
1985 config_.capture_level_adjustment.enabled) {
1986 // Use both the pre-amplifier and the capture level adjustment gains as
1987 // pre-gains.
1988 float pre_gain = 1.f;
1989 if (config_.pre_amplifier.enabled) {
1990 pre_gain *= config_.pre_amplifier.fixed_gain_factor;
1991 }
1992 if (config_.capture_level_adjustment.enabled) {
1993 pre_gain *= config_.capture_level_adjustment.pre_gain_factor;
1994 }
1995
1996 submodules_.capture_levels_adjuster =
1997 std::make_unique<CaptureLevelsAdjuster>(
1998 config_.capture_level_adjustment.analog_mic_gain_emulation.enabled,
1999 config_.capture_level_adjustment.analog_mic_gain_emulation
2000 .initial_level,
2001 pre_gain, config_.capture_level_adjustment.post_gain_factor);
Alex Loikob5c9a792018-04-16 16:31:22 +02002002 } else {
Per Åhgrendb5d7282021-03-15 16:31:04 +00002003 submodules_.capture_levels_adjuster.reset();
Alex Loikob5c9a792018-04-16 16:31:22 +02002004 }
2005}
2006
ivoc9f4a4a02016-10-28 05:39:16 -07002007void AudioProcessingImpl::InitializeResidualEchoDetector() {
Sam Zackrisson03cb7e52021-12-06 15:40:04 +01002008 if (submodules_.echo_detector) {
2009 submodules_.echo_detector->Initialize(
2010 proc_fullband_sample_rate_hz(), 1,
2011 formats_.render_processing_format.sample_rate_hz(), 1);
2012 }
ivoc9f4a4a02016-10-28 05:39:16 -07002013}
2014
Valeriia Nemychnikovaf06eb572018-08-29 10:37:09 +02002015void AudioProcessingImpl::InitializeAnalyzer() {
saza1d600522019-10-18 13:29:43 +02002016 if (submodules_.capture_analyzer) {
2017 submodules_.capture_analyzer->Initialize(proc_fullband_sample_rate_hz(),
2018 num_proc_channels());
Valeriia Nemychnikovaf06eb572018-08-29 10:37:09 +02002019 }
2020}
2021
Sam Zackrisson0beac582017-09-25 12:04:02 +02002022void AudioProcessingImpl::InitializePostProcessor() {
saza1d600522019-10-18 13:29:43 +02002023 if (submodules_.capture_post_processor) {
2024 submodules_.capture_post_processor->Initialize(
Gustaf Ullberg422b9e02019-10-09 13:02:14 +02002025 proc_fullband_sample_rate_hz(), num_proc_channels());
Sam Zackrisson0beac582017-09-25 12:04:02 +02002026 }
2027}
2028
Alex Loiko5825aa62017-12-18 16:02:40 +01002029void AudioProcessingImpl::InitializePreProcessor() {
saza1d600522019-10-18 13:29:43 +02002030 if (submodules_.render_pre_processor) {
2031 submodules_.render_pre_processor->Initialize(
Alex Loiko5825aa62017-12-18 16:02:40 +01002032 formats_.render_processing_format.sample_rate_hz(),
2033 formats_.render_processing_format.num_channels());
2034 }
2035}
2036
aleloi868f32f2017-05-23 07:20:05 -07002037void AudioProcessingImpl::WriteAecDumpConfigMessage(bool forced) {
2038 if (!aec_dump_) {
2039 return;
2040 }
Per Åhgrenf204faf2019-04-25 15:18:06 +02002041
2042 std::string experiments_description = "";
aleloi868f32f2017-05-23 07:20:05 -07002043 // TODO(peah): Add semicolon-separated concatenations of experiment
2044 // descriptions for other submodules.
Per Åhgren0695df12020-01-13 14:43:13 +01002045 if (config_.gain_controller1.analog_gain_controller.clipped_level_min !=
2046 kClippedLevelMin) {
aleloi868f32f2017-05-23 07:20:05 -07002047 experiments_description += "AgcClippingLevelExperiment;";
2048 }
Sam Zackrisson701bd172020-02-18 14:50:28 +01002049 if (!!submodules_.capture_post_processor) {
2050 experiments_description += "CapturePostProcessor;";
2051 }
2052 if (!!submodules_.render_pre_processor) {
2053 experiments_description += "RenderPreProcessor;";
2054 }
Gustaf Ullbergce045ac2017-10-16 13:49:04 +02002055 if (capture_nonlocked_.echo_controller_enabled) {
2056 experiments_description += "EchoController;";
aleloi868f32f2017-05-23 07:20:05 -07002057 }
Alessio Bazzica270f7b52017-10-13 11:05:17 +02002058 if (config_.gain_controller2.enabled) {
2059 experiments_description += "GainController2;";
2060 }
aleloi868f32f2017-05-23 07:20:05 -07002061
2062 InternalAPMConfig apm_config;
2063
Per Åhgren200feba2019-03-06 04:16:46 +01002064 apm_config.aec_enabled = config_.echo_canceller.enabled;
Per Åhgren62ea0aa2019-12-09 10:18:44 +01002065 apm_config.aec_delay_agnostic_enabled = false;
2066 apm_config.aec_extended_filter_enabled = false;
2067 apm_config.aec_suppression_level = 0;
aleloi868f32f2017-05-23 07:20:05 -07002068
saza1d600522019-10-18 13:29:43 +02002069 apm_config.aecm_enabled = !!submodules_.echo_control_mobile;
aleloi868f32f2017-05-23 07:20:05 -07002070 apm_config.aecm_comfort_noise_enabled =
saza1d600522019-10-18 13:29:43 +02002071 submodules_.echo_control_mobile &&
2072 submodules_.echo_control_mobile->is_comfort_noise_enabled();
Per Åhgrenb6e24d72019-04-29 12:14:50 +02002073 apm_config.aecm_routing_mode =
saza1d600522019-10-18 13:29:43 +02002074 submodules_.echo_control_mobile
2075 ? static_cast<int>(submodules_.echo_control_mobile->routing_mode())
Per Åhgrenb6e24d72019-04-29 12:14:50 +02002076 : 0;
aleloi868f32f2017-05-23 07:20:05 -07002077
Per Åhgren0695df12020-01-13 14:43:13 +01002078 apm_config.agc_enabled = !!submodules_.gain_control;
2079
2080 apm_config.agc_mode = submodules_.gain_control
2081 ? static_cast<int>(submodules_.gain_control->mode())
2082 : GainControl::kAdaptiveAnalog;
aleloi868f32f2017-05-23 07:20:05 -07002083 apm_config.agc_limiter_enabled =
Per Åhgren0695df12020-01-13 14:43:13 +01002084 submodules_.gain_control ? submodules_.gain_control->is_limiter_enabled()
2085 : false;
Per Åhgren0e3198e2019-11-18 08:52:22 +01002086 apm_config.noise_robust_agc_enabled = !!submodules_.agc_manager;
aleloi868f32f2017-05-23 07:20:05 -07002087
2088 apm_config.hpf_enabled = config_.high_pass_filter.enabled;
2089
saza0bad15f2019-10-16 11:46:11 +02002090 apm_config.ns_enabled = config_.noise_suppression.enabled;
2091 apm_config.ns_level = static_cast<int>(config_.noise_suppression.level);
aleloi868f32f2017-05-23 07:20:05 -07002092
2093 apm_config.transient_suppression_enabled =
Per Åhgrenc0734712020-01-02 15:15:36 +01002094 config_.transient_suppression.enabled;
aleloi868f32f2017-05-23 07:20:05 -07002095 apm_config.experiments_description = experiments_description;
Alex Loiko5feb30e2018-04-16 13:52:32 +02002096 apm_config.pre_amplifier_enabled = config_.pre_amplifier.enabled;
2097 apm_config.pre_amplifier_fixed_gain_factor =
2098 config_.pre_amplifier.fixed_gain_factor;
aleloi868f32f2017-05-23 07:20:05 -07002099
2100 if (!forced && apm_config == apm_config_for_aec_dump_) {
2101 return;
2102 }
2103 aec_dump_->WriteConfig(apm_config);
2104 apm_config_for_aec_dump_ = apm_config;
2105}
2106
2107void AudioProcessingImpl::RecordUnprocessedCaptureStream(
2108 const float* const* src) {
2109 RTC_DCHECK(aec_dump_);
2110 WriteAecDumpConfigMessage(false);
2111
2112 const size_t channel_size = formats_.api_format.input_stream().num_frames();
2113 const size_t num_channels = formats_.api_format.input_stream().num_channels();
2114 aec_dump_->AddCaptureStreamInput(
Alex Loikoe36e8bb2018-02-16 11:54:07 +01002115 AudioFrameView<const float>(src, num_channels, channel_size));
aleloi868f32f2017-05-23 07:20:05 -07002116 RecordAudioProcessingState();
2117}
2118
2119void AudioProcessingImpl::RecordUnprocessedCaptureStream(
Per Åhgren645f24c2020-03-16 12:06:02 +01002120 const int16_t* const data,
2121 const StreamConfig& config) {
aleloi868f32f2017-05-23 07:20:05 -07002122 RTC_DCHECK(aec_dump_);
2123 WriteAecDumpConfigMessage(false);
2124
Per Åhgren645f24c2020-03-16 12:06:02 +01002125 aec_dump_->AddCaptureStreamInput(data, config.num_channels(),
2126 config.num_frames());
aleloi868f32f2017-05-23 07:20:05 -07002127 RecordAudioProcessingState();
2128}
2129
2130void AudioProcessingImpl::RecordProcessedCaptureStream(
2131 const float* const* processed_capture_stream) {
2132 RTC_DCHECK(aec_dump_);
2133
2134 const size_t channel_size = formats_.api_format.output_stream().num_frames();
2135 const size_t num_channels =
2136 formats_.api_format.output_stream().num_channels();
Alex Loikoe36e8bb2018-02-16 11:54:07 +01002137 aec_dump_->AddCaptureStreamOutput(AudioFrameView<const float>(
2138 processed_capture_stream, num_channels, channel_size));
aleloi868f32f2017-05-23 07:20:05 -07002139 aec_dump_->WriteCaptureStreamMessage();
2140}
2141
2142void AudioProcessingImpl::RecordProcessedCaptureStream(
Per Åhgren645f24c2020-03-16 12:06:02 +01002143 const int16_t* const data,
2144 const StreamConfig& config) {
aleloi868f32f2017-05-23 07:20:05 -07002145 RTC_DCHECK(aec_dump_);
2146
Per Åhgren645f24c2020-03-16 12:06:02 +01002147 aec_dump_->AddCaptureStreamOutput(data, config.num_channels(),
Per Åhgren088329f2020-03-18 21:59:52 +01002148 config.num_frames());
aleloi868f32f2017-05-23 07:20:05 -07002149 aec_dump_->WriteCaptureStreamMessage();
2150}
2151
2152void AudioProcessingImpl::RecordAudioProcessingState() {
2153 RTC_DCHECK(aec_dump_);
2154 AecDump::AudioProcessingState audio_proc_state;
2155 audio_proc_state.delay = capture_nonlocked_.stream_delay_ms;
Per Åhgren62ea0aa2019-12-09 10:18:44 +01002156 audio_proc_state.drift = 0;
Markus Handell02ba1d22020-05-14 14:31:18 +02002157 audio_proc_state.level = recommended_stream_analog_level_locked();
aleloi868f32f2017-05-23 07:20:05 -07002158 audio_proc_state.keypress = capture_.key_pressed;
2159 aec_dump_->AddAudioProcessingState(audio_proc_state);
2160}
2161
Per Åhgrenc0734712020-01-02 15:15:36 +01002162AudioProcessingImpl::ApmCaptureState::ApmCaptureState()
Sam Zackrisson2d31aea2020-01-17 10:55:09 +01002163 : was_stream_delay_set(false),
Per Åhgren0a144a72021-02-09 08:47:51 +01002164 capture_output_used(true),
Per Åhgren55bc0772021-03-12 14:18:36 +00002165 capture_output_used_last_frame(true),
kwiberg83ffe452016-08-29 14:46:07 -07002166 key_pressed(false),
peahde65ddc2016-09-16 15:02:15 -07002167 capture_processing_format(kSampleRate16kHz),
peah67995532017-04-10 14:12:41 -07002168 split_rate(kSampleRate16kHz),
Per Åhgren88cf0502018-07-16 17:08:41 +02002169 echo_path_gain_change(false),
Per Åhgrend2650d12018-10-02 17:00:59 +02002170 prev_analog_mic_level(-1),
Per Åhgrendb5d7282021-03-15 16:31:04 +00002171 prev_pre_adjustment_gain(-1.f),
Fredrik Hernqvistca362852019-05-10 15:50:02 +02002172 playout_volume(-1),
2173 prev_playout_volume(-1) {}
kwiberg83ffe452016-08-29 14:46:07 -07002174
2175AudioProcessingImpl::ApmCaptureState::~ApmCaptureState() = default;
2176
2177AudioProcessingImpl::ApmRenderState::ApmRenderState() = default;
2178
2179AudioProcessingImpl::ApmRenderState::~ApmRenderState() = default;
2180
Per Åhgrencf4c8722019-12-30 14:32:14 +01002181AudioProcessingImpl::ApmStatsReporter::ApmStatsReporter()
2182 : stats_message_queue_(1) {}
2183
2184AudioProcessingImpl::ApmStatsReporter::~ApmStatsReporter() = default;
2185
2186AudioProcessingStats AudioProcessingImpl::ApmStatsReporter::GetStatistics() {
Markus Handell0df0fae2020-07-07 15:53:34 +02002187 MutexLock lock_stats(&mutex_stats_);
Per Åhgrencf4c8722019-12-30 14:32:14 +01002188 bool new_stats_available = stats_message_queue_.Remove(&cached_stats_);
2189 // If the message queue is full, return the cached stats.
2190 static_cast<void>(new_stats_available);
2191
2192 return cached_stats_;
2193}
2194
2195void AudioProcessingImpl::ApmStatsReporter::UpdateStatistics(
2196 const AudioProcessingStats& new_stats) {
2197 AudioProcessingStats stats_to_queue = new_stats;
2198 bool stats_message_passed = stats_message_queue_.Insert(&stats_to_queue);
2199 // If the message queue is full, discard the new stats.
2200 static_cast<void>(stats_message_passed);
2201}
2202
niklase@google.com470e71d2011-07-07 08:21:25 +00002203} // namespace webrtc