blob: 6a87891164d1a29c58460427d1e1ef848764fcd9 [file] [log] [blame]
niklase@google.com470e71d2011-07-07 08:21:25 +00001/*
andrew@webrtc.org40654032012-01-30 20:51:15 +00002 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
niklase@google.com470e71d2011-07-07 08:21:25 +00003 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020011#include "modules/audio_processing/audio_processing_impl.h"
niklase@google.com470e71d2011-07-07 08:21:25 +000012
Michael Graczyk86c6d332015-07-23 11:41:39 -070013#include <algorithm>
Yves Gerey988cc082018-10-23 12:03:01 +020014#include <cstdint>
Mirko Bonadei317a1f02019-09-17 17:06:18 +020015#include <memory>
alessiob3ec96df2017-05-22 06:57:06 -070016#include <string>
Yves Gerey988cc082018-10-23 12:03:01 +020017#include <type_traits>
18#include <utility>
niklase@google.com470e71d2011-07-07 08:21:25 +000019
Yves Gerey988cc082018-10-23 12:03:01 +020020#include "absl/types/optional.h"
21#include "api/array_view.h"
Per Åhgren645f24c2020-03-16 12:06:02 +010022#include "api/audio/audio_frame.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020023#include "common_audio/audio_converter.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020024#include "common_audio/include/audio_util.h"
Alex Loikob5c9a792018-04-16 16:31:22 +020025#include "modules/audio_processing/agc2/gain_applier.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020026#include "modules/audio_processing/audio_buffer.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020027#include "modules/audio_processing/common.h"
Yves Gerey988cc082018-10-23 12:03:01 +020028#include "modules/audio_processing/include/audio_frame_view.h"
Per Åhgren13735822018-02-12 21:42:56 +010029#include "modules/audio_processing/logging/apm_data_dumper.h"
Sam Zackrissonb37e59d2020-04-27 08:39:33 +020030#include "modules/audio_processing/optionally_built_submodule_creators.h"
Steve Anton10542f22019-01-11 09:11:00 -080031#include "rtc_base/atomic_ops.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020032#include "rtc_base/checks.h"
Steve Anton10542f22019-01-11 09:11:00 -080033#include "rtc_base/constructor_magic.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020034#include "rtc_base/logging.h"
Steve Anton10542f22019-01-11 09:11:00 -080035#include "rtc_base/ref_counted_object.h"
36#include "rtc_base/time_utils.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020037#include "rtc_base/trace_event.h"
Sam Zackrissonfeee1e42019-09-20 07:50:35 +020038#include "system_wrappers/include/field_trial.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020039#include "system_wrappers/include/metrics.h"
andrew@webrtc.org7bf26462011-12-03 00:03:31 +000040
Michael Graczyk86c6d332015-07-23 11:41:39 -070041#define RETURN_ON_ERR(expr) \
42 do { \
43 int err = (expr); \
44 if (err != kNoError) { \
45 return err; \
46 } \
andrew@webrtc.org60730cf2014-01-07 17:45:09 +000047 } while (0)
48
niklase@google.com470e71d2011-07-07 08:21:25 +000049namespace webrtc {
aluebsdf6416a2016-03-16 18:26:35 -070050
Alex Loiko73ec0192018-05-15 10:52:28 +020051constexpr int kRuntimeSettingQueueSize = 100;
aluebsdf6416a2016-03-16 18:26:35 -070052
Michael Graczyk86c6d332015-07-23 11:41:39 -070053namespace {
54
55static bool LayoutHasKeyboard(AudioProcessing::ChannelLayout layout) {
56 switch (layout) {
57 case AudioProcessing::kMono:
58 case AudioProcessing::kStereo:
59 return false;
60 case AudioProcessing::kMonoAndKeyboard:
61 case AudioProcessing::kStereoAndKeyboard:
62 return true;
63 }
64
kwiberg9e2be5f2016-09-14 05:23:22 -070065 RTC_NOTREACHED();
Michael Graczyk86c6d332015-07-23 11:41:39 -070066 return false;
67}
aluebsdf6416a2016-03-16 18:26:35 -070068
peah2ace3f92016-09-10 04:42:27 -070069bool SampleRateSupportsMultiBand(int sample_rate_hz) {
aluebsdf6416a2016-03-16 18:26:35 -070070 return sample_rate_hz == AudioProcessing::kSampleRate32kHz ||
71 sample_rate_hz == AudioProcessing::kSampleRate48kHz;
72}
73
Per Åhgrenc0424252019-12-10 13:04:15 +010074// Checks whether the high-pass filter should be done in the full-band.
75bool EnforceSplitBandHpf() {
76 return field_trial::IsEnabled("WebRTC-FullBandHpfKillSwitch");
77}
78
Per Åhgrenb2b58d82019-12-02 14:59:40 +010079// Checks whether AEC3 should be allowed to decide what the default
80// configuration should be based on the render and capture channel configuration
81// at hand.
82bool UseSetupSpecificDefaultAec3Congfig() {
83 return !field_trial::IsEnabled(
84 "WebRTC-Aec3SetupSpecificDefaultConfigDefaultsKillSwitch");
85}
86
Per Åhgrenc8626b62019-08-23 15:49:51 +020087// Identify the native processing rate that best handles a sample rate.
Per Åhgrenfcbe4072019-09-15 00:27:58 +020088int SuitableProcessRate(int minimum_rate,
89 int max_splitting_rate,
90 bool band_splitting_required) {
Per Åhgrenc8626b62019-08-23 15:49:51 +020091 const int uppermost_native_rate =
Per Åhgrenfcbe4072019-09-15 00:27:58 +020092 band_splitting_required ? max_splitting_rate : 48000;
Per Åhgrenc8626b62019-08-23 15:49:51 +020093 for (auto rate : {16000, 32000, 48000}) {
peah2ace3f92016-09-10 04:42:27 -070094 if (rate >= uppermost_native_rate) {
95 return uppermost_native_rate;
96 }
97 if (rate >= minimum_rate) {
aluebsdf6416a2016-03-16 18:26:35 -070098 return rate;
99 }
100 }
peah2ace3f92016-09-10 04:42:27 -0700101 RTC_NOTREACHED();
102 return uppermost_native_rate;
aluebsdf6416a2016-03-16 18:26:35 -0700103}
104
Sam Zackrissonf0d1c032019-03-27 13:28:08 +0100105GainControl::Mode Agc1ConfigModeToInterfaceMode(
106 AudioProcessing::Config::GainController1::Mode mode) {
107 using Agc1Config = AudioProcessing::Config::GainController1;
108 switch (mode) {
109 case Agc1Config::kAdaptiveAnalog:
110 return GainControl::kAdaptiveAnalog;
111 case Agc1Config::kAdaptiveDigital:
112 return GainControl::kAdaptiveDigital;
113 case Agc1Config::kFixedDigital:
114 return GainControl::kFixedDigital;
115 }
116}
117
peah9e6a2902017-05-15 07:19:21 -0700118// Maximum lengths that frame of samples being passed from the render side to
119// the capture side can have (does not apply to AEC3).
120static const size_t kMaxAllowedValuesOfSamplesPerBand = 160;
121static const size_t kMaxAllowedValuesOfSamplesPerFrame = 480;
122
peah764e3642016-10-22 05:04:30 -0700123// Maximum number of frames to buffer in the render queue.
124// TODO(peah): Decrease this once we properly handle hugely unbalanced
125// reverse and forward call numbers.
126static const size_t kMaxNumFramesToBuffer = 100;
Michael Graczyk86c6d332015-07-23 11:41:39 -0700127} // namespace
andrew@webrtc.org60730cf2014-01-07 17:45:09 +0000128
129// Throughout webrtc, it's assumed that success is represented by zero.
kwiberg@webrtc.org2ebfac52015-01-14 10:51:54 +0000130static_assert(AudioProcessing::kNoError == 0, "kNoError must be zero");
andrew@webrtc.org60730cf2014-01-07 17:45:09 +0000131
saza1d600522019-10-18 13:29:43 +0200132AudioProcessingImpl::SubmoduleStates::SubmoduleStates(
Alex Loiko5825aa62017-12-18 16:02:40 +0100133 bool capture_post_processor_enabled,
Valeriia Nemychnikovaf06eb572018-08-29 10:37:09 +0200134 bool render_pre_processor_enabled,
135 bool capture_analyzer_enabled)
Alex Loiko5825aa62017-12-18 16:02:40 +0100136 : capture_post_processor_enabled_(capture_post_processor_enabled),
Valeriia Nemychnikovaf06eb572018-08-29 10:37:09 +0200137 render_pre_processor_enabled_(render_pre_processor_enabled),
138 capture_analyzer_enabled_(capture_analyzer_enabled) {}
peah2ace3f92016-09-10 04:42:27 -0700139
saza1d600522019-10-18 13:29:43 +0200140bool AudioProcessingImpl::SubmoduleStates::Update(
Sam Zackrissoncb1b5562018-09-28 14:15:09 +0200141 bool high_pass_filter_enabled,
peah2ace3f92016-09-10 04:42:27 -0700142 bool mobile_echo_controller_enabled,
ivoc9f4a4a02016-10-28 05:39:16 -0700143 bool residual_echo_detector_enabled,
peah2ace3f92016-09-10 04:42:27 -0700144 bool noise_suppressor_enabled,
peah2ace3f92016-09-10 04:42:27 -0700145 bool adaptive_gain_controller_enabled,
alessiob3ec96df2017-05-22 06:57:06 -0700146 bool gain_controller2_enabled,
Alex Loikob5c9a792018-04-16 16:31:22 +0200147 bool pre_amplifier_enabled,
Gustaf Ullbergce045ac2017-10-16 13:49:04 +0200148 bool echo_controller_enabled,
Sam Zackrisson0824c6f2019-10-07 14:03:56 +0200149 bool voice_detector_enabled,
peah2ace3f92016-09-10 04:42:27 -0700150 bool transient_suppressor_enabled) {
151 bool changed = false;
Sam Zackrissoncb1b5562018-09-28 14:15:09 +0200152 changed |= (high_pass_filter_enabled != high_pass_filter_enabled_);
peah2ace3f92016-09-10 04:42:27 -0700153 changed |=
154 (mobile_echo_controller_enabled != mobile_echo_controller_enabled_);
ivoc9f4a4a02016-10-28 05:39:16 -0700155 changed |=
156 (residual_echo_detector_enabled != residual_echo_detector_enabled_);
peah2ace3f92016-09-10 04:42:27 -0700157 changed |= (noise_suppressor_enabled != noise_suppressor_enabled_);
158 changed |=
peah2ace3f92016-09-10 04:42:27 -0700159 (adaptive_gain_controller_enabled != adaptive_gain_controller_enabled_);
Per Åhgren6ee75fd2019-04-26 11:33:37 +0200160 changed |= (gain_controller2_enabled != gain_controller2_enabled_);
Alex Loikob5c9a792018-04-16 16:31:22 +0200161 changed |= (pre_amplifier_enabled_ != pre_amplifier_enabled);
Gustaf Ullbergce045ac2017-10-16 13:49:04 +0200162 changed |= (echo_controller_enabled != echo_controller_enabled_);
Sam Zackrisson0824c6f2019-10-07 14:03:56 +0200163 changed |= (voice_detector_enabled != voice_detector_enabled_);
peah2ace3f92016-09-10 04:42:27 -0700164 changed |= (transient_suppressor_enabled != transient_suppressor_enabled_);
165 if (changed) {
Sam Zackrissoncb1b5562018-09-28 14:15:09 +0200166 high_pass_filter_enabled_ = high_pass_filter_enabled;
peah2ace3f92016-09-10 04:42:27 -0700167 mobile_echo_controller_enabled_ = mobile_echo_controller_enabled;
ivoc9f4a4a02016-10-28 05:39:16 -0700168 residual_echo_detector_enabled_ = residual_echo_detector_enabled;
peah2ace3f92016-09-10 04:42:27 -0700169 noise_suppressor_enabled_ = noise_suppressor_enabled;
peah2ace3f92016-09-10 04:42:27 -0700170 adaptive_gain_controller_enabled_ = adaptive_gain_controller_enabled;
alessiob3ec96df2017-05-22 06:57:06 -0700171 gain_controller2_enabled_ = gain_controller2_enabled;
Alex Loikob5c9a792018-04-16 16:31:22 +0200172 pre_amplifier_enabled_ = pre_amplifier_enabled;
Gustaf Ullbergce045ac2017-10-16 13:49:04 +0200173 echo_controller_enabled_ = echo_controller_enabled;
Sam Zackrisson0824c6f2019-10-07 14:03:56 +0200174 voice_detector_enabled_ = voice_detector_enabled;
peah2ace3f92016-09-10 04:42:27 -0700175 transient_suppressor_enabled_ = transient_suppressor_enabled;
176 }
177
178 changed |= first_update_;
179 first_update_ = false;
180 return changed;
181}
182
saza1d600522019-10-18 13:29:43 +0200183bool AudioProcessingImpl::SubmoduleStates::CaptureMultiBandSubModulesActive()
peah2ace3f92016-09-10 04:42:27 -0700184 const {
Gustaf Ullberg8675eee2019-10-09 13:34:36 +0200185 return CaptureMultiBandProcessingPresent() || voice_detector_enabled_;
peah2ace3f92016-09-10 04:42:27 -0700186}
187
saza1d600522019-10-18 13:29:43 +0200188bool AudioProcessingImpl::SubmoduleStates::CaptureMultiBandProcessingPresent()
189 const {
Gustaf Ullberg8675eee2019-10-09 13:34:36 +0200190 // If echo controller is present, assume it performs active processing.
191 return CaptureMultiBandProcessingActive(/*ec_processing_active=*/true);
192}
193
saza1d600522019-10-18 13:29:43 +0200194bool AudioProcessingImpl::SubmoduleStates::CaptureMultiBandProcessingActive(
Gustaf Ullberg8675eee2019-10-09 13:34:36 +0200195 bool ec_processing_active) const {
Per Åhgren62ea0aa2019-12-09 10:18:44 +0100196 return high_pass_filter_enabled_ || mobile_echo_controller_enabled_ ||
197 noise_suppressor_enabled_ || adaptive_gain_controller_enabled_ ||
Gustaf Ullberg8675eee2019-10-09 13:34:36 +0200198 (echo_controller_enabled_ && ec_processing_active);
peah2ace3f92016-09-10 04:42:27 -0700199}
200
saza1d600522019-10-18 13:29:43 +0200201bool AudioProcessingImpl::SubmoduleStates::CaptureFullBandProcessingActive()
peah23ac8b42017-05-23 05:33:56 -0700202 const {
Alex Loikob5c9a792018-04-16 16:31:22 +0200203 return gain_controller2_enabled_ || capture_post_processor_enabled_ ||
204 pre_amplifier_enabled_;
peah23ac8b42017-05-23 05:33:56 -0700205}
206
saza1d600522019-10-18 13:29:43 +0200207bool AudioProcessingImpl::SubmoduleStates::CaptureAnalyzerActive() const {
Valeriia Nemychnikovaf06eb572018-08-29 10:37:09 +0200208 return capture_analyzer_enabled_;
209}
210
saza1d600522019-10-18 13:29:43 +0200211bool AudioProcessingImpl::SubmoduleStates::RenderMultiBandSubModulesActive()
peah2ace3f92016-09-10 04:42:27 -0700212 const {
Per Åhgren62ea0aa2019-12-09 10:18:44 +0100213 return RenderMultiBandProcessingActive() || mobile_echo_controller_enabled_ ||
214 adaptive_gain_controller_enabled_ || echo_controller_enabled_;
peah2ace3f92016-09-10 04:42:27 -0700215}
216
saza1d600522019-10-18 13:29:43 +0200217bool AudioProcessingImpl::SubmoduleStates::RenderFullBandProcessingActive()
Alex Loiko5825aa62017-12-18 16:02:40 +0100218 const {
219 return render_pre_processor_enabled_;
220}
221
saza1d600522019-10-18 13:29:43 +0200222bool AudioProcessingImpl::SubmoduleStates::RenderMultiBandProcessingActive()
peah2ace3f92016-09-10 04:42:27 -0700223 const {
peah2ace3f92016-09-10 04:42:27 -0700224 return false;
peah2ace3f92016-09-10 04:42:27 -0700225}
226
saza1d600522019-10-18 13:29:43 +0200227bool AudioProcessingImpl::SubmoduleStates::HighPassFilteringRequired() const {
Per Åhgren62ea0aa2019-12-09 10:18:44 +0100228 return high_pass_filter_enabled_ || mobile_echo_controller_enabled_ ||
229 noise_suppressor_enabled_;
Sam Zackrissoncb1b5562018-09-28 14:15:09 +0200230}
231
peah88ac8532016-09-12 16:47:25 -0700232AudioProcessingImpl::AudioProcessingImpl(const webrtc::Config& config)
Sam Zackrissonfeee1e42019-09-20 07:50:35 +0200233 : AudioProcessingImpl(config,
234 /*capture_post_processor=*/nullptr,
235 /*render_pre_processor=*/nullptr,
236 /*echo_control_factory=*/nullptr,
237 /*echo_detector=*/nullptr,
238 /*capture_analyzer=*/nullptr) {}
aluebs@webrtc.orgd82f55d2015-01-15 18:07:21 +0000239
Per Åhgren13735822018-02-12 21:42:56 +0100240int AudioProcessingImpl::instance_count_ = 0;
241
Sam Zackrisson0beac582017-09-25 12:04:02 +0200242AudioProcessingImpl::AudioProcessingImpl(
243 const webrtc::Config& config,
Alex Loiko5825aa62017-12-18 16:02:40 +0100244 std::unique_ptr<CustomProcessing> capture_post_processor,
245 std::unique_ptr<CustomProcessing> render_pre_processor,
Gustaf Ullberg002ef282017-10-12 15:13:17 +0200246 std::unique_ptr<EchoControlFactory> echo_control_factory,
Valeriia Nemychnikovaf06eb572018-08-29 10:37:09 +0200247 rtc::scoped_refptr<EchoDetector> echo_detector,
248 std::unique_ptr<CustomAudioAnalyzer> capture_analyzer)
Per Åhgren13735822018-02-12 21:42:56 +0100249 : data_dumper_(
250 new ApmDataDumper(rtc::AtomicOps::Increment(&instance_count_))),
Per Åhgrenb2b58d82019-12-02 14:59:40 +0100251 use_setup_specific_default_aec3_config_(
252 UseSetupSpecificDefaultAec3Congfig()),
Alex Loiko73ec0192018-05-15 10:52:28 +0200253 capture_runtime_settings_(kRuntimeSettingQueueSize),
254 render_runtime_settings_(kRuntimeSettingQueueSize),
255 capture_runtime_settings_enqueuer_(&capture_runtime_settings_),
256 render_runtime_settings_enqueuer_(&render_runtime_settings_),
Gustaf Ullberg002ef282017-10-12 15:13:17 +0200257 echo_control_factory_(std::move(echo_control_factory)),
Valeriia Nemychnikovaf06eb572018-08-29 10:37:09 +0200258 submodule_states_(!!capture_post_processor,
259 !!render_pre_processor,
260 !!capture_analyzer),
saza1d600522019-10-18 13:29:43 +0200261 submodules_(std::move(capture_post_processor),
262 std::move(render_pre_processor),
263 std::move(echo_detector),
Per Åhgren3daedb62019-11-22 12:11:40 +0100264 std::move(capture_analyzer)),
Per Åhgren0695df12020-01-13 14:43:13 +0100265 constants_(!field_trial::IsEnabled(
Sam Zackrissonfeee1e42019-09-20 07:50:35 +0200266 "WebRTC-ApmExperimentalMultiChannelRenderKillSwitch"),
267 !field_trial::IsEnabled(
Per Åhgrenc0424252019-12-10 13:04:15 +0100268 "WebRTC-ApmExperimentalMultiChannelCaptureKillSwitch"),
269 EnforceSplitBandHpf()),
Alessio Bazzicacc22f512018-08-30 13:01:34 +0200270 capture_nonlocked_() {
Sam Zackrisson72cc71c2019-10-21 12:54:02 +0200271 RTC_LOG(LS_INFO) << "Injected APM submodules:"
Jonas Olssonb2b20312020-01-14 12:11:31 +0100272 "\nEcho control factory: "
273 << !!echo_control_factory_
Sam Zackrisson72cc71c2019-10-21 12:54:02 +0200274 << "\nEcho detector: " << !!submodules_.echo_detector
275 << "\nCapture analyzer: " << !!submodules_.capture_analyzer
276 << "\nCapture post processor: "
277 << !!submodules_.capture_post_processor
278 << "\nRender pre processor: "
279 << !!submodules_.render_pre_processor;
280
Sam Zackrisson421c8592019-02-11 13:39:46 +0100281 // Mark Echo Controller enabled if a factory is injected.
282 capture_nonlocked_.echo_controller_enabled =
283 static_cast<bool>(echo_control_factory_);
niklase@google.com470e71d2011-07-07 08:21:25 +0000284
Sam Zackrisson421c8592019-02-11 13:39:46 +0100285 // If no echo detector is injected, use the ResidualEchoDetector.
saza1d600522019-10-18 13:29:43 +0200286 if (!submodules_.echo_detector) {
287 submodules_.echo_detector =
Sam Zackrisson421c8592019-02-11 13:39:46 +0100288 new rtc::RefCountedObject<ResidualEchoDetector>();
peahdf3efa82015-11-28 12:35:15 -0800289 }
pbos@webrtc.org788acd12014-12-15 09:41:24 +0000290
Per Åhgren0695df12020-01-13 14:43:13 +0100291#if !(defined(WEBRTC_ANDROID) || defined(WEBRTC_IOS))
Per Åhgrenc0734712020-01-02 15:15:36 +0100292 // TODO(webrtc:5298): Remove once the use of ExperimentalNs has been
293 // deprecated.
Per Åhgrenc0734712020-01-02 15:15:36 +0100294 config_.transient_suppression.enabled = config.Get<ExperimentalNs>().enabled;
Per Åhgren0695df12020-01-13 14:43:13 +0100295
296 // TODO(webrtc:5298): Remove once the use of ExperimentalAgc has been
297 // deprecated.
298 config_.gain_controller1.analog_gain_controller.enabled =
299 config.Get<ExperimentalAgc>().enabled;
300 config_.gain_controller1.analog_gain_controller.startup_min_volume =
301 config.Get<ExperimentalAgc>().startup_min_volume;
302 config_.gain_controller1.analog_gain_controller.clipped_level_min =
303 config.Get<ExperimentalAgc>().clipped_level_min;
304 config_.gain_controller1.analog_gain_controller.enable_agc2_level_estimator =
305 config.Get<ExperimentalAgc>().enabled_agc2_level_estimator;
306 config_.gain_controller1.analog_gain_controller.enable_digital_adaptive =
307 !config.Get<ExperimentalAgc>().digital_adaptive_disabled;
Per Åhgrenc0734712020-01-02 15:15:36 +0100308#endif
niklase@google.com470e71d2011-07-07 08:21:25 +0000309}
310
Per Åhgren0e3198e2019-11-18 08:52:22 +0100311AudioProcessingImpl::~AudioProcessingImpl() = default;
niklase@google.com470e71d2011-07-07 08:21:25 +0000312
niklase@google.com470e71d2011-07-07 08:21:25 +0000313int AudioProcessingImpl::Initialize() {
peahdf3efa82015-11-28 12:35:15 -0800314 // Run in a single-threaded manner during initialization.
315 rtc::CritScope cs_render(&crit_render_);
316 rtc::CritScope cs_capture(&crit_capture_);
niklase@google.com470e71d2011-07-07 08:21:25 +0000317 return InitializeLocked();
318}
319
peahde65ddc2016-09-16 15:02:15 -0700320int AudioProcessingImpl::Initialize(int capture_input_sample_rate_hz,
321 int capture_output_sample_rate_hz,
322 int render_input_sample_rate_hz,
323 ChannelLayout capture_input_layout,
324 ChannelLayout capture_output_layout,
325 ChannelLayout render_input_layout) {
Michael Graczyk86c6d332015-07-23 11:41:39 -0700326 const ProcessingConfig processing_config = {
peahde65ddc2016-09-16 15:02:15 -0700327 {{capture_input_sample_rate_hz, ChannelsFromLayout(capture_input_layout),
328 LayoutHasKeyboard(capture_input_layout)},
329 {capture_output_sample_rate_hz,
330 ChannelsFromLayout(capture_output_layout),
331 LayoutHasKeyboard(capture_output_layout)},
332 {render_input_sample_rate_hz, ChannelsFromLayout(render_input_layout),
333 LayoutHasKeyboard(render_input_layout)},
334 {render_input_sample_rate_hz, ChannelsFromLayout(render_input_layout),
335 LayoutHasKeyboard(render_input_layout)}}};
Michael Graczyk86c6d332015-07-23 11:41:39 -0700336
337 return Initialize(processing_config);
338}
339
340int AudioProcessingImpl::Initialize(const ProcessingConfig& processing_config) {
peahdf3efa82015-11-28 12:35:15 -0800341 // Run in a single-threaded manner during initialization.
342 rtc::CritScope cs_render(&crit_render_);
343 rtc::CritScope cs_capture(&crit_capture_);
Michael Graczyk86c6d332015-07-23 11:41:39 -0700344 return InitializeLocked(processing_config);
andrew@webrtc.orga8b97372014-03-10 22:26:12 +0000345}
346
peahdf3efa82015-11-28 12:35:15 -0800347int AudioProcessingImpl::MaybeInitializeRender(
peah81b9bfe2015-11-27 02:47:28 -0800348 const ProcessingConfig& processing_config) {
peahdf3efa82015-11-28 12:35:15 -0800349 // Called from both threads. Thread check is therefore not possible.
Oskar Sundbom4b276482019-05-23 14:28:00 +0200350 if (processing_config == formats_.api_format) {
peah192164e2015-11-17 02:16:45 -0800351 return kNoError;
352 }
peahdf3efa82015-11-28 12:35:15 -0800353
354 rtc::CritScope cs_capture(&crit_capture_);
peah192164e2015-11-17 02:16:45 -0800355 return InitializeLocked(processing_config);
356}
357
niklase@google.com470e71d2011-07-07 08:21:25 +0000358int AudioProcessingImpl::InitializeLocked() {
Per Åhgren4bdced52017-06-27 16:00:38 +0200359 UpdateActiveSubmoduleStates();
360
Per Åhgrend47941e2019-08-22 11:51:13 +0200361 const int render_audiobuffer_sample_rate_hz =
peahdf3efa82015-11-28 12:35:15 -0800362 formats_.api_format.reverse_output_stream().num_frames() == 0
Per Åhgrend47941e2019-08-22 11:51:13 +0200363 ? formats_.render_processing_format.sample_rate_hz()
364 : formats_.api_format.reverse_output_stream().sample_rate_hz();
peahdf3efa82015-11-28 12:35:15 -0800365 if (formats_.api_format.reverse_input_stream().num_channels() > 0) {
366 render_.render_audio.reset(new AudioBuffer(
Per Åhgrend47941e2019-08-22 11:51:13 +0200367 formats_.api_format.reverse_input_stream().sample_rate_hz(),
peahdf3efa82015-11-28 12:35:15 -0800368 formats_.api_format.reverse_input_stream().num_channels(),
Per Åhgrend47941e2019-08-22 11:51:13 +0200369 formats_.render_processing_format.sample_rate_hz(),
peahde65ddc2016-09-16 15:02:15 -0700370 formats_.render_processing_format.num_channels(),
Per Åhgrend47941e2019-08-22 11:51:13 +0200371 render_audiobuffer_sample_rate_hz,
372 formats_.render_processing_format.num_channels()));
peah2ace3f92016-09-10 04:42:27 -0700373 if (formats_.api_format.reverse_input_stream() !=
374 formats_.api_format.reverse_output_stream()) {
kwibergc2b785d2016-02-24 05:22:32 -0800375 render_.render_converter = AudioConverter::Create(
peahdf3efa82015-11-28 12:35:15 -0800376 formats_.api_format.reverse_input_stream().num_channels(),
377 formats_.api_format.reverse_input_stream().num_frames(),
378 formats_.api_format.reverse_output_stream().num_channels(),
kwibergc2b785d2016-02-24 05:22:32 -0800379 formats_.api_format.reverse_output_stream().num_frames());
ekmeyerson60d9b332015-08-14 10:35:55 -0700380 } else {
peahdf3efa82015-11-28 12:35:15 -0800381 render_.render_converter.reset(nullptr);
ekmeyerson60d9b332015-08-14 10:35:55 -0700382 }
Michael Graczyk86c6d332015-07-23 11:41:39 -0700383 } else {
peahdf3efa82015-11-28 12:35:15 -0800384 render_.render_audio.reset(nullptr);
385 render_.render_converter.reset(nullptr);
Michael Graczyk86c6d332015-07-23 11:41:39 -0700386 }
peahce4d9152017-05-19 01:28:05 -0700387
Per Åhgrend47941e2019-08-22 11:51:13 +0200388 capture_.capture_audio.reset(new AudioBuffer(
389 formats_.api_format.input_stream().sample_rate_hz(),
390 formats_.api_format.input_stream().num_channels(),
391 capture_nonlocked_.capture_processing_format.sample_rate_hz(),
392 formats_.api_format.output_stream().num_channels(),
393 formats_.api_format.output_stream().sample_rate_hz(),
394 formats_.api_format.output_stream().num_channels()));
niklase@google.com470e71d2011-07-07 08:21:25 +0000395
Gustaf Ullberg422b9e02019-10-09 13:02:14 +0200396 if (capture_nonlocked_.capture_processing_format.sample_rate_hz() <
397 formats_.api_format.output_stream().sample_rate_hz() &&
398 formats_.api_format.output_stream().sample_rate_hz() == 48000) {
399 capture_.capture_fullband_audio.reset(
400 new AudioBuffer(formats_.api_format.input_stream().sample_rate_hz(),
401 formats_.api_format.input_stream().num_channels(),
402 formats_.api_format.output_stream().sample_rate_hz(),
403 formats_.api_format.output_stream().num_channels(),
404 formats_.api_format.output_stream().sample_rate_hz(),
405 formats_.api_format.output_stream().num_channels()));
406 } else {
407 capture_.capture_fullband_audio.reset();
408 }
409
peah764e3642016-10-22 05:04:30 -0700410 AllocateRenderQueue();
411
Per Åhgren0695df12020-01-13 14:43:13 +0100412 InitializeGainController1();
Per Åhgrenc0734712020-01-02 15:15:36 +0100413 InitializeTransientSuppressor();
Per Åhgren0f14db22020-01-03 14:27:14 +0100414 InitializeHighPassFilter(true);
Sam Zackrisson0824c6f2019-10-07 14:03:56 +0200415 InitializeVoiceDetector();
ivoc9f4a4a02016-10-28 05:39:16 -0700416 InitializeResidualEchoDetector();
Gustaf Ullberg8eb9c7d2017-10-14 08:28:46 +0200417 InitializeEchoController();
alessiob3ec96df2017-05-22 06:57:06 -0700418 InitializeGainController2();
saza0bad15f2019-10-16 11:46:11 +0200419 InitializeNoiseSuppressor();
Valeriia Nemychnikovaf06eb572018-08-29 10:37:09 +0200420 InitializeAnalyzer();
Sam Zackrisson0beac582017-09-25 12:04:02 +0200421 InitializePostProcessor();
Alex Loiko5825aa62017-12-18 16:02:40 +0100422 InitializePreProcessor();
solenberg70f99032015-12-08 11:07:32 -0800423
aleloi868f32f2017-05-23 07:20:05 -0700424 if (aec_dump_) {
Minyue Li656d6092018-08-10 15:38:52 +0200425 aec_dump_->WriteInitMessage(formats_.api_format, rtc::TimeUTCMillis());
aleloi868f32f2017-05-23 07:20:05 -0700426 }
niklase@google.com470e71d2011-07-07 08:21:25 +0000427 return kNoError;
428}
429
Michael Graczyk86c6d332015-07-23 11:41:39 -0700430int AudioProcessingImpl::InitializeLocked(const ProcessingConfig& config) {
Per Åhgren4bdced52017-06-27 16:00:38 +0200431 UpdateActiveSubmoduleStates();
432
Michael Graczyk86c6d332015-07-23 11:41:39 -0700433 for (const auto& stream : config.streams) {
Michael Graczyk86c6d332015-07-23 11:41:39 -0700434 if (stream.num_channels() > 0 && stream.sample_rate_hz() <= 0) {
435 return kBadSampleRateError;
436 }
andrew@webrtc.orga8b97372014-03-10 22:26:12 +0000437 }
Michael Graczyk86c6d332015-07-23 11:41:39 -0700438
Peter Kasting69558702016-01-12 16:26:35 -0800439 const size_t num_in_channels = config.input_stream().num_channels();
440 const size_t num_out_channels = config.output_stream().num_channels();
Michael Graczyk86c6d332015-07-23 11:41:39 -0700441
442 // Need at least one input channel.
443 // Need either one output channel or as many outputs as there are inputs.
444 if (num_in_channels == 0 ||
445 !(num_out_channels == 1 || num_out_channels == num_in_channels)) {
Michael Graczykc2047542015-07-22 21:06:11 -0700446 return kBadNumberChannelsError;
447 }
448
peahdf3efa82015-11-28 12:35:15 -0800449 formats_.api_format = config;
andrew@webrtc.orgddbb8a22014-04-22 21:00:04 +0000450
Per Åhgrenfcbe4072019-09-15 00:27:58 +0200451 // Choose maximum rate to use for the split filtering.
452 RTC_DCHECK(config_.pipeline.maximum_internal_processing_rate == 48000 ||
453 config_.pipeline.maximum_internal_processing_rate == 32000);
454 int max_splitting_rate = 48000;
455 if (config_.pipeline.maximum_internal_processing_rate == 32000) {
456 max_splitting_rate = config_.pipeline.maximum_internal_processing_rate;
457 }
458
Per Åhgrenc8626b62019-08-23 15:49:51 +0200459 int capture_processing_rate = SuitableProcessRate(
peah423d2362016-04-09 16:06:52 -0700460 std::min(formats_.api_format.input_stream().sample_rate_hz(),
peah2ace3f92016-09-10 04:42:27 -0700461 formats_.api_format.output_stream().sample_rate_hz()),
Per Åhgrenfcbe4072019-09-15 00:27:58 +0200462 max_splitting_rate,
peah2ace3f92016-09-10 04:42:27 -0700463 submodule_states_.CaptureMultiBandSubModulesActive() ||
464 submodule_states_.RenderMultiBandSubModulesActive());
Per Åhgrenc8626b62019-08-23 15:49:51 +0200465 RTC_DCHECK_NE(8000, capture_processing_rate);
andrew@webrtc.orga8b97372014-03-10 22:26:12 +0000466
peahde65ddc2016-09-16 15:02:15 -0700467 capture_nonlocked_.capture_processing_format =
468 StreamConfig(capture_processing_rate);
peah2ace3f92016-09-10 04:42:27 -0700469
peah2ce640f2017-04-07 03:57:48 -0700470 int render_processing_rate;
Gustaf Ullbergbd83b912017-10-18 12:32:42 +0200471 if (!capture_nonlocked_.echo_controller_enabled) {
Per Åhgrenc8626b62019-08-23 15:49:51 +0200472 render_processing_rate = SuitableProcessRate(
peah2ce640f2017-04-07 03:57:48 -0700473 std::min(formats_.api_format.reverse_input_stream().sample_rate_hz(),
474 formats_.api_format.reverse_output_stream().sample_rate_hz()),
Per Åhgrenfcbe4072019-09-15 00:27:58 +0200475 max_splitting_rate,
peah2ce640f2017-04-07 03:57:48 -0700476 submodule_states_.CaptureMultiBandSubModulesActive() ||
477 submodule_states_.RenderMultiBandSubModulesActive());
478 } else {
479 render_processing_rate = capture_processing_rate;
480 }
481
peahde65ddc2016-09-16 15:02:15 -0700482 // If the forward sample rate is 8 kHz, the render stream is also processed
aluebseb3603b2016-04-20 15:27:58 -0700483 // at this rate.
peahde65ddc2016-09-16 15:02:15 -0700484 if (capture_nonlocked_.capture_processing_format.sample_rate_hz() ==
485 kSampleRate8kHz) {
486 render_processing_rate = kSampleRate8kHz;
andrew@webrtc.orga8b97372014-03-10 22:26:12 +0000487 } else {
peahde65ddc2016-09-16 15:02:15 -0700488 render_processing_rate =
489 std::max(render_processing_rate, static_cast<int>(kSampleRate16kHz));
andrew@webrtc.orgddbb8a22014-04-22 21:00:04 +0000490 }
491
Per Åhgrenc8626b62019-08-23 15:49:51 +0200492 RTC_DCHECK_NE(8000, render_processing_rate);
493
peahce4d9152017-05-19 01:28:05 -0700494 if (submodule_states_.RenderMultiBandSubModulesActive()) {
Sam Zackrissonfeee1e42019-09-20 07:50:35 +0200495 // By default, downmix the render stream to mono for analysis. This has been
496 // demonstrated to work well for AEC in most practical scenarios.
Per Åhgrene14cb992019-11-27 09:34:22 +0100497 const bool multi_channel_render = config_.pipeline.multi_channel_render &&
498 constants_.multi_channel_render_support;
Sam Zackrissonfeee1e42019-09-20 07:50:35 +0200499 int render_processing_num_channels =
Per Åhgrene14cb992019-11-27 09:34:22 +0100500 multi_channel_render
Sam Zackrissonfeee1e42019-09-20 07:50:35 +0200501 ? formats_.api_format.reverse_input_stream().num_channels()
502 : 1;
503 formats_.render_processing_format =
504 StreamConfig(render_processing_rate, render_processing_num_channels);
peahce4d9152017-05-19 01:28:05 -0700505 } else {
506 formats_.render_processing_format = StreamConfig(
507 formats_.api_format.reverse_input_stream().sample_rate_hz(),
508 formats_.api_format.reverse_input_stream().num_channels());
509 }
andrew@webrtc.orgddbb8a22014-04-22 21:00:04 +0000510
peahde65ddc2016-09-16 15:02:15 -0700511 if (capture_nonlocked_.capture_processing_format.sample_rate_hz() ==
512 kSampleRate32kHz ||
513 capture_nonlocked_.capture_processing_format.sample_rate_hz() ==
514 kSampleRate48kHz) {
peahdf3efa82015-11-28 12:35:15 -0800515 capture_nonlocked_.split_rate = kSampleRate16kHz;
andrew@webrtc.orgddbb8a22014-04-22 21:00:04 +0000516 } else {
peahdf3efa82015-11-28 12:35:15 -0800517 capture_nonlocked_.split_rate =
peahde65ddc2016-09-16 15:02:15 -0700518 capture_nonlocked_.capture_processing_format.sample_rate_hz();
andrew@webrtc.orga8b97372014-03-10 22:26:12 +0000519 }
520
521 return InitializeLocked();
522}
523
peah88ac8532016-09-12 16:47:25 -0700524void AudioProcessingImpl::ApplyConfig(const AudioProcessing::Config& config) {
Sam Zackrisson72cc71c2019-10-21 12:54:02 +0200525 RTC_LOG(LS_INFO) << "AudioProcessing::ApplyConfig: " << config.ToString();
526
peah88ac8532016-09-12 16:47:25 -0700527 // Run in a single-threaded manner when applying the settings.
528 rtc::CritScope cs_render(&crit_render_);
529 rtc::CritScope cs_capture(&crit_capture_);
530
Sam Zackrissonfeee1e42019-09-20 07:50:35 +0200531 const bool pipeline_config_changed =
Per Åhgrene14cb992019-11-27 09:34:22 +0100532 config_.pipeline.multi_channel_render !=
533 config.pipeline.multi_channel_render ||
534 config_.pipeline.multi_channel_capture !=
Per Åhgrenc0424252019-12-10 13:04:15 +0100535 config.pipeline.multi_channel_capture ||
536 config_.pipeline.maximum_internal_processing_rate !=
537 config.pipeline.maximum_internal_processing_rate;
Sam Zackrissonfeee1e42019-09-20 07:50:35 +0200538
Per Åhgren200feba2019-03-06 04:16:46 +0100539 const bool aec_config_changed =
540 config_.echo_canceller.enabled != config.echo_canceller.enabled ||
Per Åhgren62ea0aa2019-12-09 10:18:44 +0100541 config_.echo_canceller.mobile_mode != config.echo_canceller.mobile_mode;
Per Åhgren200feba2019-03-06 04:16:46 +0100542
Sam Zackrissonf0d1c032019-03-27 13:28:08 +0100543 const bool agc1_config_changed =
544 config_.gain_controller1.enabled != config.gain_controller1.enabled ||
545 config_.gain_controller1.mode != config.gain_controller1.mode ||
546 config_.gain_controller1.target_level_dbfs !=
547 config.gain_controller1.target_level_dbfs ||
548 config_.gain_controller1.compression_gain_db !=
549 config.gain_controller1.compression_gain_db ||
550 config_.gain_controller1.enable_limiter !=
551 config.gain_controller1.enable_limiter ||
552 config_.gain_controller1.analog_level_minimum !=
553 config.gain_controller1.analog_level_minimum ||
554 config_.gain_controller1.analog_level_maximum !=
Per Åhgren0695df12020-01-13 14:43:13 +0100555 config.gain_controller1.analog_level_maximum ||
556 config_.gain_controller1.analog_gain_controller.enabled !=
557 config.gain_controller1.analog_gain_controller.enabled ||
558 config_.gain_controller1.analog_gain_controller.startup_min_volume !=
559 config.gain_controller1.analog_gain_controller.startup_min_volume ||
560 config_.gain_controller1.analog_gain_controller.clipped_level_min !=
561 config.gain_controller1.analog_gain_controller.clipped_level_min ||
562 config_.gain_controller1.analog_gain_controller
563 .enable_agc2_level_estimator !=
564 config.gain_controller1.analog_gain_controller
565 .enable_agc2_level_estimator ||
566 config_.gain_controller1.analog_gain_controller.enable_digital_adaptive !=
567 config.gain_controller1.analog_gain_controller
568 .enable_digital_adaptive;
Sam Zackrissonf0d1c032019-03-27 13:28:08 +0100569
Per Åhgren2bd85ab2020-01-03 10:36:34 +0100570 const bool agc2_config_changed =
571 config_.gain_controller2.enabled != config.gain_controller2.enabled;
572
Sam Zackrisson0824c6f2019-10-07 14:03:56 +0200573 const bool voice_detection_config_changed =
574 config_.voice_detection.enabled != config.voice_detection.enabled;
575
saza0bad15f2019-10-16 11:46:11 +0200576 const bool ns_config_changed =
577 config_.noise_suppression.enabled != config.noise_suppression.enabled ||
578 config_.noise_suppression.level != config.noise_suppression.level;
579
Per Åhgrenc0734712020-01-02 15:15:36 +0100580 const bool ts_config_changed = config_.transient_suppression.enabled !=
581 config.transient_suppression.enabled;
582
Per Åhgren0f14db22020-01-03 14:27:14 +0100583 const bool pre_amplifier_config_changed =
584 config_.pre_amplifier.enabled != config.pre_amplifier.enabled ||
585 config_.pre_amplifier.fixed_gain_factor !=
586 config.pre_amplifier.fixed_gain_factor;
587
Yves Gerey499bc6c2018-10-10 18:29:07 +0200588 config_ = config;
589
Per Åhgren200feba2019-03-06 04:16:46 +0100590 if (aec_config_changed) {
591 InitializeEchoController();
592 }
Sam Zackrissoncdf0e6d2018-09-17 11:05:17 +0200593
saza0bad15f2019-10-16 11:46:11 +0200594 if (ns_config_changed) {
595 InitializeNoiseSuppressor();
596 }
Sam Zackrisson23513132019-01-11 15:10:32 +0100597
Per Åhgrenc0734712020-01-02 15:15:36 +0100598 if (ts_config_changed) {
599 InitializeTransientSuppressor();
600 }
601
Per Åhgren0f14db22020-01-03 14:27:14 +0100602 InitializeHighPassFilter(false);
peah8271d042016-11-22 07:24:52 -0800603
Sam Zackrissonf0d1c032019-03-27 13:28:08 +0100604 if (agc1_config_changed) {
Per Åhgren0695df12020-01-13 14:43:13 +0100605 InitializeGainController1();
Sam Zackrissonf0d1c032019-03-27 13:28:08 +0100606 }
607
Sam Zackrissonab1aee02018-03-05 15:59:06 +0100608 const bool config_ok = GainController2::Validate(config_.gain_controller2);
alessiob3ec96df2017-05-22 06:57:06 -0700609 if (!config_ok) {
Jonas Olsson645b0272018-02-15 15:16:27 +0100610 RTC_LOG(LS_ERROR) << "AudioProcessing module config error\n"
611 "Gain Controller 2: "
Mirko Bonadei675513b2017-11-09 11:09:25 +0100612 << GainController2::ToString(config_.gain_controller2)
Jonas Olsson645b0272018-02-15 15:16:27 +0100613 << "\nReverting to default parameter set";
alessiob3ec96df2017-05-22 06:57:06 -0700614 config_.gain_controller2 = AudioProcessing::Config::GainController2();
615 }
Per Åhgren0f14db22020-01-03 14:27:14 +0100616
Per Åhgren2bd85ab2020-01-03 10:36:34 +0100617 if (agc2_config_changed) {
618 InitializeGainController2();
619 }
Per Åhgren0f14db22020-01-03 14:27:14 +0100620
621 if (pre_amplifier_config_changed) {
622 InitializePreAmplifier();
623 }
Sam Zackrissonb24c00f2018-11-26 16:18:25 +0100624
saza1d600522019-10-18 13:29:43 +0200625 if (config_.level_estimation.enabled && !submodules_.output_level_estimator) {
626 submodules_.output_level_estimator = std::make_unique<LevelEstimator>();
Sam Zackrissonb24c00f2018-11-26 16:18:25 +0100627 }
Sam Zackrisson4db667b2018-12-21 16:29:27 +0100628
Sam Zackrisson0824c6f2019-10-07 14:03:56 +0200629 if (voice_detection_config_changed) {
630 InitializeVoiceDetector();
Sam Zackrisson4db667b2018-12-21 16:29:27 +0100631 }
Sam Zackrissonfeee1e42019-09-20 07:50:35 +0200632
633 // Reinitialization must happen after all submodule configuration to avoid
634 // additional reinitializations on the next capture / render processing call.
635 if (pipeline_config_changed) {
636 InitializeLocked(formats_.api_format);
637 }
peah88ac8532016-09-12 16:47:25 -0700638}
639
Per Åhgrenc0734712020-01-02 15:15:36 +0100640// TODO(webrtc:5298): Remove.
Sam Zackrisson12e319a2020-01-03 14:54:20 +0100641void AudioProcessingImpl::SetExtraOptions(const webrtc::Config& config) {}
andrew@webrtc.org61e596f2013-07-25 18:28:29 +0000642
Sam Zackrissonb37e59d2020-04-27 08:39:33 +0200643void AudioProcessingImpl::OverrideSubmoduleCreationForTesting(
644 const ApmSubmoduleCreationOverrides& overrides) {
645 rtc::CritScope cs(&crit_capture_);
646 submodule_creation_overrides_ = overrides;
647}
648
andrew@webrtc.orgddbb8a22014-04-22 21:00:04 +0000649int AudioProcessingImpl::proc_sample_rate_hz() const {
peahdf3efa82015-11-28 12:35:15 -0800650 // Used as callback from submodules, hence locking is not allowed.
peahde65ddc2016-09-16 15:02:15 -0700651 return capture_nonlocked_.capture_processing_format.sample_rate_hz();
niklase@google.com470e71d2011-07-07 08:21:25 +0000652}
653
Gustaf Ullberg422b9e02019-10-09 13:02:14 +0200654int AudioProcessingImpl::proc_fullband_sample_rate_hz() const {
655 return capture_.capture_fullband_audio
656 ? capture_.capture_fullband_audio->num_frames() * 100
657 : capture_nonlocked_.capture_processing_format.sample_rate_hz();
658}
659
andrew@webrtc.orgddbb8a22014-04-22 21:00:04 +0000660int AudioProcessingImpl::proc_split_sample_rate_hz() const {
peahdf3efa82015-11-28 12:35:15 -0800661 // Used as callback from submodules, hence locking is not allowed.
662 return capture_nonlocked_.split_rate;
niklase@google.com470e71d2011-07-07 08:21:25 +0000663}
664
Peter Kasting69558702016-01-12 16:26:35 -0800665size_t AudioProcessingImpl::num_reverse_channels() const {
peahdf3efa82015-11-28 12:35:15 -0800666 // Used as callback from submodules, hence locking is not allowed.
peahde65ddc2016-09-16 15:02:15 -0700667 return formats_.render_processing_format.num_channels();
niklase@google.com470e71d2011-07-07 08:21:25 +0000668}
669
Peter Kasting69558702016-01-12 16:26:35 -0800670size_t AudioProcessingImpl::num_input_channels() const {
peahdf3efa82015-11-28 12:35:15 -0800671 // Used as callback from submodules, hence locking is not allowed.
672 return formats_.api_format.input_stream().num_channels();
niklase@google.com470e71d2011-07-07 08:21:25 +0000673}
674
Peter Kasting69558702016-01-12 16:26:35 -0800675size_t AudioProcessingImpl::num_proc_channels() const {
aluebsb2328d12016-01-11 20:32:29 -0800676 // Used as callback from submodules, hence locking is not allowed.
Per Åhgrene14cb992019-11-27 09:34:22 +0100677 const bool multi_channel_capture = config_.pipeline.multi_channel_capture &&
678 constants_.multi_channel_capture_support;
679 if (capture_nonlocked_.echo_controller_enabled && !multi_channel_capture) {
Sam Zackrissonfeee1e42019-09-20 07:50:35 +0200680 return 1;
681 }
682 return num_output_channels();
aluebsb2328d12016-01-11 20:32:29 -0800683}
684
Peter Kasting69558702016-01-12 16:26:35 -0800685size_t AudioProcessingImpl::num_output_channels() const {
peahdf3efa82015-11-28 12:35:15 -0800686 // Used as callback from submodules, hence locking is not allowed.
687 return formats_.api_format.output_stream().num_channels();
niklase@google.com470e71d2011-07-07 08:21:25 +0000688}
689
andrew@webrtc.org17342e52014-02-12 22:28:31 +0000690void AudioProcessingImpl::set_output_will_be_muted(bool muted) {
peahdf3efa82015-11-28 12:35:15 -0800691 rtc::CritScope cs(&crit_capture_);
692 capture_.output_will_be_muted = muted;
saza1d600522019-10-18 13:29:43 +0200693 if (submodules_.agc_manager.get()) {
694 submodules_.agc_manager->SetCaptureMuted(capture_.output_will_be_muted);
pbos@webrtc.org788acd12014-12-15 09:41:24 +0000695 }
andrew@webrtc.org17342e52014-02-12 22:28:31 +0000696}
697
Alessio Bazzicac054e782018-04-16 12:10:09 +0200698void AudioProcessingImpl::SetRuntimeSetting(RuntimeSetting setting) {
Alex Loiko73ec0192018-05-15 10:52:28 +0200699 switch (setting.type()) {
700 case RuntimeSetting::Type::kCustomRenderProcessingRuntimeSetting:
Alessio Bazzica7c19a702019-11-07 13:22:00 +0100701 case RuntimeSetting::Type::kPlayoutAudioDeviceChange:
Alex Loiko73ec0192018-05-15 10:52:28 +0200702 render_runtime_settings_enqueuer_.Enqueue(setting);
703 return;
Alex Loiko73ec0192018-05-15 10:52:28 +0200704 case RuntimeSetting::Type::kCapturePreGain:
Sam Zackrissonf0d1c032019-03-27 13:28:08 +0100705 case RuntimeSetting::Type::kCaptureCompressionGain:
Per Åhgren6ee75fd2019-04-26 11:33:37 +0200706 case RuntimeSetting::Type::kCaptureFixedPostGain:
Alessio Bazzica7c19a702019-11-07 13:22:00 +0100707 capture_runtime_settings_enqueuer_.Enqueue(setting);
708 return;
Fredrik Hernqvistca362852019-05-10 15:50:02 +0200709 case RuntimeSetting::Type::kPlayoutVolumeChange:
Alex Loiko73ec0192018-05-15 10:52:28 +0200710 capture_runtime_settings_enqueuer_.Enqueue(setting);
Alessio Bazzica7c19a702019-11-07 13:22:00 +0100711 render_runtime_settings_enqueuer_.Enqueue(setting);
712 return;
713 case RuntimeSetting::Type::kNotSpecified:
714 RTC_NOTREACHED();
Alex Loiko73ec0192018-05-15 10:52:28 +0200715 return;
716 }
717 // The language allows the enum to have a non-enumerator
718 // value. Check that this doesn't happen.
719 RTC_NOTREACHED();
Alessio Bazzicac054e782018-04-16 12:10:09 +0200720}
721
722AudioProcessingImpl::RuntimeSettingEnqueuer::RuntimeSettingEnqueuer(
723 SwapQueue<RuntimeSetting>* runtime_settings)
Alessio Bazzica33444dc2018-04-20 13:16:55 +0200724 : runtime_settings_(*runtime_settings) {
725 RTC_DCHECK(runtime_settings);
Alessio Bazzicac054e782018-04-16 12:10:09 +0200726}
727
728AudioProcessingImpl::RuntimeSettingEnqueuer::~RuntimeSettingEnqueuer() =
729 default;
730
731void AudioProcessingImpl::RuntimeSettingEnqueuer::Enqueue(
732 RuntimeSetting setting) {
733 size_t remaining_attempts = 10;
Alessio Bazzica33444dc2018-04-20 13:16:55 +0200734 while (!runtime_settings_.Insert(&setting) && remaining_attempts-- > 0) {
Alessio Bazzicac054e782018-04-16 12:10:09 +0200735 RuntimeSetting setting_to_discard;
Alessio Bazzica33444dc2018-04-20 13:16:55 +0200736 if (runtime_settings_.Remove(&setting_to_discard))
Alessio Bazzicac054e782018-04-16 12:10:09 +0200737 RTC_LOG(LS_ERROR)
738 << "The runtime settings queue is full. Oldest setting discarded.";
739 }
740 if (remaining_attempts == 0)
741 RTC_LOG(LS_ERROR) << "Cannot enqueue a new runtime setting.";
742}
andrew@webrtc.org17342e52014-02-12 22:28:31 +0000743
Sam Zackrisson12e319a2020-01-03 14:54:20 +0100744int AudioProcessingImpl::MaybeInitializeCapture(
745 const StreamConfig& input_config,
746 const StreamConfig& output_config) {
peahdf3efa82015-11-28 12:35:15 -0800747 ProcessingConfig processing_config;
peah2ace3f92016-09-10 04:42:27 -0700748 bool reinitialization_required = false;
peahdf3efa82015-11-28 12:35:15 -0800749 {
Sam Zackrisson12e319a2020-01-03 14:54:20 +0100750 // Acquire the capture lock in order to access api_format. The lock is
751 // released immediately, as we may need to acquire the render lock as part
752 // of the conditional reinitialization.
peahdf3efa82015-11-28 12:35:15 -0800753 rtc::CritScope cs_capture(&crit_capture_);
peahdf3efa82015-11-28 12:35:15 -0800754 processing_config = formats_.api_format;
peah2ace3f92016-09-10 04:42:27 -0700755 reinitialization_required = UpdateActiveSubmoduleStates();
niklase@google.com470e71d2011-07-07 08:21:25 +0000756 }
andrew@webrtc.org17e40642014-03-04 20:58:13 +0000757
Oskar Sundbom4b276482019-05-23 14:28:00 +0200758 if (processing_config.input_stream() != input_config) {
759 processing_config.input_stream() = input_config;
760 reinitialization_required = true;
peahdf3efa82015-11-28 12:35:15 -0800761 }
Oskar Sundbom4b276482019-05-23 14:28:00 +0200762
763 if (processing_config.output_stream() != output_config) {
764 processing_config.output_stream() = output_config;
765 reinitialization_required = true;
766 }
767
768 if (reinitialization_required) {
Oskar Sundbom4b276482019-05-23 14:28:00 +0200769 rtc::CritScope cs_render(&crit_render_);
770 rtc::CritScope cs_capture(&crit_capture_);
771 RETURN_ON_ERR(InitializeLocked(processing_config));
772 }
Sam Zackrisson12e319a2020-01-03 14:54:20 +0100773 return kNoError;
774}
775
776int AudioProcessingImpl::ProcessStream(const float* const* src,
777 const StreamConfig& input_config,
778 const StreamConfig& output_config,
779 float* const* dest) {
780 TRACE_EVENT0("webrtc", "AudioProcessing::ProcessStream_StreamConfig");
781 if (!src || !dest) {
782 return kNullPointerError;
783 }
784
785 RETURN_ON_ERR(MaybeInitializeCapture(input_config, output_config));
Oskar Sundbom4b276482019-05-23 14:28:00 +0200786
peahdf3efa82015-11-28 12:35:15 -0800787 rtc::CritScope cs_capture(&crit_capture_);
andrew@webrtc.org17e40642014-03-04 20:58:13 +0000788
aleloi868f32f2017-05-23 07:20:05 -0700789 if (aec_dump_) {
790 RecordUnprocessedCaptureStream(src);
791 }
792
Per Åhgrena1351272019-08-15 12:15:46 +0200793 capture_.keyboard_info.Extract(src, formats_.api_format.input_stream());
peahdf3efa82015-11-28 12:35:15 -0800794 capture_.capture_audio->CopyFrom(src, formats_.api_format.input_stream());
Gustaf Ullberg8675eee2019-10-09 13:34:36 +0200795 if (capture_.capture_fullband_audio) {
796 capture_.capture_fullband_audio->CopyFrom(
797 src, formats_.api_format.input_stream());
798 }
peahde65ddc2016-09-16 15:02:15 -0700799 RETURN_ON_ERR(ProcessCaptureStreamLocked());
Gustaf Ullberg422b9e02019-10-09 13:02:14 +0200800 if (capture_.capture_fullband_audio) {
801 capture_.capture_fullband_audio->CopyTo(formats_.api_format.output_stream(),
802 dest);
803 } else {
804 capture_.capture_audio->CopyTo(formats_.api_format.output_stream(), dest);
805 }
andrew@webrtc.org17e40642014-03-04 20:58:13 +0000806
aleloi868f32f2017-05-23 07:20:05 -0700807 if (aec_dump_) {
808 RecordProcessedCaptureStream(dest);
809 }
andrew@webrtc.org17e40642014-03-04 20:58:13 +0000810 return kNoError;
811}
812
Alex Loiko73ec0192018-05-15 10:52:28 +0200813void AudioProcessingImpl::HandleCaptureRuntimeSettings() {
Alessio Bazzicac054e782018-04-16 12:10:09 +0200814 RuntimeSetting setting;
Alex Loiko73ec0192018-05-15 10:52:28 +0200815 while (capture_runtime_settings_.Remove(&setting)) {
Alex Loiko62347222018-09-10 10:18:07 +0200816 if (aec_dump_) {
817 aec_dump_->WriteRuntimeSetting(setting);
818 }
Alessio Bazzicac054e782018-04-16 12:10:09 +0200819 switch (setting.type()) {
820 case RuntimeSetting::Type::kCapturePreGain:
Alex Loikob5c9a792018-04-16 16:31:22 +0200821 if (config_.pre_amplifier.enabled) {
822 float value;
823 setting.GetFloat(&value);
Sam Zackrisson21bfa402019-10-23 09:43:01 +0200824 config_.pre_amplifier.fixed_gain_factor = value;
saza1d600522019-10-18 13:29:43 +0200825 submodules_.pre_amplifier->SetGainFactor(value);
Alex Loikob5c9a792018-04-16 16:31:22 +0200826 }
827 // TODO(bugs.chromium.org/9138): Log setting handling by Aec Dump.
Alessio Bazzicac054e782018-04-16 12:10:09 +0200828 break;
Sam Zackrissonf0d1c032019-03-27 13:28:08 +0100829 case RuntimeSetting::Type::kCaptureCompressionGain: {
Per Åhgren0e3198e2019-11-18 08:52:22 +0100830 if (!submodules_.agc_manager) {
831 float value;
832 setting.GetFloat(&value);
833 int int_value = static_cast<int>(value + .5f);
834 config_.gain_controller1.compression_gain_db = int_value;
Per Åhgren0695df12020-01-13 14:43:13 +0100835 if (submodules_.gain_control) {
836 int error =
837 submodules_.gain_control->set_compression_gain_db(int_value);
838 RTC_DCHECK_EQ(kNoError, error);
839 }
Per Åhgren0e3198e2019-11-18 08:52:22 +0100840 }
Sam Zackrissonf0d1c032019-03-27 13:28:08 +0100841 break;
842 }
Per Åhgren6ee75fd2019-04-26 11:33:37 +0200843 case RuntimeSetting::Type::kCaptureFixedPostGain: {
Per Åhgren2bd85ab2020-01-03 10:36:34 +0100844 if (submodules_.gain_controller2) {
Per Åhgren6ee75fd2019-04-26 11:33:37 +0200845 float value;
846 setting.GetFloat(&value);
847 config_.gain_controller2.fixed_digital.gain_db = value;
saza1d600522019-10-18 13:29:43 +0200848 submodules_.gain_controller2->ApplyConfig(config_.gain_controller2);
Per Åhgren6ee75fd2019-04-26 11:33:37 +0200849 }
850 break;
851 }
Fredrik Hernqvistca362852019-05-10 15:50:02 +0200852 case RuntimeSetting::Type::kPlayoutVolumeChange: {
853 int value;
854 setting.GetInt(&value);
855 capture_.playout_volume = value;
856 break;
857 }
Alessio Bazzica7c19a702019-11-07 13:22:00 +0100858 case RuntimeSetting::Type::kPlayoutAudioDeviceChange:
859 RTC_NOTREACHED();
860 break;
Alex Loiko73ec0192018-05-15 10:52:28 +0200861 case RuntimeSetting::Type::kCustomRenderProcessingRuntimeSetting:
862 RTC_NOTREACHED();
863 break;
864 case RuntimeSetting::Type::kNotSpecified:
865 RTC_NOTREACHED();
866 break;
867 }
868 }
869}
870
871void AudioProcessingImpl::HandleRenderRuntimeSettings() {
872 RuntimeSetting setting;
873 while (render_runtime_settings_.Remove(&setting)) {
Alex Loiko62347222018-09-10 10:18:07 +0200874 if (aec_dump_) {
875 aec_dump_->WriteRuntimeSetting(setting);
876 }
Alex Loiko73ec0192018-05-15 10:52:28 +0200877 switch (setting.type()) {
Alessio Bazzica7c19a702019-11-07 13:22:00 +0100878 case RuntimeSetting::Type::kPlayoutAudioDeviceChange: // fall-through
Alessio Bazzica7587de42019-11-11 13:32:20 +0100879 case RuntimeSetting::Type::kPlayoutVolumeChange: // fall-through
Alex Loiko73ec0192018-05-15 10:52:28 +0200880 case RuntimeSetting::Type::kCustomRenderProcessingRuntimeSetting:
saza1d600522019-10-18 13:29:43 +0200881 if (submodules_.render_pre_processor) {
882 submodules_.render_pre_processor->SetRuntimeSetting(setting);
Alex Loiko73ec0192018-05-15 10:52:28 +0200883 }
884 break;
Sam Zackrissonf0d1c032019-03-27 13:28:08 +0100885 case RuntimeSetting::Type::kCapturePreGain: // fall-through
886 case RuntimeSetting::Type::kCaptureCompressionGain: // fall-through
Per Åhgren6ee75fd2019-04-26 11:33:37 +0200887 case RuntimeSetting::Type::kCaptureFixedPostGain: // fall-through
Alessio Bazzica33444dc2018-04-20 13:16:55 +0200888 case RuntimeSetting::Type::kNotSpecified:
Alessio Bazzicac054e782018-04-16 12:10:09 +0200889 RTC_NOTREACHED();
890 break;
891 }
892 }
893}
894
peah9e6a2902017-05-15 07:19:21 -0700895void AudioProcessingImpl::QueueBandedRenderAudio(AudioBuffer* audio) {
kwibergaf476c72016-11-28 15:21:39 -0800896 RTC_DCHECK_GE(160, audio->num_frames_per_band());
peah764e3642016-10-22 05:04:30 -0700897
saza1d600522019-10-18 13:29:43 +0200898 if (submodules_.echo_control_mobile) {
Per Åhgrenb6e24d72019-04-29 12:14:50 +0200899 EchoControlMobileImpl::PackRenderAudioBuffer(audio, num_output_channels(),
900 num_reverse_channels(),
901 &aecm_render_queue_buffer_);
902 RTC_DCHECK(aecm_render_signal_queue_);
903 // Insert the samples into the queue.
904 if (!aecm_render_signal_queue_->Insert(&aecm_render_queue_buffer_)) {
905 // The data queue is full and needs to be emptied.
906 EmptyQueuedRenderAudio();
peaha0624602016-10-25 04:45:24 -0700907
Per Åhgrenb6e24d72019-04-29 12:14:50 +0200908 // Retry the insert (should always work).
909 bool result =
910 aecm_render_signal_queue_->Insert(&aecm_render_queue_buffer_);
911 RTC_DCHECK(result);
912 }
peah764e3642016-10-22 05:04:30 -0700913 }
peah701d6282016-10-25 05:42:20 -0700914
Per Åhgren0695df12020-01-13 14:43:13 +0100915 if (!submodules_.agc_manager && submodules_.gain_control) {
Per Åhgrene35b32c2019-11-22 18:22:04 +0100916 GainControlImpl::PackRenderAudioBuffer(*audio, &agc_render_queue_buffer_);
peah701d6282016-10-25 05:42:20 -0700917 // Insert the samples into the queue.
918 if (!agc_render_signal_queue_->Insert(&agc_render_queue_buffer_)) {
919 // The data queue is full and needs to be emptied.
920 EmptyQueuedRenderAudio();
921
922 // Retry the insert (should always work).
923 bool result = agc_render_signal_queue_->Insert(&agc_render_queue_buffer_);
924 RTC_DCHECK(result);
925 }
926 }
peah9e6a2902017-05-15 07:19:21 -0700927}
ivoc9f4a4a02016-10-28 05:39:16 -0700928
peah9e6a2902017-05-15 07:19:21 -0700929void AudioProcessingImpl::QueueNonbandedRenderAudio(AudioBuffer* audio) {
ivoc9f4a4a02016-10-28 05:39:16 -0700930 ResidualEchoDetector::PackRenderAudioBuffer(audio, &red_render_queue_buffer_);
931
932 // Insert the samples into the queue.
933 if (!red_render_signal_queue_->Insert(&red_render_queue_buffer_)) {
934 // The data queue is full and needs to be emptied.
935 EmptyQueuedRenderAudio();
936
937 // Retry the insert (should always work).
938 bool result = red_render_signal_queue_->Insert(&red_render_queue_buffer_);
939 RTC_DCHECK(result);
940 }
peah764e3642016-10-22 05:04:30 -0700941}
942
943void AudioProcessingImpl::AllocateRenderQueue() {
peah701d6282016-10-25 05:42:20 -0700944 const size_t new_agc_render_queue_element_max_size =
peah9e6a2902017-05-15 07:19:21 -0700945 std::max(static_cast<size_t>(1), kMaxAllowedValuesOfSamplesPerBand);
peah701d6282016-10-25 05:42:20 -0700946
ivoc9f4a4a02016-10-28 05:39:16 -0700947 const size_t new_red_render_queue_element_max_size =
948 std::max(static_cast<size_t>(1), kMaxAllowedValuesOfSamplesPerFrame);
949
peaha0624602016-10-25 04:45:24 -0700950 // Reallocate the queues if the queue item sizes are too small to fit the
951 // data to put in the queues.
peah701d6282016-10-25 05:42:20 -0700952
953 if (agc_render_queue_element_max_size_ <
954 new_agc_render_queue_element_max_size) {
955 agc_render_queue_element_max_size_ = new_agc_render_queue_element_max_size;
956
957 std::vector<int16_t> template_queue_element(
958 agc_render_queue_element_max_size_);
959
960 agc_render_signal_queue_.reset(
961 new SwapQueue<std::vector<int16_t>, RenderQueueItemVerifier<int16_t>>(
962 kMaxNumFramesToBuffer, template_queue_element,
963 RenderQueueItemVerifier<int16_t>(
964 agc_render_queue_element_max_size_)));
965
966 agc_render_queue_buffer_.resize(agc_render_queue_element_max_size_);
967 agc_capture_queue_buffer_.resize(agc_render_queue_element_max_size_);
968 } else {
969 agc_render_signal_queue_->Clear();
peah764e3642016-10-22 05:04:30 -0700970 }
ivoc9f4a4a02016-10-28 05:39:16 -0700971
972 if (red_render_queue_element_max_size_ <
973 new_red_render_queue_element_max_size) {
974 red_render_queue_element_max_size_ = new_red_render_queue_element_max_size;
975
976 std::vector<float> template_queue_element(
977 red_render_queue_element_max_size_);
978
979 red_render_signal_queue_.reset(
980 new SwapQueue<std::vector<float>, RenderQueueItemVerifier<float>>(
981 kMaxNumFramesToBuffer, template_queue_element,
982 RenderQueueItemVerifier<float>(
983 red_render_queue_element_max_size_)));
984
985 red_render_queue_buffer_.resize(red_render_queue_element_max_size_);
986 red_capture_queue_buffer_.resize(red_render_queue_element_max_size_);
987 } else {
988 red_render_signal_queue_->Clear();
989 }
peah764e3642016-10-22 05:04:30 -0700990}
991
992void AudioProcessingImpl::EmptyQueuedRenderAudio() {
993 rtc::CritScope cs_capture(&crit_capture_);
saza1d600522019-10-18 13:29:43 +0200994 if (submodules_.echo_control_mobile) {
Per Åhgrenb6e24d72019-04-29 12:14:50 +0200995 RTC_DCHECK(aecm_render_signal_queue_);
996 while (aecm_render_signal_queue_->Remove(&aecm_capture_queue_buffer_)) {
saza1d600522019-10-18 13:29:43 +0200997 submodules_.echo_control_mobile->ProcessRenderAudio(
Per Åhgrenb6e24d72019-04-29 12:14:50 +0200998 aecm_capture_queue_buffer_);
999 }
peah701d6282016-10-25 05:42:20 -07001000 }
1001
Per Åhgren0695df12020-01-13 14:43:13 +01001002 if (submodules_.gain_control) {
1003 while (agc_render_signal_queue_->Remove(&agc_capture_queue_buffer_)) {
1004 submodules_.gain_control->ProcessRenderAudio(agc_capture_queue_buffer_);
1005 }
peah764e3642016-10-22 05:04:30 -07001006 }
ivoc9f4a4a02016-10-28 05:39:16 -07001007
1008 while (red_render_signal_queue_->Remove(&red_capture_queue_buffer_)) {
saza1d600522019-10-18 13:29:43 +02001009 RTC_DCHECK(submodules_.echo_detector);
1010 submodules_.echo_detector->AnalyzeRenderAudio(red_capture_queue_buffer_);
ivoc9f4a4a02016-10-28 05:39:16 -07001011 }
peah764e3642016-10-22 05:04:30 -07001012}
1013
Per Åhgren645f24c2020-03-16 12:06:02 +01001014int AudioProcessingImpl::ProcessStream(const int16_t* const src,
1015 const StreamConfig& input_config,
1016 const StreamConfig& output_config,
Per Åhgrendc5522b2020-03-19 14:55:58 +01001017 int16_t* const dest) {
Per Åhgren71652f42020-03-17 13:23:58 +01001018 TRACE_EVENT0("webrtc", "AudioProcessing::ProcessStream_AudioFrame");
Sam Zackrisson12e319a2020-01-03 14:54:20 +01001019 RETURN_ON_ERR(MaybeInitializeCapture(input_config, output_config));
Oskar Sundbom4b276482019-05-23 14:28:00 +02001020
peahdf3efa82015-11-28 12:35:15 -08001021 rtc::CritScope cs_capture(&crit_capture_);
niklase@google.com470e71d2011-07-07 08:21:25 +00001022
aleloi868f32f2017-05-23 07:20:05 -07001023 if (aec_dump_) {
Per Åhgren645f24c2020-03-16 12:06:02 +01001024 RecordUnprocessedCaptureStream(src, input_config);
aleloi868f32f2017-05-23 07:20:05 -07001025 }
1026
Per Åhgren645f24c2020-03-16 12:06:02 +01001027 capture_.capture_audio->CopyFrom(src, input_config);
Gustaf Ullberg3c918b12019-10-11 13:14:44 +02001028 if (capture_.capture_fullband_audio) {
Per Åhgren645f24c2020-03-16 12:06:02 +01001029 capture_.capture_fullband_audio->CopyFrom(src, input_config);
Gustaf Ullberg3c918b12019-10-11 13:14:44 +02001030 }
peahde65ddc2016-09-16 15:02:15 -07001031 RETURN_ON_ERR(ProcessCaptureStreamLocked());
Gustaf Ullberg8675eee2019-10-09 13:34:36 +02001032 if (submodule_states_.CaptureMultiBandProcessingPresent() ||
Per Åhgrena1351272019-08-15 12:15:46 +02001033 submodule_states_.CaptureFullBandProcessingActive()) {
Gustaf Ullberg422b9e02019-10-09 13:02:14 +02001034 if (capture_.capture_fullband_audio) {
Per Åhgren645f24c2020-03-16 12:06:02 +01001035 capture_.capture_fullband_audio->CopyTo(output_config, dest);
Gustaf Ullberg422b9e02019-10-09 13:02:14 +02001036 } else {
Per Åhgren645f24c2020-03-16 12:06:02 +01001037 capture_.capture_audio->CopyTo(output_config, dest);
Gustaf Ullberg422b9e02019-10-09 13:02:14 +02001038 }
Per Åhgrena1351272019-08-15 12:15:46 +02001039 }
Per Åhgren645f24c2020-03-16 12:06:02 +01001040
aleloi868f32f2017-05-23 07:20:05 -07001041 if (aec_dump_) {
Per Åhgren645f24c2020-03-16 12:06:02 +01001042 RecordProcessedCaptureStream(dest, output_config);
aleloi868f32f2017-05-23 07:20:05 -07001043 }
andrew@webrtc.org17e40642014-03-04 20:58:13 +00001044
1045 return kNoError;
1046}
1047
peahde65ddc2016-09-16 15:02:15 -07001048int AudioProcessingImpl::ProcessCaptureStreamLocked() {
Sam Zackrisson12e319a2020-01-03 14:54:20 +01001049 EmptyQueuedRenderAudio();
Alex Loiko73ec0192018-05-15 10:52:28 +02001050 HandleCaptureRuntimeSettings();
Alessio Bazzicac054e782018-04-16 12:10:09 +02001051
peahb58a1582016-03-15 09:34:24 -07001052 // Ensure that not both the AEC and AECM are active at the same time.
Sam Zackrisson2a959d92018-07-23 14:48:07 +00001053 // TODO(peah): Simplify once the public API Enable functions for these
1054 // are moved to APM.
Sam Zackrisson308bc642019-12-23 10:22:08 +01001055 RTC_DCHECK_LE(
1056 !!submodules_.echo_controller + !!submodules_.echo_control_mobile, 1);
peahb58a1582016-03-15 09:34:24 -07001057
peahde65ddc2016-09-16 15:02:15 -07001058 AudioBuffer* capture_buffer = capture_.capture_audio.get(); // For brevity.
Per Åhgren2e8e1c62019-12-20 00:42:22 +01001059 AudioBuffer* linear_aec_buffer = capture_.linear_aec_output.get();
ekmeyerson60d9b332015-08-14 10:35:55 -07001060
Per Åhgrenc0424252019-12-10 13:04:15 +01001061 if (submodules_.high_pass_filter &&
1062 config_.high_pass_filter.apply_in_full_band &&
1063 !constants_.enforce_split_band_hpf) {
1064 submodules_.high_pass_filter->Process(capture_buffer,
1065 /*use_split_band_data=*/false);
1066 }
1067
saza1d600522019-10-18 13:29:43 +02001068 if (submodules_.pre_amplifier) {
1069 submodules_.pre_amplifier->ApplyGain(AudioFrameView<float>(
Per Åhgrend47941e2019-08-22 11:51:13 +02001070 capture_buffer->channels(), capture_buffer->num_channels(),
Alex Loikob5c9a792018-04-16 16:31:22 +02001071 capture_buffer->num_frames()));
1072 }
1073
Per Åhgren928146f2019-08-20 09:19:21 +02001074 capture_input_rms_.Analyze(rtc::ArrayView<const float>(
Per Åhgrend47941e2019-08-22 11:51:13 +02001075 capture_buffer->channels_const()[0],
henrik.lundin290d43a2016-11-29 08:09:09 -08001076 capture_nonlocked_.capture_processing_format.num_frames()));
peah1b08dc32016-12-20 13:45:58 -08001077 const bool log_rms = ++capture_rms_interval_counter_ >= 1000;
1078 if (log_rms) {
1079 capture_rms_interval_counter_ = 0;
1080 RmsLevel::Levels levels = capture_input_rms_.AverageAndPeak();
henrik.lundin45bb5132016-12-06 04:28:04 -08001081 RTC_HISTOGRAM_COUNTS_LINEAR("WebRTC.Audio.ApmCaptureInputLevelAverageRms",
1082 levels.average, 1, RmsLevel::kMinLevelDb, 64);
1083 RTC_HISTOGRAM_COUNTS_LINEAR("WebRTC.Audio.ApmCaptureInputLevelPeakRms",
1084 levels.peak, 1, RmsLevel::kMinLevelDb, 64);
henrik.lundin290d43a2016-11-29 08:09:09 -08001085 }
1086
saza1d600522019-10-18 13:29:43 +02001087 if (submodules_.echo_controller) {
Per Åhgren88cf0502018-07-16 17:08:41 +02001088 // Detect and flag any change in the analog gain.
Per Åhgren0e3198e2019-11-18 08:52:22 +01001089 int analog_mic_level = recommended_stream_analog_level();
Per Åhgren88cf0502018-07-16 17:08:41 +02001090 capture_.echo_path_gain_change =
1091 capture_.prev_analog_mic_level != analog_mic_level &&
1092 capture_.prev_analog_mic_level != -1;
1093 capture_.prev_analog_mic_level = analog_mic_level;
1094
Per Åhgrend2650d12018-10-02 17:00:59 +02001095 // Detect and flag any change in the pre-amplifier gain.
saza1d600522019-10-18 13:29:43 +02001096 if (submodules_.pre_amplifier) {
1097 float pre_amp_gain = submodules_.pre_amplifier->GetGainFactor();
Per Åhgrend2650d12018-10-02 17:00:59 +02001098 capture_.echo_path_gain_change =
1099 capture_.echo_path_gain_change ||
1100 (capture_.prev_pre_amp_gain != pre_amp_gain &&
Per Åhgrene8a55692018-10-02 23:10:38 +02001101 capture_.prev_pre_amp_gain >= 0.f);
Per Åhgrend2650d12018-10-02 17:00:59 +02001102 capture_.prev_pre_amp_gain = pre_amp_gain;
1103 }
Fredrik Hernqvistca362852019-05-10 15:50:02 +02001104
1105 // Detect volume change.
1106 capture_.echo_path_gain_change =
1107 capture_.echo_path_gain_change ||
1108 (capture_.prev_playout_volume != capture_.playout_volume &&
1109 capture_.prev_playout_volume >= 0);
1110 capture_.prev_playout_volume = capture_.playout_volume;
1111
saza1d600522019-10-18 13:29:43 +02001112 submodules_.echo_controller->AnalyzeCapture(capture_buffer);
peahe0eae3c2016-12-14 01:16:23 -08001113 }
1114
Per Åhgren0695df12020-01-13 14:43:13 +01001115 if (submodules_.agc_manager) {
Per Åhgren3daedb62019-11-22 12:11:40 +01001116 submodules_.agc_manager->AnalyzePreProcess(capture_buffer);
pbos@webrtc.org788acd12014-12-15 09:41:24 +00001117 }
1118
peah2ace3f92016-09-10 04:42:27 -07001119 if (submodule_states_.CaptureMultiBandSubModulesActive() &&
1120 SampleRateSupportsMultiBand(
peahde65ddc2016-09-16 15:02:15 -07001121 capture_nonlocked_.capture_processing_format.sample_rate_hz())) {
1122 capture_buffer->SplitIntoFrequencyBands();
niklase@google.com470e71d2011-07-07 08:21:25 +00001123 }
1124
Per Åhgrene14cb992019-11-27 09:34:22 +01001125 const bool multi_channel_capture = config_.pipeline.multi_channel_capture &&
1126 constants_.multi_channel_capture_support;
1127 if (submodules_.echo_controller && !multi_channel_capture) {
peah522d71b2017-02-23 05:16:26 -08001128 // Force down-mixing of the number of channels after the detection of
1129 // capture signal saturation.
1130 // TODO(peah): Look into ensuring that this kind of tampering with the
1131 // AudioBuffer functionality should not be needed.
1132 capture_buffer->set_num_channels(1);
1133 }
1134
Per Åhgrenc0424252019-12-10 13:04:15 +01001135 if (submodules_.high_pass_filter &&
1136 (!config_.high_pass_filter.apply_in_full_band ||
1137 constants_.enforce_split_band_hpf)) {
1138 submodules_.high_pass_filter->Process(capture_buffer,
1139 /*use_split_band_data=*/true);
peah8271d042016-11-22 07:24:52 -08001140 }
Per Åhgren0cbb58e2019-10-29 22:59:44 +01001141
Per Åhgren0695df12020-01-13 14:43:13 +01001142 if (submodules_.gain_control) {
1143 RETURN_ON_ERR(
1144 submodules_.gain_control->AnalyzeCaptureAudio(*capture_buffer));
1145 }
Per Åhgren2e8e1c62019-12-20 00:42:22 +01001146
Per Åhgren8ad9e742020-01-30 07:40:58 +01001147 if ((!config_.noise_suppression.analyze_linear_aec_output_when_available ||
1148 !linear_aec_buffer || submodules_.echo_control_mobile) &&
1149 submodules_.noise_suppressor) {
1150 submodules_.noise_suppressor->Analyze(*capture_buffer);
saza0bad15f2019-10-16 11:46:11 +02001151 }
peahb58a1582016-03-15 09:34:24 -07001152
saza1d600522019-10-18 13:29:43 +02001153 if (submodules_.echo_control_mobile) {
Per Åhgrenb6e24d72019-04-29 12:14:50 +02001154 // Ensure that the stream delay was set before the call to the
1155 // AECM ProcessCaptureAudio function.
Sam Zackrisson2d31aea2020-01-17 10:55:09 +01001156 if (!capture_.was_stream_delay_set) {
Per Åhgrenb6e24d72019-04-29 12:14:50 +02001157 return AudioProcessing::kStreamParameterNotSetError;
Per Åhgrend0fa8202018-04-18 09:35:13 +02001158 }
1159
saza1d600522019-10-18 13:29:43 +02001160 if (submodules_.noise_suppressor) {
Per Åhgren0cbb58e2019-10-29 22:59:44 +01001161 submodules_.noise_suppressor->Process(capture_buffer);
Per Åhgrenb6e24d72019-04-29 12:14:50 +02001162 }
peahe0eae3c2016-12-14 01:16:23 -08001163
saza1d600522019-10-18 13:29:43 +02001164 RETURN_ON_ERR(submodules_.echo_control_mobile->ProcessCaptureAudio(
Per Åhgren46537a32017-06-07 10:08:10 +02001165 capture_buffer, stream_delay_ms()));
Per Åhgrenb6e24d72019-04-29 12:14:50 +02001166 } else {
saza1d600522019-10-18 13:29:43 +02001167 if (submodules_.echo_controller) {
Per Åhgrenb6e24d72019-04-29 12:14:50 +02001168 data_dumper_->DumpRaw("stream_delay", stream_delay_ms());
1169
Sam Zackrisson2d31aea2020-01-17 10:55:09 +01001170 if (capture_.was_stream_delay_set) {
saza1d600522019-10-18 13:29:43 +02001171 submodules_.echo_controller->SetAudioBufferDelay(stream_delay_ms());
Per Åhgrenb6e24d72019-04-29 12:14:50 +02001172 }
1173
saza1d600522019-10-18 13:29:43 +02001174 submodules_.echo_controller->ProcessCapture(
Per Åhgrenc20a19c2019-11-13 11:12:29 +01001175 capture_buffer, linear_aec_buffer, capture_.echo_path_gain_change);
Per Åhgrenb6e24d72019-04-29 12:14:50 +02001176 }
1177
Per Åhgren2e8e1c62019-12-20 00:42:22 +01001178 if (config_.noise_suppression.analyze_linear_aec_output_when_available &&
Per Åhgren8ad9e742020-01-30 07:40:58 +01001179 linear_aec_buffer && submodules_.noise_suppressor) {
1180 submodules_.noise_suppressor->Analyze(*linear_aec_buffer);
Per Åhgren2e8e1c62019-12-20 00:42:22 +01001181 }
1182
saza1d600522019-10-18 13:29:43 +02001183 if (submodules_.noise_suppressor) {
Per Åhgren0cbb58e2019-10-29 22:59:44 +01001184 submodules_.noise_suppressor->Process(capture_buffer);
saza0bad15f2019-10-16 11:46:11 +02001185 }
Per Åhgren46537a32017-06-07 10:08:10 +02001186 }
ivoc9f4a4a02016-10-28 05:39:16 -07001187
Sam Zackrisson4db667b2018-12-21 16:29:27 +01001188 if (config_.voice_detection.enabled) {
Sam Zackrisson4db667b2018-12-21 16:29:27 +01001189 capture_.stats.voice_detected =
saza1d600522019-10-18 13:29:43 +02001190 submodules_.voice_detector->ProcessCaptureAudio(capture_buffer);
Sam Zackrisson4db667b2018-12-21 16:29:27 +01001191 } else {
1192 capture_.stats.voice_detected = absl::nullopt;
1193 }
pbos@webrtc.org788acd12014-12-15 09:41:24 +00001194
Per Åhgren0695df12020-01-13 14:43:13 +01001195 if (submodules_.agc_manager) {
Per Åhgren3daedb62019-11-22 12:11:40 +01001196 submodules_.agc_manager->Process(capture_buffer);
1197
1198 absl::optional<int> new_digital_gain =
1199 submodules_.agc_manager->GetDigitalComressionGain();
Per Åhgren0695df12020-01-13 14:43:13 +01001200 if (new_digital_gain && submodules_.gain_control) {
Per Åhgren3daedb62019-11-22 12:11:40 +01001201 submodules_.gain_control->set_compression_gain_db(*new_digital_gain);
1202 }
pbos@webrtc.org788acd12014-12-15 09:41:24 +00001203 }
Per Åhgren0695df12020-01-13 14:43:13 +01001204
1205 if (submodules_.gain_control) {
1206 // TODO(peah): Add reporting from AEC3 whether there is echo.
1207 RETURN_ON_ERR(submodules_.gain_control->ProcessCaptureAudio(
1208 capture_buffer, /*stream_has_echo*/ false));
1209 }
niklase@google.com470e71d2011-07-07 08:21:25 +00001210
Gustaf Ullberg8675eee2019-10-09 13:34:36 +02001211 if (submodule_states_.CaptureMultiBandProcessingPresent() &&
peah2ace3f92016-09-10 04:42:27 -07001212 SampleRateSupportsMultiBand(
peahde65ddc2016-09-16 15:02:15 -07001213 capture_nonlocked_.capture_processing_format.sample_rate_hz())) {
1214 capture_buffer->MergeFrequencyBands();
niklase@google.com470e71d2011-07-07 08:21:25 +00001215 }
1216
Gustaf Ullberg422b9e02019-10-09 13:02:14 +02001217 if (capture_.capture_fullband_audio) {
saza1d600522019-10-18 13:29:43 +02001218 const auto& ec = submodules_.echo_controller;
Gustaf Ullberg8675eee2019-10-09 13:34:36 +02001219 bool ec_active = ec ? ec->ActiveProcessing() : false;
1220 // Only update the fullband buffer if the multiband processing has changed
1221 // the signal. Keep the original signal otherwise.
1222 if (submodule_states_.CaptureMultiBandProcessingActive(ec_active)) {
1223 capture_buffer->CopyTo(capture_.capture_fullband_audio.get());
1224 }
Gustaf Ullberg422b9e02019-10-09 13:02:14 +02001225 capture_buffer = capture_.capture_fullband_audio.get();
1226 }
1227
peah9e6a2902017-05-15 07:19:21 -07001228 if (config_.residual_echo_detector.enabled) {
saza1d600522019-10-18 13:29:43 +02001229 RTC_DCHECK(submodules_.echo_detector);
1230 submodules_.echo_detector->AnalyzeCaptureAudio(rtc::ArrayView<const float>(
1231 capture_buffer->channels()[0], capture_buffer->num_frames()));
peah9e6a2902017-05-15 07:19:21 -07001232 }
1233
pbos@webrtc.org788acd12014-12-15 09:41:24 +00001234 // TODO(aluebs): Investigate if the transient suppression placement should be
1235 // before or after the AGC.
Per Åhgrenc0734712020-01-02 15:15:36 +01001236 if (submodules_.transient_suppressor) {
saza1d600522019-10-18 13:29:43 +02001237 float voice_probability = submodules_.agc_manager.get()
1238 ? submodules_.agc_manager->voice_probability()
1239 : 1.f;
pbos@webrtc.org788acd12014-12-15 09:41:24 +00001240
saza1d600522019-10-18 13:29:43 +02001241 submodules_.transient_suppressor->Suppress(
Per Åhgrend47941e2019-08-22 11:51:13 +02001242 capture_buffer->channels()[0], capture_buffer->num_frames(),
peahde65ddc2016-09-16 15:02:15 -07001243 capture_buffer->num_channels(),
Per Åhgrend47941e2019-08-22 11:51:13 +02001244 capture_buffer->split_bands_const(0)[kBand0To8kHz],
Per Åhgrena1351272019-08-15 12:15:46 +02001245 capture_buffer->num_frames_per_band(),
1246 capture_.keyboard_info.keyboard_data,
1247 capture_.keyboard_info.num_keyboard_frames, voice_probability,
peahdf3efa82015-11-28 12:35:15 -08001248 capture_.key_pressed);
pbos@webrtc.org788acd12014-12-15 09:41:24 +00001249 }
1250
Valeriia Nemychnikovaf06eb572018-08-29 10:37:09 +02001251 // Experimental APM sub-module that analyzes |capture_buffer|.
saza1d600522019-10-18 13:29:43 +02001252 if (submodules_.capture_analyzer) {
1253 submodules_.capture_analyzer->Analyze(capture_buffer);
Valeriia Nemychnikovaf06eb572018-08-29 10:37:09 +02001254 }
1255
Per Åhgren2bd85ab2020-01-03 10:36:34 +01001256 if (submodules_.gain_controller2) {
saza1d600522019-10-18 13:29:43 +02001257 submodules_.gain_controller2->NotifyAnalogLevel(
Per Åhgren0e3198e2019-11-18 08:52:22 +01001258 recommended_stream_analog_level());
saza1d600522019-10-18 13:29:43 +02001259 submodules_.gain_controller2->Process(capture_buffer);
alessiob3ec96df2017-05-22 06:57:06 -07001260 }
1261
saza1d600522019-10-18 13:29:43 +02001262 if (submodules_.capture_post_processor) {
1263 submodules_.capture_post_processor->Process(capture_buffer);
Sam Zackrisson0beac582017-09-25 12:04:02 +02001264 }
1265
andrew@webrtc.org755b04a2011-11-15 16:57:56 +00001266 // The level estimator operates on the recombined data.
Sam Zackrissonb24c00f2018-11-26 16:18:25 +01001267 if (config_.level_estimation.enabled) {
saza1d600522019-10-18 13:29:43 +02001268 submodules_.output_level_estimator->ProcessStream(*capture_buffer);
1269 capture_.stats.output_rms_dbfs = submodules_.output_level_estimator->RMS();
Sam Zackrissonb24c00f2018-11-26 16:18:25 +01001270 } else {
1271 capture_.stats.output_rms_dbfs = absl::nullopt;
1272 }
ajm@google.com808e0e02011-08-03 21:08:51 +00001273
Per Åhgren928146f2019-08-20 09:19:21 +02001274 capture_output_rms_.Analyze(rtc::ArrayView<const float>(
Per Åhgrend47941e2019-08-22 11:51:13 +02001275 capture_buffer->channels_const()[0],
peah1b08dc32016-12-20 13:45:58 -08001276 capture_nonlocked_.capture_processing_format.num_frames()));
1277 if (log_rms) {
1278 RmsLevel::Levels levels = capture_output_rms_.AverageAndPeak();
1279 RTC_HISTOGRAM_COUNTS_LINEAR("WebRTC.Audio.ApmCaptureOutputLevelAverageRms",
1280 levels.average, 1, RmsLevel::kMinLevelDb, 64);
1281 RTC_HISTOGRAM_COUNTS_LINEAR("WebRTC.Audio.ApmCaptureOutputLevelPeakRms",
1282 levels.peak, 1, RmsLevel::kMinLevelDb, 64);
1283 }
1284
Per Åhgren0e3198e2019-11-18 08:52:22 +01001285 if (submodules_.agc_manager) {
1286 int level = recommended_stream_analog_level();
1287 data_dumper_->DumpRaw("experimental_gain_control_stream_analog_level", 1,
1288 &level);
1289 }
1290
Per Åhgrencf4c8722019-12-30 14:32:14 +01001291 // Compute echo-related stats.
1292 if (submodules_.echo_controller) {
1293 auto ec_metrics = submodules_.echo_controller->GetMetrics();
1294 capture_.stats.echo_return_loss = ec_metrics.echo_return_loss;
1295 capture_.stats.echo_return_loss_enhancement =
1296 ec_metrics.echo_return_loss_enhancement;
1297 capture_.stats.delay_ms = ec_metrics.delay_ms;
1298 }
1299 if (config_.residual_echo_detector.enabled) {
1300 RTC_DCHECK(submodules_.echo_detector);
1301 auto ed_metrics = submodules_.echo_detector->GetMetrics();
1302 capture_.stats.residual_echo_likelihood = ed_metrics.echo_likelihood;
1303 capture_.stats.residual_echo_likelihood_recent_max =
1304 ed_metrics.echo_likelihood_recent_max;
1305 }
1306
1307 // Pass stats for reporting.
1308 stats_reporter_.UpdateStatistics(capture_.stats);
1309
peahdf3efa82015-11-28 12:35:15 -08001310 capture_.was_stream_delay_set = false;
niklase@google.com470e71d2011-07-07 08:21:25 +00001311 return kNoError;
1312}
1313
Gustaf Ullberg8c51f2e2019-10-22 15:21:31 +02001314int AudioProcessingImpl::AnalyzeReverseStream(
1315 const float* const* data,
1316 const StreamConfig& reverse_config) {
1317 TRACE_EVENT0("webrtc", "AudioProcessing::AnalyzeReverseStream_StreamConfig");
1318 rtc::CritScope cs(&crit_render_);
1319 return AnalyzeReverseStreamLocked(data, reverse_config, reverse_config);
1320}
1321
peahde65ddc2016-09-16 15:02:15 -07001322int AudioProcessingImpl::ProcessReverseStream(const float* const* src,
1323 const StreamConfig& input_config,
1324 const StreamConfig& output_config,
1325 float* const* dest) {
peah369f8282015-12-17 06:42:29 -08001326 TRACE_EVENT0("webrtc", "AudioProcessing::ProcessReverseStream_StreamConfig");
peahdf3efa82015-11-28 12:35:15 -08001327 rtc::CritScope cs(&crit_render_);
peahde65ddc2016-09-16 15:02:15 -07001328 RETURN_ON_ERR(AnalyzeReverseStreamLocked(src, input_config, output_config));
Alex Loiko5825aa62017-12-18 16:02:40 +01001329 if (submodule_states_.RenderMultiBandProcessingActive() ||
1330 submodule_states_.RenderFullBandProcessingActive()) {
peahdf3efa82015-11-28 12:35:15 -08001331 render_.render_audio->CopyTo(formats_.api_format.reverse_output_stream(),
1332 dest);
peah2ace3f92016-09-10 04:42:27 -07001333 } else if (formats_.api_format.reverse_input_stream() !=
1334 formats_.api_format.reverse_output_stream()) {
peahde65ddc2016-09-16 15:02:15 -07001335 render_.render_converter->Convert(src, input_config.num_samples(), dest,
1336 output_config.num_samples());
ekmeyerson60d9b332015-08-14 10:35:55 -07001337 } else {
peahde65ddc2016-09-16 15:02:15 -07001338 CopyAudioIfNeeded(src, input_config.num_frames(),
1339 input_config.num_channels(), dest);
ekmeyerson60d9b332015-08-14 10:35:55 -07001340 }
1341
1342 return kNoError;
Michael Graczyk86c6d332015-07-23 11:41:39 -07001343}
1344
peahdf3efa82015-11-28 12:35:15 -08001345int AudioProcessingImpl::AnalyzeReverseStreamLocked(
ekmeyerson60d9b332015-08-14 10:35:55 -07001346 const float* const* src,
peahde65ddc2016-09-16 15:02:15 -07001347 const StreamConfig& input_config,
1348 const StreamConfig& output_config) {
peahdf3efa82015-11-28 12:35:15 -08001349 if (src == nullptr) {
andrew@webrtc.org17e40642014-03-04 20:58:13 +00001350 return kNullPointerError;
1351 }
andrew@webrtc.org17e40642014-03-04 20:58:13 +00001352
peahde65ddc2016-09-16 15:02:15 -07001353 if (input_config.num_channels() == 0) {
Michael Graczyk86c6d332015-07-23 11:41:39 -07001354 return kBadNumberChannelsError;
andrew@webrtc.org17e40642014-03-04 20:58:13 +00001355 }
1356
peahdf3efa82015-11-28 12:35:15 -08001357 ProcessingConfig processing_config = formats_.api_format;
peahde65ddc2016-09-16 15:02:15 -07001358 processing_config.reverse_input_stream() = input_config;
1359 processing_config.reverse_output_stream() = output_config;
Michael Graczyk86c6d332015-07-23 11:41:39 -07001360
peahdf3efa82015-11-28 12:35:15 -08001361 RETURN_ON_ERR(MaybeInitializeRender(processing_config));
Fredrik Solenbergbbf21a32018-04-12 22:44:09 +02001362 RTC_DCHECK_EQ(input_config.num_frames(),
1363 formats_.api_format.reverse_input_stream().num_frames());
Michael Graczyk86c6d332015-07-23 11:41:39 -07001364
aleloi868f32f2017-05-23 07:20:05 -07001365 if (aec_dump_) {
1366 const size_t channel_size =
1367 formats_.api_format.reverse_input_stream().num_frames();
1368 const size_t num_channels =
1369 formats_.api_format.reverse_input_stream().num_channels();
1370 aec_dump_->WriteRenderStreamMessage(
Alex Loikoe36e8bb2018-02-16 11:54:07 +01001371 AudioFrameView<const float>(src, num_channels, channel_size));
aleloi868f32f2017-05-23 07:20:05 -07001372 }
peahdf3efa82015-11-28 12:35:15 -08001373 render_.render_audio->CopyFrom(src,
1374 formats_.api_format.reverse_input_stream());
peahde65ddc2016-09-16 15:02:15 -07001375 return ProcessRenderStreamLocked();
ekmeyerson60d9b332015-08-14 10:35:55 -07001376}
1377
Per Åhgren645f24c2020-03-16 12:06:02 +01001378int AudioProcessingImpl::ProcessReverseStream(const int16_t* const src,
1379 const StreamConfig& input_config,
1380 const StreamConfig& output_config,
1381 int16_t* const dest) {
Per Åhgren71652f42020-03-17 13:23:58 +01001382 TRACE_EVENT0("webrtc", "AudioProcessing::ProcessReverseStream_AudioFrame");
Per Åhgren2507f8c2020-03-19 12:33:29 +01001383
1384 if (input_config.num_channels() <= 0) {
1385 return AudioProcessing::Error::kBadNumberChannelsError;
1386 }
1387
Per Åhgren645f24c2020-03-16 12:06:02 +01001388 rtc::CritScope cs(&crit_render_);
peahdf3efa82015-11-28 12:35:15 -08001389 ProcessingConfig processing_config = formats_.api_format;
ekmeyerson60d9b332015-08-14 10:35:55 -07001390 processing_config.reverse_input_stream().set_sample_rate_hz(
Per Åhgren645f24c2020-03-16 12:06:02 +01001391 input_config.sample_rate_hz());
ekmeyerson60d9b332015-08-14 10:35:55 -07001392 processing_config.reverse_input_stream().set_num_channels(
Per Åhgren645f24c2020-03-16 12:06:02 +01001393 input_config.num_channels());
ekmeyerson60d9b332015-08-14 10:35:55 -07001394 processing_config.reverse_output_stream().set_sample_rate_hz(
Per Åhgren645f24c2020-03-16 12:06:02 +01001395 output_config.sample_rate_hz());
ekmeyerson60d9b332015-08-14 10:35:55 -07001396 processing_config.reverse_output_stream().set_num_channels(
Per Åhgren645f24c2020-03-16 12:06:02 +01001397 output_config.num_channels());
Michael Graczyk86c6d332015-07-23 11:41:39 -07001398
peahdf3efa82015-11-28 12:35:15 -08001399 RETURN_ON_ERR(MaybeInitializeRender(processing_config));
Per Åhgren645f24c2020-03-16 12:06:02 +01001400 if (input_config.num_frames() !=
peahdf3efa82015-11-28 12:35:15 -08001401 formats_.api_format.reverse_input_stream().num_frames()) {
andrew@webrtc.org17e40642014-03-04 20:58:13 +00001402 return kBadDataLengthError;
1403 }
niklase@google.com470e71d2011-07-07 08:21:25 +00001404
aleloi868f32f2017-05-23 07:20:05 -07001405 if (aec_dump_) {
Per Åhgren645f24c2020-03-16 12:06:02 +01001406 aec_dump_->WriteRenderStreamMessage(src, input_config.num_frames(),
1407 input_config.num_channels());
aleloi868f32f2017-05-23 07:20:05 -07001408 }
1409
Per Åhgren645f24c2020-03-16 12:06:02 +01001410 render_.render_audio->CopyFrom(src, input_config);
peahde65ddc2016-09-16 15:02:15 -07001411 RETURN_ON_ERR(ProcessRenderStreamLocked());
Per Åhgrena1351272019-08-15 12:15:46 +02001412 if (submodule_states_.RenderMultiBandProcessingActive() ||
1413 submodule_states_.RenderFullBandProcessingActive()) {
Per Åhgren645f24c2020-03-16 12:06:02 +01001414 render_.render_audio->CopyTo(output_config, dest);
Per Åhgrena1351272019-08-15 12:15:46 +02001415 }
aluebsb0319552016-03-17 20:39:53 -07001416 return kNoError;
andrew@webrtc.org17e40642014-03-04 20:58:13 +00001417}
niklase@google.com470e71d2011-07-07 08:21:25 +00001418
peahde65ddc2016-09-16 15:02:15 -07001419int AudioProcessingImpl::ProcessRenderStreamLocked() {
1420 AudioBuffer* render_buffer = render_.render_audio.get(); // For brevity.
peah9e6a2902017-05-15 07:19:21 -07001421
Alex Loiko73ec0192018-05-15 10:52:28 +02001422 HandleRenderRuntimeSettings();
1423
saza1d600522019-10-18 13:29:43 +02001424 if (submodules_.render_pre_processor) {
1425 submodules_.render_pre_processor->Process(render_buffer);
Alex Loiko5825aa62017-12-18 16:02:40 +01001426 }
1427
Alessio Bazzicad2b97402018-08-09 14:23:11 +02001428 QueueNonbandedRenderAudio(render_buffer);
1429
peah2ace3f92016-09-10 04:42:27 -07001430 if (submodule_states_.RenderMultiBandSubModulesActive() &&
peahde65ddc2016-09-16 15:02:15 -07001431 SampleRateSupportsMultiBand(
1432 formats_.render_processing_format.sample_rate_hz())) {
1433 render_buffer->SplitIntoFrequencyBands();
niklase@google.com470e71d2011-07-07 08:21:25 +00001434 }
1435
peahce4d9152017-05-19 01:28:05 -07001436 if (submodule_states_.RenderMultiBandSubModulesActive()) {
1437 QueueBandedRenderAudio(render_buffer);
1438 }
1439
Alessio Bazzicad2b97402018-08-09 14:23:11 +02001440 // TODO(peah): Perform the queuing inside QueueRenderAudiuo().
saza1d600522019-10-18 13:29:43 +02001441 if (submodules_.echo_controller) {
1442 submodules_.echo_controller->AnalyzeRender(render_buffer);
peahe0eae3c2016-12-14 01:16:23 -08001443 }
niklase@google.com470e71d2011-07-07 08:21:25 +00001444
peah2ace3f92016-09-10 04:42:27 -07001445 if (submodule_states_.RenderMultiBandProcessingActive() &&
peahde65ddc2016-09-16 15:02:15 -07001446 SampleRateSupportsMultiBand(
1447 formats_.render_processing_format.sample_rate_hz())) {
1448 render_buffer->MergeFrequencyBands();
ekmeyerson60d9b332015-08-14 10:35:55 -07001449 }
1450
andrew@webrtc.org17e40642014-03-04 20:58:13 +00001451 return kNoError;
niklase@google.com470e71d2011-07-07 08:21:25 +00001452}
1453
1454int AudioProcessingImpl::set_stream_delay_ms(int delay) {
peahdf3efa82015-11-28 12:35:15 -08001455 rtc::CritScope cs(&crit_capture_);
andrew@webrtc.org5f23d642012-05-29 21:14:06 +00001456 Error retval = kNoError;
peahdf3efa82015-11-28 12:35:15 -08001457 capture_.was_stream_delay_set = true;
andrew@webrtc.org6f9f8172012-03-06 19:03:39 +00001458
niklase@google.com470e71d2011-07-07 08:21:25 +00001459 if (delay < 0) {
andrew@webrtc.org5f23d642012-05-29 21:14:06 +00001460 delay = 0;
1461 retval = kBadStreamParameterWarning;
niklase@google.com470e71d2011-07-07 08:21:25 +00001462 }
1463
1464 // TODO(ajm): the max is rather arbitrarily chosen; investigate.
1465 if (delay > 500) {
andrew@webrtc.org5f23d642012-05-29 21:14:06 +00001466 delay = 500;
1467 retval = kBadStreamParameterWarning;
niklase@google.com470e71d2011-07-07 08:21:25 +00001468 }
1469
peahdf3efa82015-11-28 12:35:15 -08001470 capture_nonlocked_.stream_delay_ms = delay;
andrew@webrtc.org5f23d642012-05-29 21:14:06 +00001471 return retval;
niklase@google.com470e71d2011-07-07 08:21:25 +00001472}
1473
Per Åhgrenc20a19c2019-11-13 11:12:29 +01001474bool AudioProcessingImpl::GetLinearAecOutput(
1475 rtc::ArrayView<std::array<float, 160>> linear_output) const {
1476 rtc::CritScope cs(&crit_capture_);
1477 AudioBuffer* linear_aec_buffer = capture_.linear_aec_output.get();
1478
1479 RTC_DCHECK(linear_aec_buffer);
1480 if (linear_aec_buffer) {
1481 RTC_DCHECK_EQ(1, linear_aec_buffer->num_bands());
1482 RTC_DCHECK_EQ(linear_output.size(), linear_aec_buffer->num_channels());
1483
1484 for (size_t ch = 0; ch < linear_aec_buffer->num_channels(); ++ch) {
1485 RTC_DCHECK_EQ(linear_output[ch].size(), linear_aec_buffer->num_frames());
1486 rtc::ArrayView<const float> channel_view =
1487 rtc::ArrayView<const float>(linear_aec_buffer->channels_const()[ch],
1488 linear_aec_buffer->num_frames());
1489 std::copy(channel_view.begin(), channel_view.end(),
1490 linear_output[ch].begin());
1491 }
1492 return true;
1493 }
1494 RTC_LOG(LS_ERROR) << "No linear AEC output available";
1495 RTC_NOTREACHED();
1496 return false;
1497}
1498
niklase@google.com470e71d2011-07-07 08:21:25 +00001499int AudioProcessingImpl::stream_delay_ms() const {
peahdf3efa82015-11-28 12:35:15 -08001500 // Used as callback from submodules, hence locking is not allowed.
1501 return capture_nonlocked_.stream_delay_ms;
niklase@google.com470e71d2011-07-07 08:21:25 +00001502}
1503
andrew@webrtc.org17e40642014-03-04 20:58:13 +00001504void AudioProcessingImpl::set_stream_key_pressed(bool key_pressed) {
peahdf3efa82015-11-28 12:35:15 -08001505 rtc::CritScope cs(&crit_capture_);
1506 capture_.key_pressed = key_pressed;
andrew@webrtc.org17e40642014-03-04 20:58:13 +00001507}
1508
Sam Zackrissonf0d1c032019-03-27 13:28:08 +01001509void AudioProcessingImpl::set_stream_analog_level(int level) {
1510 rtc::CritScope cs_capture(&crit_capture_);
Per Åhgren0e3198e2019-11-18 08:52:22 +01001511
1512 if (submodules_.agc_manager) {
1513 submodules_.agc_manager->set_stream_analog_level(level);
1514 data_dumper_->DumpRaw("experimental_gain_control_set_stream_analog_level",
1515 1, &level);
Per Åhgren0695df12020-01-13 14:43:13 +01001516 } else if (submodules_.gain_control) {
Per Åhgren0e3198e2019-11-18 08:52:22 +01001517 int error = submodules_.gain_control->set_stream_analog_level(level);
1518 RTC_DCHECK_EQ(kNoError, error);
Per Åhgren0695df12020-01-13 14:43:13 +01001519 } else {
1520 capture_.cached_stream_analog_level_ = level;
Per Åhgren0e3198e2019-11-18 08:52:22 +01001521 }
Sam Zackrissonf0d1c032019-03-27 13:28:08 +01001522}
1523
1524int AudioProcessingImpl::recommended_stream_analog_level() const {
1525 rtc::CritScope cs_capture(&crit_capture_);
Per Åhgren0e3198e2019-11-18 08:52:22 +01001526 if (submodules_.agc_manager) {
1527 return submodules_.agc_manager->stream_analog_level();
Per Åhgren0695df12020-01-13 14:43:13 +01001528 } else if (submodules_.gain_control) {
1529 return submodules_.gain_control->stream_analog_level();
1530 } else {
1531 return capture_.cached_stream_analog_level_;
Per Åhgren0e3198e2019-11-18 08:52:22 +01001532 }
Sam Zackrissonf0d1c032019-03-27 13:28:08 +01001533}
1534
aleloi868f32f2017-05-23 07:20:05 -07001535void AudioProcessingImpl::AttachAecDump(std::unique_ptr<AecDump> aec_dump) {
1536 RTC_DCHECK(aec_dump);
1537 rtc::CritScope cs_render(&crit_render_);
1538 rtc::CritScope cs_capture(&crit_capture_);
1539
1540 // The previously attached AecDump will be destroyed with the
1541 // 'aec_dump' parameter, which is after locks are released.
1542 aec_dump_.swap(aec_dump);
1543 WriteAecDumpConfigMessage(true);
Minyue Li656d6092018-08-10 15:38:52 +02001544 aec_dump_->WriteInitMessage(formats_.api_format, rtc::TimeUTCMillis());
aleloi868f32f2017-05-23 07:20:05 -07001545}
1546
1547void AudioProcessingImpl::DetachAecDump() {
1548 // The d-tor of a task-queue based AecDump blocks until all pending
1549 // tasks are done. This construction avoids blocking while holding
1550 // the render and capture locks.
1551 std::unique_ptr<AecDump> aec_dump = nullptr;
1552 {
1553 rtc::CritScope cs_render(&crit_render_);
1554 rtc::CritScope cs_capture(&crit_capture_);
1555 aec_dump = std::move(aec_dump_);
1556 }
1557}
1558
Sam Zackrisson4d364492018-03-02 16:03:21 +01001559void AudioProcessingImpl::AttachPlayoutAudioGenerator(
1560 std::unique_ptr<AudioGenerator> audio_generator) {
1561 // TODO(bugs.webrtc.org/8882) Stub.
1562 // Reset internal audio generator with audio_generator.
1563}
1564
1565void AudioProcessingImpl::DetachPlayoutAudioGenerator() {
1566 // TODO(bugs.webrtc.org/8882) Stub.
1567 // Delete audio generator, if one is attached.
1568}
1569
peah8271d042016-11-22 07:24:52 -08001570void AudioProcessingImpl::MutateConfig(
1571 rtc::FunctionView<void(AudioProcessing::Config*)> mutator) {
1572 rtc::CritScope cs_render(&crit_render_);
1573 rtc::CritScope cs_capture(&crit_capture_);
1574 mutator(&config_);
1575 ApplyConfig(config_);
1576}
1577
1578AudioProcessing::Config AudioProcessingImpl::GetConfig() const {
1579 rtc::CritScope cs_render(&crit_render_);
1580 rtc::CritScope cs_capture(&crit_capture_);
1581 return config_;
1582}
1583
peah2ace3f92016-09-10 04:42:27 -07001584bool AudioProcessingImpl::UpdateActiveSubmoduleStates() {
1585 return submodule_states_.Update(
Per Åhgren62ea0aa2019-12-09 10:18:44 +01001586 config_.high_pass_filter.enabled, !!submodules_.echo_control_mobile,
Per Åhgren8ad9e742020-01-30 07:40:58 +01001587 config_.residual_echo_detector.enabled, !!submodules_.noise_suppressor,
Per Åhgren0695df12020-01-13 14:43:13 +01001588 !!submodules_.gain_control, !!submodules_.gain_controller2,
Per Åhgren0cbb58e2019-10-29 22:59:44 +01001589 config_.pre_amplifier.enabled, capture_nonlocked_.echo_controller_enabled,
Per Åhgrenc0734712020-01-02 15:15:36 +01001590 config_.voice_detection.enabled, !!submodules_.transient_suppressor);
ekmeyerson60d9b332015-08-14 10:35:55 -07001591}
1592
Per Åhgrenc0734712020-01-02 15:15:36 +01001593void AudioProcessingImpl::InitializeTransientSuppressor() {
1594 if (config_.transient_suppression.enabled) {
sazaaa42ecd2020-04-01 15:24:40 +02001595 // Attempt to create a transient suppressor, if one is not already created.
Per Åhgrenc0734712020-01-02 15:15:36 +01001596 if (!submodules_.transient_suppressor) {
Sam Zackrissonb37e59d2020-04-27 08:39:33 +02001597 submodules_.transient_suppressor =
1598 CreateTransientSuppressor(submodule_creation_overrides_);
pbos@webrtc.org788acd12014-12-15 09:41:24 +00001599 }
sazaaa42ecd2020-04-01 15:24:40 +02001600 if (submodules_.transient_suppressor) {
1601 submodules_.transient_suppressor->Initialize(
1602 proc_fullband_sample_rate_hz(), capture_nonlocked_.split_rate,
1603 num_proc_channels());
1604 } else {
1605 RTC_LOG(LS_WARNING)
1606 << "No transient suppressor created (probably disabled)";
1607 }
Per Åhgrenc0734712020-01-02 15:15:36 +01001608 } else {
1609 submodules_.transient_suppressor.reset();
pbos@webrtc.org788acd12014-12-15 09:41:24 +00001610 }
pbos@webrtc.org788acd12014-12-15 09:41:24 +00001611}
1612
Per Åhgren0f14db22020-01-03 14:27:14 +01001613void AudioProcessingImpl::InitializeHighPassFilter(bool forced_reset) {
Per Åhgrenb8106462019-12-04 08:34:12 +01001614 bool high_pass_filter_needed_by_aec =
1615 config_.echo_canceller.enabled &&
1616 config_.echo_canceller.enforce_high_pass_filtering &&
1617 !config_.echo_canceller.mobile_mode;
1618 if (submodule_states_.HighPassFilteringRequired() ||
1619 high_pass_filter_needed_by_aec) {
Per Åhgrenc0424252019-12-10 13:04:15 +01001620 bool use_full_band = config_.high_pass_filter.apply_in_full_band &&
1621 !constants_.enforce_split_band_hpf;
1622 int rate = use_full_band ? proc_fullband_sample_rate_hz()
1623 : proc_split_sample_rate_hz();
1624 size_t num_channels =
1625 use_full_band ? num_output_channels() : num_proc_channels();
1626
Per Åhgren0f14db22020-01-03 14:27:14 +01001627 if (!submodules_.high_pass_filter ||
1628 rate != submodules_.high_pass_filter->sample_rate_hz() ||
1629 forced_reset ||
1630 num_channels != submodules_.high_pass_filter->num_channels()) {
1631 submodules_.high_pass_filter.reset(
1632 new HighPassFilter(rate, num_channels));
1633 }
peah8271d042016-11-22 07:24:52 -08001634 } else {
saza1d600522019-10-18 13:29:43 +02001635 submodules_.high_pass_filter.reset();
peah8271d042016-11-22 07:24:52 -08001636 }
1637}
alessiob3ec96df2017-05-22 06:57:06 -07001638
Sam Zackrisson0824c6f2019-10-07 14:03:56 +02001639void AudioProcessingImpl::InitializeVoiceDetector() {
1640 if (config_.voice_detection.enabled) {
saza1d600522019-10-18 13:29:43 +02001641 submodules_.voice_detector = std::make_unique<VoiceDetection>(
Sam Zackrisson0824c6f2019-10-07 14:03:56 +02001642 proc_split_sample_rate_hz(), VoiceDetection::kVeryLowLikelihood);
1643 } else {
saza1d600522019-10-18 13:29:43 +02001644 submodules_.voice_detector.reset();
Sam Zackrisson0824c6f2019-10-07 14:03:56 +02001645 }
1646}
Gustaf Ullberg8eb9c7d2017-10-14 08:28:46 +02001647void AudioProcessingImpl::InitializeEchoController() {
Per Åhgrenb6e24d72019-04-29 12:14:50 +02001648 bool use_echo_controller =
1649 echo_control_factory_ ||
Per Åhgren62ea0aa2019-12-09 10:18:44 +01001650 (config_.echo_canceller.enabled && !config_.echo_canceller.mobile_mode);
Per Åhgrenb6e24d72019-04-29 12:14:50 +02001651
1652 if (use_echo_controller) {
1653 // Create and activate the echo controller.
Per Åhgren200feba2019-03-06 04:16:46 +01001654 if (echo_control_factory_) {
Per Åhgren4e5c7092019-11-01 20:44:11 +01001655 submodules_.echo_controller = echo_control_factory_->Create(
1656 proc_sample_rate_hz(), num_reverse_channels(), num_proc_channels());
Gustaf Ullberg2c6f3732019-11-07 17:15:12 +01001657 RTC_DCHECK(submodules_.echo_controller);
Per Åhgren200feba2019-03-06 04:16:46 +01001658 } else {
Per Åhgrenb2b58d82019-12-02 14:59:40 +01001659 EchoCanceller3Config config =
1660 use_setup_specific_default_aec3_config_
1661 ? EchoCanceller3::CreateDefaultConfig(num_reverse_channels(),
1662 num_proc_channels())
1663 : EchoCanceller3Config();
saza1d600522019-10-18 13:29:43 +02001664 submodules_.echo_controller = std::make_unique<EchoCanceller3>(
Per Åhgrenb2b58d82019-12-02 14:59:40 +01001665 config, proc_sample_rate_hz(), num_reverse_channels(),
Sam Zackrissonfeee1e42019-09-20 07:50:35 +02001666 num_proc_channels());
Per Åhgren200feba2019-03-06 04:16:46 +01001667 }
1668
Per Åhgrenc20a19c2019-11-13 11:12:29 +01001669 // Setup the storage for returning the linear AEC output.
1670 if (config_.echo_canceller.export_linear_aec_output) {
1671 constexpr int kLinearOutputRateHz = 16000;
1672 capture_.linear_aec_output = std::make_unique<AudioBuffer>(
1673 kLinearOutputRateHz, num_proc_channels(), kLinearOutputRateHz,
1674 num_proc_channels(), kLinearOutputRateHz, num_proc_channels());
1675 } else {
1676 capture_.linear_aec_output.reset();
1677 }
1678
Per Åhgren200feba2019-03-06 04:16:46 +01001679 capture_nonlocked_.echo_controller_enabled = true;
Per Åhgren200feba2019-03-06 04:16:46 +01001680
saza1d600522019-10-18 13:29:43 +02001681 submodules_.echo_control_mobile.reset();
Per Åhgrenb6e24d72019-04-29 12:14:50 +02001682 aecm_render_signal_queue_.reset();
Per Åhgrenf204faf2019-04-25 15:18:06 +02001683 return;
peahe0eae3c2016-12-14 01:16:23 -08001684 }
Per Åhgrenf204faf2019-04-25 15:18:06 +02001685
saza1d600522019-10-18 13:29:43 +02001686 submodules_.echo_controller.reset();
Per Åhgrenf204faf2019-04-25 15:18:06 +02001687 capture_nonlocked_.echo_controller_enabled = false;
Per Åhgrenc20a19c2019-11-13 11:12:29 +01001688 capture_.linear_aec_output.reset();
Per Åhgrenf204faf2019-04-25 15:18:06 +02001689
1690 if (!config_.echo_canceller.enabled) {
saza1d600522019-10-18 13:29:43 +02001691 submodules_.echo_control_mobile.reset();
Per Åhgrenb6e24d72019-04-29 12:14:50 +02001692 aecm_render_signal_queue_.reset();
Per Åhgrenf204faf2019-04-25 15:18:06 +02001693 return;
1694 }
1695
1696 if (config_.echo_canceller.mobile_mode) {
1697 // Create and activate AECM.
Per Åhgrenb6e24d72019-04-29 12:14:50 +02001698 size_t max_element_size =
1699 std::max(static_cast<size_t>(1),
1700 kMaxAllowedValuesOfSamplesPerBand *
1701 EchoControlMobileImpl::NumCancellersRequired(
1702 num_output_channels(), num_reverse_channels()));
1703
1704 std::vector<int16_t> template_queue_element(max_element_size);
1705
1706 aecm_render_signal_queue_.reset(
1707 new SwapQueue<std::vector<int16_t>, RenderQueueItemVerifier<int16_t>>(
1708 kMaxNumFramesToBuffer, template_queue_element,
1709 RenderQueueItemVerifier<int16_t>(max_element_size)));
1710
1711 aecm_render_queue_buffer_.resize(max_element_size);
1712 aecm_capture_queue_buffer_.resize(max_element_size);
1713
saza1d600522019-10-18 13:29:43 +02001714 submodules_.echo_control_mobile.reset(new EchoControlMobileImpl());
Per Åhgrenb6e24d72019-04-29 12:14:50 +02001715
saza1d600522019-10-18 13:29:43 +02001716 submodules_.echo_control_mobile->Initialize(proc_split_sample_rate_hz(),
1717 num_reverse_channels(),
1718 num_output_channels());
Per Åhgrenf204faf2019-04-25 15:18:06 +02001719 return;
1720 }
1721
saza1d600522019-10-18 13:29:43 +02001722 submodules_.echo_control_mobile.reset();
Per Åhgrenb6e24d72019-04-29 12:14:50 +02001723 aecm_render_signal_queue_.reset();
peahe0eae3c2016-12-14 01:16:23 -08001724}
peah8271d042016-11-22 07:24:52 -08001725
Per Åhgren0695df12020-01-13 14:43:13 +01001726void AudioProcessingImpl::InitializeGainController1() {
1727 if (!config_.gain_controller1.enabled) {
1728 submodules_.agc_manager.reset();
1729 submodules_.gain_control.reset();
1730 return;
1731 }
1732
1733 if (!submodules_.gain_control) {
1734 submodules_.gain_control.reset(new GainControlImpl());
1735 }
1736
1737 submodules_.gain_control->Initialize(num_proc_channels(),
1738 proc_sample_rate_hz());
1739
1740 if (!config_.gain_controller1.analog_gain_controller.enabled) {
1741 int error = submodules_.gain_control->set_mode(
1742 Agc1ConfigModeToInterfaceMode(config_.gain_controller1.mode));
1743 RTC_DCHECK_EQ(kNoError, error);
1744 error = submodules_.gain_control->set_target_level_dbfs(
1745 config_.gain_controller1.target_level_dbfs);
1746 RTC_DCHECK_EQ(kNoError, error);
1747 error = submodules_.gain_control->set_compression_gain_db(
1748 config_.gain_controller1.compression_gain_db);
1749 RTC_DCHECK_EQ(kNoError, error);
1750 error = submodules_.gain_control->enable_limiter(
1751 config_.gain_controller1.enable_limiter);
1752 RTC_DCHECK_EQ(kNoError, error);
1753 error = submodules_.gain_control->set_analog_level_limits(
1754 config_.gain_controller1.analog_level_minimum,
1755 config_.gain_controller1.analog_level_maximum);
1756 RTC_DCHECK_EQ(kNoError, error);
1757
1758 submodules_.agc_manager.reset();
1759 return;
1760 }
1761
1762 if (!submodules_.agc_manager.get() ||
1763 submodules_.agc_manager->num_channels() !=
1764 static_cast<int>(num_proc_channels()) ||
1765 submodules_.agc_manager->sample_rate_hz() !=
1766 capture_nonlocked_.split_rate) {
1767 int stream_analog_level = -1;
1768 const bool re_creation = !!submodules_.agc_manager;
1769 if (re_creation) {
1770 stream_analog_level = submodules_.agc_manager->stream_analog_level();
1771 }
1772 submodules_.agc_manager.reset(new AgcManagerDirect(
1773 num_proc_channels(),
1774 config_.gain_controller1.analog_gain_controller.startup_min_volume,
1775 config_.gain_controller1.analog_gain_controller.clipped_level_min,
1776 config_.gain_controller1.analog_gain_controller
1777 .enable_agc2_level_estimator,
1778 !config_.gain_controller1.analog_gain_controller
1779 .enable_digital_adaptive,
1780 capture_nonlocked_.split_rate));
1781 if (re_creation) {
1782 submodules_.agc_manager->set_stream_analog_level(stream_analog_level);
1783 }
1784 }
1785 submodules_.agc_manager->Initialize();
1786 submodules_.agc_manager->SetupDigitalGainControl(
1787 submodules_.gain_control.get());
1788 submodules_.agc_manager->SetCaptureMuted(capture_.output_will_be_muted);
1789}
1790
alessiob3ec96df2017-05-22 06:57:06 -07001791void AudioProcessingImpl::InitializeGainController2() {
Alessio Bazzica270f7b52017-10-13 11:05:17 +02001792 if (config_.gain_controller2.enabled) {
Per Åhgren2bd85ab2020-01-03 10:36:34 +01001793 if (!submodules_.gain_controller2) {
1794 // TODO(alessiob): Move the injected gain controller once injection is
1795 // implemented.
1796 submodules_.gain_controller2.reset(new GainController2());
1797 }
1798
saza1d600522019-10-18 13:29:43 +02001799 submodules_.gain_controller2->Initialize(proc_fullband_sample_rate_hz());
Per Åhgren2bd85ab2020-01-03 10:36:34 +01001800 submodules_.gain_controller2->ApplyConfig(config_.gain_controller2);
1801 } else {
1802 submodules_.gain_controller2.reset();
alessiob3ec96df2017-05-22 06:57:06 -07001803 }
1804}
1805
saza0bad15f2019-10-16 11:46:11 +02001806void AudioProcessingImpl::InitializeNoiseSuppressor() {
Per Åhgren0cbb58e2019-10-29 22:59:44 +01001807 submodules_.noise_suppressor.reset();
1808
saza0bad15f2019-10-16 11:46:11 +02001809 if (config_.noise_suppression.enabled) {
sazaaa42ecd2020-04-01 15:24:40 +02001810 auto map_level =
1811 [](AudioProcessing::Config::NoiseSuppression::Level level) {
1812 using NoiseSuppresionConfig =
1813 AudioProcessing::Config::NoiseSuppression;
1814 switch (level) {
1815 case NoiseSuppresionConfig::kLow:
1816 return NsConfig::SuppressionLevel::k6dB;
1817 case NoiseSuppresionConfig::kModerate:
1818 return NsConfig::SuppressionLevel::k12dB;
1819 case NoiseSuppresionConfig::kHigh:
1820 return NsConfig::SuppressionLevel::k18dB;
1821 case NoiseSuppresionConfig::kVeryHigh:
1822 return NsConfig::SuppressionLevel::k21dB;
1823 default:
1824 RTC_NOTREACHED();
1825 }
1826 };
Per Åhgren0cbb58e2019-10-29 22:59:44 +01001827
sazaaa42ecd2020-04-01 15:24:40 +02001828 NsConfig cfg;
1829 cfg.target_level = map_level(config_.noise_suppression.level);
1830 submodules_.noise_suppressor = std::make_unique<NoiseSuppressor>(
1831 cfg, proc_sample_rate_hz(), num_proc_channels());
saza0bad15f2019-10-16 11:46:11 +02001832 }
1833}
1834
Alex Loikob5c9a792018-04-16 16:31:22 +02001835void AudioProcessingImpl::InitializePreAmplifier() {
1836 if (config_.pre_amplifier.enabled) {
saza1d600522019-10-18 13:29:43 +02001837 submodules_.pre_amplifier.reset(
Alex Loikob5c9a792018-04-16 16:31:22 +02001838 new GainApplier(true, config_.pre_amplifier.fixed_gain_factor));
1839 } else {
saza1d600522019-10-18 13:29:43 +02001840 submodules_.pre_amplifier.reset();
Alex Loikob5c9a792018-04-16 16:31:22 +02001841 }
1842}
1843
ivoc9f4a4a02016-10-28 05:39:16 -07001844void AudioProcessingImpl::InitializeResidualEchoDetector() {
saza1d600522019-10-18 13:29:43 +02001845 RTC_DCHECK(submodules_.echo_detector);
1846 submodules_.echo_detector->Initialize(
Gustaf Ullberg422b9e02019-10-09 13:02:14 +02001847 proc_fullband_sample_rate_hz(), 1,
Ivo Creusenb1facc12018-04-12 16:15:58 +02001848 formats_.render_processing_format.sample_rate_hz(), 1);
ivoc9f4a4a02016-10-28 05:39:16 -07001849}
1850
Valeriia Nemychnikovaf06eb572018-08-29 10:37:09 +02001851void AudioProcessingImpl::InitializeAnalyzer() {
saza1d600522019-10-18 13:29:43 +02001852 if (submodules_.capture_analyzer) {
1853 submodules_.capture_analyzer->Initialize(proc_fullband_sample_rate_hz(),
1854 num_proc_channels());
Valeriia Nemychnikovaf06eb572018-08-29 10:37:09 +02001855 }
1856}
1857
Sam Zackrisson0beac582017-09-25 12:04:02 +02001858void AudioProcessingImpl::InitializePostProcessor() {
saza1d600522019-10-18 13:29:43 +02001859 if (submodules_.capture_post_processor) {
1860 submodules_.capture_post_processor->Initialize(
Gustaf Ullberg422b9e02019-10-09 13:02:14 +02001861 proc_fullband_sample_rate_hz(), num_proc_channels());
Sam Zackrisson0beac582017-09-25 12:04:02 +02001862 }
1863}
1864
Alex Loiko5825aa62017-12-18 16:02:40 +01001865void AudioProcessingImpl::InitializePreProcessor() {
saza1d600522019-10-18 13:29:43 +02001866 if (submodules_.render_pre_processor) {
1867 submodules_.render_pre_processor->Initialize(
Alex Loiko5825aa62017-12-18 16:02:40 +01001868 formats_.render_processing_format.sample_rate_hz(),
1869 formats_.render_processing_format.num_channels());
1870 }
1871}
1872
aleloi868f32f2017-05-23 07:20:05 -07001873void AudioProcessingImpl::WriteAecDumpConfigMessage(bool forced) {
1874 if (!aec_dump_) {
1875 return;
1876 }
Per Åhgrenf204faf2019-04-25 15:18:06 +02001877
1878 std::string experiments_description = "";
aleloi868f32f2017-05-23 07:20:05 -07001879 // TODO(peah): Add semicolon-separated concatenations of experiment
1880 // descriptions for other submodules.
Per Åhgren0695df12020-01-13 14:43:13 +01001881 if (config_.gain_controller1.analog_gain_controller.clipped_level_min !=
1882 kClippedLevelMin) {
aleloi868f32f2017-05-23 07:20:05 -07001883 experiments_description += "AgcClippingLevelExperiment;";
1884 }
Sam Zackrisson701bd172020-02-18 14:50:28 +01001885 if (!!submodules_.capture_post_processor) {
1886 experiments_description += "CapturePostProcessor;";
1887 }
1888 if (!!submodules_.render_pre_processor) {
1889 experiments_description += "RenderPreProcessor;";
1890 }
Gustaf Ullbergce045ac2017-10-16 13:49:04 +02001891 if (capture_nonlocked_.echo_controller_enabled) {
1892 experiments_description += "EchoController;";
aleloi868f32f2017-05-23 07:20:05 -07001893 }
Alessio Bazzica270f7b52017-10-13 11:05:17 +02001894 if (config_.gain_controller2.enabled) {
1895 experiments_description += "GainController2;";
1896 }
aleloi868f32f2017-05-23 07:20:05 -07001897
1898 InternalAPMConfig apm_config;
1899
Per Åhgren200feba2019-03-06 04:16:46 +01001900 apm_config.aec_enabled = config_.echo_canceller.enabled;
Per Åhgren62ea0aa2019-12-09 10:18:44 +01001901 apm_config.aec_delay_agnostic_enabled = false;
1902 apm_config.aec_extended_filter_enabled = false;
1903 apm_config.aec_suppression_level = 0;
aleloi868f32f2017-05-23 07:20:05 -07001904
saza1d600522019-10-18 13:29:43 +02001905 apm_config.aecm_enabled = !!submodules_.echo_control_mobile;
aleloi868f32f2017-05-23 07:20:05 -07001906 apm_config.aecm_comfort_noise_enabled =
saza1d600522019-10-18 13:29:43 +02001907 submodules_.echo_control_mobile &&
1908 submodules_.echo_control_mobile->is_comfort_noise_enabled();
Per Åhgrenb6e24d72019-04-29 12:14:50 +02001909 apm_config.aecm_routing_mode =
saza1d600522019-10-18 13:29:43 +02001910 submodules_.echo_control_mobile
1911 ? static_cast<int>(submodules_.echo_control_mobile->routing_mode())
Per Åhgrenb6e24d72019-04-29 12:14:50 +02001912 : 0;
aleloi868f32f2017-05-23 07:20:05 -07001913
Per Åhgren0695df12020-01-13 14:43:13 +01001914 apm_config.agc_enabled = !!submodules_.gain_control;
1915
1916 apm_config.agc_mode = submodules_.gain_control
1917 ? static_cast<int>(submodules_.gain_control->mode())
1918 : GainControl::kAdaptiveAnalog;
aleloi868f32f2017-05-23 07:20:05 -07001919 apm_config.agc_limiter_enabled =
Per Åhgren0695df12020-01-13 14:43:13 +01001920 submodules_.gain_control ? submodules_.gain_control->is_limiter_enabled()
1921 : false;
Per Åhgren0e3198e2019-11-18 08:52:22 +01001922 apm_config.noise_robust_agc_enabled = !!submodules_.agc_manager;
aleloi868f32f2017-05-23 07:20:05 -07001923
1924 apm_config.hpf_enabled = config_.high_pass_filter.enabled;
1925
saza0bad15f2019-10-16 11:46:11 +02001926 apm_config.ns_enabled = config_.noise_suppression.enabled;
1927 apm_config.ns_level = static_cast<int>(config_.noise_suppression.level);
aleloi868f32f2017-05-23 07:20:05 -07001928
1929 apm_config.transient_suppression_enabled =
Per Åhgrenc0734712020-01-02 15:15:36 +01001930 config_.transient_suppression.enabled;
aleloi868f32f2017-05-23 07:20:05 -07001931 apm_config.experiments_description = experiments_description;
Alex Loiko5feb30e2018-04-16 13:52:32 +02001932 apm_config.pre_amplifier_enabled = config_.pre_amplifier.enabled;
1933 apm_config.pre_amplifier_fixed_gain_factor =
1934 config_.pre_amplifier.fixed_gain_factor;
aleloi868f32f2017-05-23 07:20:05 -07001935
1936 if (!forced && apm_config == apm_config_for_aec_dump_) {
1937 return;
1938 }
1939 aec_dump_->WriteConfig(apm_config);
1940 apm_config_for_aec_dump_ = apm_config;
1941}
1942
1943void AudioProcessingImpl::RecordUnprocessedCaptureStream(
1944 const float* const* src) {
1945 RTC_DCHECK(aec_dump_);
1946 WriteAecDumpConfigMessage(false);
1947
1948 const size_t channel_size = formats_.api_format.input_stream().num_frames();
1949 const size_t num_channels = formats_.api_format.input_stream().num_channels();
1950 aec_dump_->AddCaptureStreamInput(
Alex Loikoe36e8bb2018-02-16 11:54:07 +01001951 AudioFrameView<const float>(src, num_channels, channel_size));
aleloi868f32f2017-05-23 07:20:05 -07001952 RecordAudioProcessingState();
1953}
1954
1955void AudioProcessingImpl::RecordUnprocessedCaptureStream(
Per Åhgren645f24c2020-03-16 12:06:02 +01001956 const int16_t* const data,
1957 const StreamConfig& config) {
aleloi868f32f2017-05-23 07:20:05 -07001958 RTC_DCHECK(aec_dump_);
1959 WriteAecDumpConfigMessage(false);
1960
Per Åhgren645f24c2020-03-16 12:06:02 +01001961 aec_dump_->AddCaptureStreamInput(data, config.num_channels(),
1962 config.num_frames());
aleloi868f32f2017-05-23 07:20:05 -07001963 RecordAudioProcessingState();
1964}
1965
1966void AudioProcessingImpl::RecordProcessedCaptureStream(
1967 const float* const* processed_capture_stream) {
1968 RTC_DCHECK(aec_dump_);
1969
1970 const size_t channel_size = formats_.api_format.output_stream().num_frames();
1971 const size_t num_channels =
1972 formats_.api_format.output_stream().num_channels();
Alex Loikoe36e8bb2018-02-16 11:54:07 +01001973 aec_dump_->AddCaptureStreamOutput(AudioFrameView<const float>(
1974 processed_capture_stream, num_channels, channel_size));
aleloi868f32f2017-05-23 07:20:05 -07001975 aec_dump_->WriteCaptureStreamMessage();
1976}
1977
1978void AudioProcessingImpl::RecordProcessedCaptureStream(
Per Åhgren645f24c2020-03-16 12:06:02 +01001979 const int16_t* const data,
1980 const StreamConfig& config) {
aleloi868f32f2017-05-23 07:20:05 -07001981 RTC_DCHECK(aec_dump_);
1982
Per Åhgren645f24c2020-03-16 12:06:02 +01001983 aec_dump_->AddCaptureStreamOutput(data, config.num_channels(),
Per Åhgren088329f2020-03-18 21:59:52 +01001984 config.num_frames());
aleloi868f32f2017-05-23 07:20:05 -07001985 aec_dump_->WriteCaptureStreamMessage();
1986}
1987
1988void AudioProcessingImpl::RecordAudioProcessingState() {
1989 RTC_DCHECK(aec_dump_);
1990 AecDump::AudioProcessingState audio_proc_state;
1991 audio_proc_state.delay = capture_nonlocked_.stream_delay_ms;
Per Åhgren62ea0aa2019-12-09 10:18:44 +01001992 audio_proc_state.drift = 0;
Per Åhgren0e3198e2019-11-18 08:52:22 +01001993 audio_proc_state.level = recommended_stream_analog_level();
aleloi868f32f2017-05-23 07:20:05 -07001994 audio_proc_state.keypress = capture_.key_pressed;
1995 aec_dump_->AddAudioProcessingState(audio_proc_state);
1996}
1997
Per Åhgrenc0734712020-01-02 15:15:36 +01001998AudioProcessingImpl::ApmCaptureState::ApmCaptureState()
Sam Zackrisson2d31aea2020-01-17 10:55:09 +01001999 : was_stream_delay_set(false),
kwiberg83ffe452016-08-29 14:46:07 -07002000 output_will_be_muted(false),
2001 key_pressed(false),
peahde65ddc2016-09-16 15:02:15 -07002002 capture_processing_format(kSampleRate16kHz),
peah67995532017-04-10 14:12:41 -07002003 split_rate(kSampleRate16kHz),
Per Åhgren88cf0502018-07-16 17:08:41 +02002004 echo_path_gain_change(false),
Per Åhgrend2650d12018-10-02 17:00:59 +02002005 prev_analog_mic_level(-1),
Fredrik Hernqvistca362852019-05-10 15:50:02 +02002006 prev_pre_amp_gain(-1.f),
2007 playout_volume(-1),
2008 prev_playout_volume(-1) {}
kwiberg83ffe452016-08-29 14:46:07 -07002009
2010AudioProcessingImpl::ApmCaptureState::~ApmCaptureState() = default;
2011
Per Åhgrena1351272019-08-15 12:15:46 +02002012void AudioProcessingImpl::ApmCaptureState::KeyboardInfo::Extract(
2013 const float* const* data,
2014 const StreamConfig& stream_config) {
2015 if (stream_config.has_keyboard()) {
2016 keyboard_data = data[stream_config.num_channels()];
2017 } else {
2018 keyboard_data = NULL;
2019 }
2020 num_keyboard_frames = stream_config.num_frames();
2021}
2022
kwiberg83ffe452016-08-29 14:46:07 -07002023AudioProcessingImpl::ApmRenderState::ApmRenderState() = default;
2024
2025AudioProcessingImpl::ApmRenderState::~ApmRenderState() = default;
2026
Per Åhgrencf4c8722019-12-30 14:32:14 +01002027AudioProcessingImpl::ApmStatsReporter::ApmStatsReporter()
2028 : stats_message_queue_(1) {}
2029
2030AudioProcessingImpl::ApmStatsReporter::~ApmStatsReporter() = default;
2031
2032AudioProcessingStats AudioProcessingImpl::ApmStatsReporter::GetStatistics() {
2033 rtc::CritScope cs_stats(&crit_stats_);
2034 bool new_stats_available = stats_message_queue_.Remove(&cached_stats_);
2035 // If the message queue is full, return the cached stats.
2036 static_cast<void>(new_stats_available);
2037
2038 return cached_stats_;
2039}
2040
2041void AudioProcessingImpl::ApmStatsReporter::UpdateStatistics(
2042 const AudioProcessingStats& new_stats) {
2043 AudioProcessingStats stats_to_queue = new_stats;
2044 bool stats_message_passed = stats_message_queue_.Insert(&stats_to_queue);
2045 // If the message queue is full, discard the new stats.
2046 static_cast<void>(stats_message_passed);
2047}
2048
niklase@google.com470e71d2011-07-07 08:21:25 +00002049} // namespace webrtc