This CL adds the basic framework for AEC3 in the audio processing module.
It will be followed by a number of other CLs that extends this framework.
BUG=webrtc:6018
Review-Url: https://codereview.webrtc.org/2567513003
Cr-Commit-Position: refs/heads/master@{#15593}
diff --git a/webrtc/modules/audio_processing/audio_processing_impl.cc b/webrtc/modules/audio_processing/audio_processing_impl.cc
index 5ca9275..ce71461 100644
--- a/webrtc/modules/audio_processing/audio_processing_impl.cc
+++ b/webrtc/modules/audio_processing/audio_processing_impl.cc
@@ -20,6 +20,7 @@
#include "webrtc/common_audio/include/audio_util.h"
#include "webrtc/common_audio/signal_processing/include/signal_processing_library.h"
#include "webrtc/modules/audio_processing/aec/aec_core.h"
+#include "webrtc/modules/audio_processing/aec3/echo_canceller3.h"
#include "webrtc/modules/audio_processing/agc/agc_manager_direct.h"
#include "webrtc/modules/audio_processing/audio_buffer.h"
#include "webrtc/modules/audio_processing/beamformer/nonlinear_beamformer.h"
@@ -167,6 +168,7 @@
bool beamformer_enabled,
bool adaptive_gain_controller_enabled,
bool level_controller_enabled,
+ bool echo_canceller3_enabled,
bool voice_activity_detector_enabled,
bool level_estimator_enabled,
bool transient_suppressor_enabled) {
@@ -184,6 +186,7 @@
changed |=
(adaptive_gain_controller_enabled != adaptive_gain_controller_enabled_);
changed |= (level_controller_enabled != level_controller_enabled_);
+ changed |= (echo_canceller3_enabled != echo_canceller3_enabled_);
changed |= (level_estimator_enabled != level_estimator_enabled_);
changed |=
(voice_activity_detector_enabled != voice_activity_detector_enabled_);
@@ -198,6 +201,7 @@
beamformer_enabled_ = beamformer_enabled;
adaptive_gain_controller_enabled_ = adaptive_gain_controller_enabled;
level_controller_enabled_ = level_controller_enabled;
+ echo_canceller3_enabled_ = echo_canceller3_enabled;
level_estimator_enabled_ = level_estimator_enabled;
voice_activity_detector_enabled_ = voice_activity_detector_enabled;
transient_suppressor_enabled_ = transient_suppressor_enabled;
@@ -224,14 +228,15 @@
const {
return low_cut_filter_enabled_ || echo_canceller_enabled_ ||
mobile_echo_controller_enabled_ || noise_suppressor_enabled_ ||
- beamformer_enabled_ || adaptive_gain_controller_enabled_;
+ beamformer_enabled_ || adaptive_gain_controller_enabled_ ||
+ echo_canceller3_enabled_;
}
bool AudioProcessingImpl::ApmSubmoduleStates::RenderMultiBandSubModulesActive()
const {
return RenderMultiBandProcessingActive() || echo_canceller_enabled_ ||
mobile_echo_controller_enabled_ || adaptive_gain_controller_enabled_ ||
- residual_echo_detector_enabled_;
+ residual_echo_detector_enabled_ || echo_canceller3_enabled_;
}
bool AudioProcessingImpl::ApmSubmoduleStates::RenderMultiBandProcessingActive()
@@ -271,6 +276,7 @@
std::unique_ptr<LowCutFilter> low_cut_filter;
std::unique_ptr<LevelController> level_controller;
std::unique_ptr<ResidualEchoDetector> residual_echo_detector;
+ std::unique_ptr<EchoCanceller3> echo_canceller3;
};
AudioProcessing* AudioProcessing::Create() {
@@ -433,10 +439,18 @@
}
int AudioProcessingImpl::InitializeLocked() {
- const int capture_audiobuffer_num_channels =
- capture_nonlocked_.beamformer_enabled
- ? formats_.api_format.input_stream().num_channels()
- : formats_.api_format.output_stream().num_channels();
+ int capture_audiobuffer_num_channels;
+ if (private_submodules_->echo_canceller3) {
+ // TODO(peah): Ensure that the echo canceller can operate on more than one
+ // microphone channel.
+ RTC_DCHECK(!capture_nonlocked_.beamformer_enabled);
+ capture_audiobuffer_num_channels = 1;
+ } else {
+ capture_audiobuffer_num_channels =
+ capture_nonlocked_.beamformer_enabled
+ ? formats_.api_format.input_stream().num_channels()
+ : formats_.api_format.output_stream().num_channels();
+ }
const int render_audiobuffer_num_output_frames =
formats_.api_format.reverse_output_stream().num_frames() == 0
? formats_.render_processing_format.num_frames()
@@ -508,6 +522,7 @@
public_submodules_->level_estimator->Initialize();
InitializeLevelController();
InitializeResidualEchoDetector();
+ InitializeEchoCanceller3();
#ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP
if (debug_dump_.debug_file->is_open()) {
@@ -561,7 +576,9 @@
submodule_states_.RenderMultiBandSubModulesActive());
// TODO(aluebs): Remove this restriction once we figure out why the 3-band
// splitting filter degrades the AEC performance.
- if (render_processing_rate > kSampleRate32kHz) {
+ // TODO(peah): Verify that the band splitting is needed for the AEC3.
+ if (render_processing_rate > kSampleRate32kHz &&
+ !capture_nonlocked_.echo_canceller3_enabled) {
render_processing_rate = submodule_states_.RenderMultiBandProcessingActive()
? kSampleRate32kHz
: kSampleRate16kHz;
@@ -629,6 +646,25 @@
LOG(LS_INFO) << "Highpass filter activated: "
<< config_.high_pass_filter.enabled;
+
+ config_ok = EchoCanceller3::Validate(config_.echo_canceller3);
+ if (!config_ok) {
+ LOG(LS_ERROR) << "AudioProcessing module config error" << std::endl
+ << "echo canceller 3: "
+ << EchoCanceller3::ToString(config_.echo_canceller3)
+ << std::endl
+ << "Reverting to default parameter set";
+ config_.echo_canceller3 = AudioProcessing::Config::EchoCanceller3();
+ }
+
+ if (config.echo_canceller3.enabled !=
+ capture_nonlocked_.echo_canceller3_enabled) {
+ capture_nonlocked_.echo_canceller3_enabled =
+ config_.echo_canceller3.enabled;
+ InitializeEchoCanceller3();
+ LOG(LS_INFO) << "Echo canceller 3 activated: "
+ << capture_nonlocked_.echo_canceller3_enabled;
+ }
}
void AudioProcessingImpl::SetExtraOptions(const webrtc::Config& config) {
@@ -1108,6 +1144,10 @@
levels.peak, 1, RmsLevel::kMinLevelDb, 64);
}
+ if (private_submodules_->echo_canceller3) {
+ private_submodules_->echo_canceller3->AnalyzeCapture(capture_buffer);
+ }
+
if (constants_.use_experimental_agc &&
public_submodules_->gain_control->is_enabled()) {
private_submodules_->agc_manager->AnalyzePreProcess(
@@ -1128,7 +1168,9 @@
capture_buffer->set_num_channels(1);
}
- if (private_submodules_->low_cut_filter) {
+ // TODO(peah): Move the AEC3 low-cut filter to this place.
+ if (private_submodules_->low_cut_filter &&
+ !private_submodules_->echo_canceller3) {
private_submodules_->low_cut_filter->Process(capture_buffer);
}
RETURN_ON_ERR(
@@ -1142,6 +1184,10 @@
return AudioProcessing::kStreamParameterNotSetError;
}
+ if (private_submodules_->echo_canceller3) {
+ private_submodules_->echo_canceller3->ProcessCapture(capture_buffer, false);
+ }
+
RETURN_ON_ERR(public_submodules_->echo_cancellation->ProcessCaptureAudio(
capture_buffer, stream_delay_ms()));
@@ -1381,6 +1427,12 @@
#endif
QueueRenderAudio(render_buffer);
+ // TODO(peah): Perform the queueing ínside QueueRenderAudiuo().
+ if (private_submodules_->echo_canceller3) {
+ if (!private_submodules_->echo_canceller3->AnalyzeRender(render_buffer)) {
+ // TODO(peah): Lock and empty render queue, and try again.
+ }
+ }
if (submodule_states_.RenderMultiBandProcessingActive() &&
SampleRateSupportsMultiBand(
@@ -1604,6 +1656,7 @@
capture_nonlocked_.beamformer_enabled,
public_submodules_->gain_control->is_enabled(),
capture_nonlocked_.level_controller_enabled,
+ capture_nonlocked_.echo_canceller3_enabled,
public_submodules_->voice_detection->is_enabled(),
public_submodules_->level_estimator->is_enabled(),
capture_.transient_suppressor_enabled);
@@ -1652,6 +1705,14 @@
private_submodules_->low_cut_filter.reset();
}
}
+void AudioProcessingImpl::InitializeEchoCanceller3() {
+ if (capture_nonlocked_.echo_canceller3_enabled) {
+ private_submodules_->echo_canceller3.reset(
+ new EchoCanceller3(proc_sample_rate_hz(), true));
+ } else {
+ private_submodules_->echo_canceller3.reset();
+ }
+}
void AudioProcessingImpl::InitializeLevelController() {
private_submodules_->level_controller->Initialize(proc_sample_rate_hz());
@@ -1857,6 +1918,9 @@
if (constants_.agc_clipped_level_min != kClippedLevelMin) {
experiments_description += "AgcClippingLevelExperiment;";
}
+ if (capture_nonlocked_.echo_canceller3_enabled) {
+ experiments_description += "EchoCanceller3;";
+ }
config.set_experiments_description(experiments_description);
std::string serialized_config = config.SerializeAsString();