Add refined handling of the internal scaling of the audio in APM
This CL adds functionality that allows adjusting the audio levels
internally in APM. The main purpose of the functionality is to allow
APM to optionally be moved to an integration that does not provide an
analog gain to control, and the implementation of this has been
tailored specifically to meet the requirements for that.
More specifically, this CL does
-Add a new variant of the pre-amplifier gain that is intended to replace
the pre-amplifier gain (but at the moment can coexist with that). The
main differences with the pre-amplifier gain is that an attenuating
gain is allowed, the gain is applied jointly with any emulated analog
gain, and that its packaging fits better with the post gain.
-Add an emulation of an analog microphone gain. The emulation is
designed to match the analog mic gain functionality in Chrome OS (which
is digital) but should be usable also on other platforms.
-Add a post-gain which is applied after all processing has been applied.
The purpose of this gain is for it to work well with the integration
in ChromeOS, and be used to compensate for the offset that there is
applied on some USB audio devices.
Bug: b/177830918
Change-Id: I0f312996e4088c9bd242a713a703eaaeb17f188a
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/209707
Commit-Queue: Per Åhgren <peah@webrtc.org>
Reviewed-by: Gustaf Ullberg <gustaf@webrtc.org>
Reviewed-by: Alessio Bazzica <alessiob@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#33466}
diff --git a/modules/audio_processing/audio_processing_impl.cc b/modules/audio_processing/audio_processing_impl.cc
index ec426f6..496f985 100644
--- a/modules/audio_processing/audio_processing_impl.cc
+++ b/modules/audio_processing/audio_processing_impl.cc
@@ -23,7 +23,6 @@
#include "common_audio/audio_converter.h"
#include "common_audio/include/audio_util.h"
#include "modules/audio_processing/aec_dump/aec_dump_factory.h"
-#include "modules/audio_processing/agc2/gain_applier.h"
#include "modules/audio_processing/audio_buffer.h"
#include "modules/audio_processing/common.h"
#include "modules/audio_processing/include/audio_frame_view.h"
@@ -145,7 +144,7 @@
bool noise_suppressor_enabled,
bool adaptive_gain_controller_enabled,
bool gain_controller2_enabled,
- bool pre_amplifier_enabled,
+ bool gain_adjustment_enabled,
bool echo_controller_enabled,
bool voice_detector_enabled,
bool transient_suppressor_enabled) {
@@ -159,7 +158,7 @@
changed |=
(adaptive_gain_controller_enabled != adaptive_gain_controller_enabled_);
changed |= (gain_controller2_enabled != gain_controller2_enabled_);
- changed |= (pre_amplifier_enabled_ != pre_amplifier_enabled);
+ changed |= (gain_adjustment_enabled != gain_adjustment_enabled_);
changed |= (echo_controller_enabled != echo_controller_enabled_);
changed |= (voice_detector_enabled != voice_detector_enabled_);
changed |= (transient_suppressor_enabled != transient_suppressor_enabled_);
@@ -170,7 +169,7 @@
noise_suppressor_enabled_ = noise_suppressor_enabled;
adaptive_gain_controller_enabled_ = adaptive_gain_controller_enabled;
gain_controller2_enabled_ = gain_controller2_enabled;
- pre_amplifier_enabled_ = pre_amplifier_enabled;
+ gain_adjustment_enabled_ = gain_adjustment_enabled;
echo_controller_enabled_ = echo_controller_enabled;
voice_detector_enabled_ = voice_detector_enabled;
transient_suppressor_enabled_ = transient_suppressor_enabled;
@@ -202,7 +201,7 @@
bool AudioProcessingImpl::SubmoduleStates::CaptureFullBandProcessingActive()
const {
return gain_controller2_enabled_ || capture_post_processor_enabled_ ||
- pre_amplifier_enabled_;
+ gain_adjustment_enabled_;
}
bool AudioProcessingImpl::SubmoduleStates::CaptureAnalyzerActive() const {
@@ -422,6 +421,7 @@
InitializeAnalyzer();
InitializePostProcessor();
InitializePreProcessor();
+ InitializeCaptureLevelsAdjuster();
if (aec_dump_) {
aec_dump_->WriteInitMessage(formats_.api_format, rtc::TimeUTCMillis());
@@ -563,6 +563,9 @@
config_.pre_amplifier.fixed_gain_factor !=
config.pre_amplifier.fixed_gain_factor;
+ const bool gain_adjustment_config_changed =
+ config_.capture_level_adjustment != config.capture_level_adjustment;
+
config_ = config;
if (aec_config_changed) {
@@ -594,8 +597,8 @@
InitializeGainController2();
}
- if (pre_amplifier_config_changed) {
- InitializePreAmplifier();
+ if (pre_amplifier_config_changed || gain_adjustment_config_changed) {
+ InitializeCaptureLevelsAdjuster();
}
if (config_.level_estimation.enabled && !submodules_.output_level_estimator) {
@@ -688,6 +691,7 @@
case RuntimeSetting::Type::kPlayoutAudioDeviceChange:
return render_runtime_settings_enqueuer_.Enqueue(setting);
case RuntimeSetting::Type::kCapturePreGain:
+ case RuntimeSetting::Type::kCapturePostGain:
case RuntimeSetting::Type::kCaptureCompressionGain:
case RuntimeSetting::Type::kCaptureFixedPostGain:
case RuntimeSetting::Type::kCaptureOutputUsed:
@@ -809,11 +813,41 @@
}
switch (setting.type()) {
case RuntimeSetting::Type::kCapturePreGain:
- if (config_.pre_amplifier.enabled) {
+ if (config_.pre_amplifier.enabled ||
+ config_.capture_level_adjustment.enabled) {
float value;
setting.GetFloat(&value);
- config_.pre_amplifier.fixed_gain_factor = value;
- submodules_.pre_amplifier->SetGainFactor(value);
+ // If the pre-amplifier is used, apply the new gain to the
+ // pre-amplifier regardless if the capture level adjustment is
+ // activated. This approach allows both functionalities to coexist
+ // until they have been properly merged.
+ if (config_.pre_amplifier.enabled) {
+ config_.pre_amplifier.fixed_gain_factor = value;
+ } else {
+ config_.capture_level_adjustment.pre_gain_factor = value;
+ }
+
+ // Use both the pre-amplifier and the capture level adjustment gains
+ // as pre-gains.
+ float gain = 1.f;
+ if (config_.pre_amplifier.enabled) {
+ gain *= config_.pre_amplifier.fixed_gain_factor;
+ }
+ if (config_.capture_level_adjustment.enabled) {
+ gain *= config_.capture_level_adjustment.pre_gain_factor;
+ }
+
+ submodules_.capture_levels_adjuster->SetPreGain(gain);
+ }
+ // TODO(bugs.chromium.org/9138): Log setting handling by Aec Dump.
+ break;
+ case RuntimeSetting::Type::kCapturePostGain:
+ if (config_.capture_level_adjustment.enabled) {
+ float value;
+ setting.GetFloat(&value);
+ config_.capture_level_adjustment.post_gain_factor = value;
+ submodules_.capture_levels_adjuster->SetPostGain(
+ config_.capture_level_adjustment.post_gain_factor);
}
// TODO(bugs.chromium.org/9138): Log setting handling by Aec Dump.
break;
@@ -896,6 +930,7 @@
}
break;
case RuntimeSetting::Type::kCapturePreGain: // fall-through
+ case RuntimeSetting::Type::kCapturePostGain: // fall-through
case RuntimeSetting::Type::kCaptureCompressionGain: // fall-through
case RuntimeSetting::Type::kCaptureFixedPostGain: // fall-through
case RuntimeSetting::Type::kCaptureOutputUsed: // fall-through
@@ -1083,10 +1118,21 @@
/*use_split_band_data=*/false);
}
- if (submodules_.pre_amplifier) {
- submodules_.pre_amplifier->ApplyGain(AudioFrameView<float>(
- capture_buffer->channels(), capture_buffer->num_channels(),
- capture_buffer->num_frames()));
+ if (submodules_.capture_levels_adjuster) {
+ // If the analog mic gain emulation is active, get the emulated analog mic
+ // gain and pass it to the analog gain control functionality.
+ if (config_.capture_level_adjustment.analog_mic_gain_emulation.enabled) {
+ int level = submodules_.capture_levels_adjuster->GetAnalogMicGainLevel();
+ if (submodules_.agc_manager) {
+ submodules_.agc_manager->set_stream_analog_level(level);
+ } else if (submodules_.gain_control) {
+ int error = submodules_.gain_control->set_stream_analog_level(level);
+ RTC_DCHECK_EQ(kNoError, error);
+ }
+ }
+
+ submodules_.capture_levels_adjuster->ApplyPreLevelAdjustment(
+ *capture_buffer);
}
capture_input_rms_.Analyze(rtc::ArrayView<const float>(
@@ -1110,14 +1156,15 @@
capture_.prev_analog_mic_level != -1;
capture_.prev_analog_mic_level = analog_mic_level;
- // Detect and flag any change in the pre-amplifier gain.
- if (submodules_.pre_amplifier) {
- float pre_amp_gain = submodules_.pre_amplifier->GetGainFactor();
+ // Detect and flag any change in the capture level adjustment pre-gain.
+ if (submodules_.capture_levels_adjuster) {
+ float pre_adjustment_gain =
+ submodules_.capture_levels_adjuster->GetPreAdjustmentGain();
capture_.echo_path_gain_change =
capture_.echo_path_gain_change ||
- (capture_.prev_pre_amp_gain != pre_amp_gain &&
- capture_.prev_pre_amp_gain >= 0.f);
- capture_.prev_pre_amp_gain = pre_amp_gain;
+ (capture_.prev_pre_adjustment_gain != pre_adjustment_gain &&
+ capture_.prev_pre_adjustment_gain >= 0.f);
+ capture_.prev_pre_adjustment_gain = pre_adjustment_gain;
}
// Detect volume change.
@@ -1325,6 +1372,23 @@
// Pass stats for reporting.
stats_reporter_.UpdateStatistics(capture_.stats);
+ if (submodules_.capture_levels_adjuster) {
+ submodules_.capture_levels_adjuster->ApplyPostLevelAdjustment(
+ *capture_buffer);
+
+ // If the analog mic gain emulation is active, retrieve the level from the
+ // analog gain control and set it to mic gain emulator.
+ if (config_.capture_level_adjustment.analog_mic_gain_emulation.enabled) {
+ if (submodules_.agc_manager) {
+ submodules_.capture_levels_adjuster->SetAnalogMicGainLevel(
+ submodules_.agc_manager->stream_analog_level());
+ } else if (submodules_.gain_control) {
+ submodules_.capture_levels_adjuster->SetAnalogMicGainLevel(
+ submodules_.gain_control->stream_analog_level());
+ }
+ }
+ }
+
// Temporarily set the output to zero after the stream has been unmuted
// (capture output is again used). The purpose of this is to avoid clicks and
// artefacts in the audio that results when the processing again is
@@ -1541,16 +1605,29 @@
void AudioProcessingImpl::set_stream_analog_level(int level) {
MutexLock lock_capture(&mutex_capture_);
+ if (config_.capture_level_adjustment.analog_mic_gain_emulation.enabled) {
+ // If the analog mic gain is emulated internally, simply cache the level for
+ // later reporting back as the recommended stream analog level to use.
+ capture_.cached_stream_analog_level_ = level;
+ return;
+ }
+
if (submodules_.agc_manager) {
submodules_.agc_manager->set_stream_analog_level(level);
data_dumper_->DumpRaw("experimental_gain_control_set_stream_analog_level",
1, &level);
- } else if (submodules_.gain_control) {
+ return;
+ }
+
+ if (submodules_.gain_control) {
int error = submodules_.gain_control->set_stream_analog_level(level);
RTC_DCHECK_EQ(kNoError, error);
- } else {
- capture_.cached_stream_analog_level_ = level;
+ return;
}
+
+ // If no analog mic gain control functionality is in place, cache the level
+ // for later reporting back as the recommended stream analog level to use.
+ capture_.cached_stream_analog_level_ = level;
}
int AudioProcessingImpl::recommended_stream_analog_level() const {
@@ -1559,13 +1636,19 @@
}
int AudioProcessingImpl::recommended_stream_analog_level_locked() const {
- if (submodules_.agc_manager) {
- return submodules_.agc_manager->stream_analog_level();
- } else if (submodules_.gain_control) {
- return submodules_.gain_control->stream_analog_level();
- } else {
+ if (config_.capture_level_adjustment.analog_mic_gain_emulation.enabled) {
return capture_.cached_stream_analog_level_;
}
+
+ if (submodules_.agc_manager) {
+ return submodules_.agc_manager->stream_analog_level();
+ }
+
+ if (submodules_.gain_control) {
+ return submodules_.gain_control->stream_analog_level();
+ }
+
+ return capture_.cached_stream_analog_level_;
}
bool AudioProcessingImpl::CreateAndAttachAecDump(const std::string& file_name,
@@ -1629,7 +1712,8 @@
config_.high_pass_filter.enabled, !!submodules_.echo_control_mobile,
config_.residual_echo_detector.enabled, !!submodules_.noise_suppressor,
!!submodules_.gain_control, !!submodules_.gain_controller2,
- config_.pre_amplifier.enabled, capture_nonlocked_.echo_controller_enabled,
+ config_.pre_amplifier.enabled || config_.capture_level_adjustment.enabled,
+ capture_nonlocked_.echo_controller_enabled,
config_.voice_detection.enabled, !!submodules_.transient_suppressor);
}
@@ -1873,12 +1957,27 @@
}
}
-void AudioProcessingImpl::InitializePreAmplifier() {
- if (config_.pre_amplifier.enabled) {
- submodules_.pre_amplifier.reset(
- new GainApplier(true, config_.pre_amplifier.fixed_gain_factor));
+void AudioProcessingImpl::InitializeCaptureLevelsAdjuster() {
+ if (config_.pre_amplifier.enabled ||
+ config_.capture_level_adjustment.enabled) {
+ // Use both the pre-amplifier and the capture level adjustment gains as
+ // pre-gains.
+ float pre_gain = 1.f;
+ if (config_.pre_amplifier.enabled) {
+ pre_gain *= config_.pre_amplifier.fixed_gain_factor;
+ }
+ if (config_.capture_level_adjustment.enabled) {
+ pre_gain *= config_.capture_level_adjustment.pre_gain_factor;
+ }
+
+ submodules_.capture_levels_adjuster =
+ std::make_unique<CaptureLevelsAdjuster>(
+ config_.capture_level_adjustment.analog_mic_gain_emulation.enabled,
+ config_.capture_level_adjustment.analog_mic_gain_emulation
+ .initial_level,
+ pre_gain, config_.capture_level_adjustment.post_gain_factor);
} else {
- submodules_.pre_amplifier.reset();
+ submodules_.capture_levels_adjuster.reset();
}
}
@@ -2045,7 +2144,7 @@
split_rate(kSampleRate16kHz),
echo_path_gain_change(false),
prev_analog_mic_level(-1),
- prev_pre_amp_gain(-1.f),
+ prev_pre_adjustment_gain(-1.f),
playout_volume(-1),
prev_playout_volume(-1) {}