Make VectorBuffer in AEC3 multi-channel
All dependent modules are hardcoded to do their regular mono processing on the first channel.
This _almost_ makes RenderBuffer multi-channel: FftData is still only mono.
Bug: webrtc:10913
Change-Id: Id5cc34dbabfe59e1cc72a9675dc7979794e870ed
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/151139
Commit-Queue: Sam Zackrisson <saza@webrtc.org>
Reviewed-by: Per Åhgren <peah@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#29074}
diff --git a/modules/audio_processing/aec3/aec_state.cc b/modules/audio_processing/aec3/aec_state.cc
index 566c62f..8ff2930 100644
--- a/modules/audio_processing/aec3/aec_state.cc
+++ b/modules/audio_processing/aec3/aec_state.cc
@@ -153,7 +153,8 @@
erle_estimator_.Reset(false);
}
- const auto& X2 = render_buffer.Spectrum(delay_state_.DirectPathFilterDelay());
+ const auto& X2 = render_buffer.Spectrum(delay_state_.DirectPathFilterDelay(),
+ /*channel=*/0);
const auto& X2_input_erle = X2_reverb;
erle_estimator_.Update(render_buffer, adaptive_filter_frequency_response,
diff --git a/modules/audio_processing/aec3/echo_audibility.cc b/modules/audio_processing/aec3/echo_audibility.cc
index 4154e53..f1b0760 100644
--- a/modules/audio_processing/aec3/echo_audibility.cc
+++ b/modules/audio_processing/aec3/echo_audibility.cc
@@ -83,7 +83,8 @@
for (int idx = render_spectrum_write_prev_.value();
idx != render_spectrum_write_current;
idx = spectrum_buffer.DecIndex(idx)) {
- render_stationarity_.UpdateNoiseEstimator(spectrum_buffer.buffer[idx]);
+ render_stationarity_.UpdateNoiseEstimator(
+ spectrum_buffer.buffer[idx][/*channel=*/0]);
}
}
render_spectrum_write_prev_ = render_spectrum_write_current;
diff --git a/modules/audio_processing/aec3/echo_remover.cc b/modules/audio_processing/aec3/echo_remover.cc
index 8f288de..21eb12f 100644
--- a/modules/audio_processing/aec3/echo_remover.cc
+++ b/modules/audio_processing/aec3/echo_remover.cc
@@ -346,7 +346,8 @@
data_dumper_->DumpRaw("aec3_S2_linear", S2_linear);
data_dumper_->DumpRaw("aec3_Y2", Y2);
data_dumper_->DumpRaw(
- "aec3_X2", render_buffer->Spectrum(aec_state_.FilterDelayBlocks()));
+ "aec3_X2",
+ render_buffer->Spectrum(aec_state_.FilterDelayBlocks(), /*channel=*/0));
data_dumper_->DumpRaw("aec3_R2", R2);
data_dumper_->DumpRaw("aec3_R2_reverb",
residual_echo_estimator_.GetReverbPowerSpectrum());
diff --git a/modules/audio_processing/aec3/erle_estimator_unittest.cc b/modules/audio_processing/aec3/erle_estimator_unittest.cc
index 18ba25a..dc8ed34 100644
--- a/modules/audio_processing/aec3/erle_estimator_unittest.cc
+++ b/modules/audio_processing/aec3/erle_estimator_unittest.cc
@@ -70,7 +70,8 @@
std::array<float, kFftLengthBy2Plus1>* Y2,
float erle) {
const auto& spectrum_buffer = render_buffer.GetSpectrumBuffer();
- const auto& X2_from_buffer = spectrum_buffer.buffer[spectrum_buffer.write];
+ const auto& X2_from_buffer =
+ spectrum_buffer.buffer[spectrum_buffer.write][/*channel=*/0];
std::copy(X2_from_buffer.begin(), X2_from_buffer.end(), X2->begin());
std::transform(X2->begin(), X2->end(), Y2->begin(),
[](float a) { return a * kEchoPathGain * kEchoPathGain; });
diff --git a/modules/audio_processing/aec3/mock/mock_render_delay_buffer.cc b/modules/audio_processing/aec3/mock/mock_render_delay_buffer.cc
index de87000..56569a2 100644
--- a/modules/audio_processing/aec3/mock/mock_render_delay_buffer.cc
+++ b/modules/audio_processing/aec3/mock/mock_render_delay_buffer.cc
@@ -19,7 +19,9 @@
NumBandsForRate(sample_rate_hz),
num_channels,
kBlockSize),
- spectrum_buffer_(block_buffer_.buffer.size(), kFftLengthBy2Plus1),
+ spectrum_buffer_(block_buffer_.buffer.size(),
+ num_channels,
+ kFftLengthBy2Plus1),
fft_buffer_(block_buffer_.buffer.size()),
render_buffer_(&block_buffer_, &spectrum_buffer_, &fft_buffer_),
downsampled_render_buffer_(GetDownSampledBufferSize(4, 4)) {
diff --git a/modules/audio_processing/aec3/render_buffer.cc b/modules/audio_processing/aec3/render_buffer.cc
index f6ffa04..bef2b2f 100644
--- a/modules/audio_processing/aec3/render_buffer.cc
+++ b/modules/audio_processing/aec3/render_buffer.cc
@@ -41,9 +41,10 @@
X2->fill(0.f);
int position = spectrum_buffer_->read;
for (size_t j = 0; j < num_spectra; ++j) {
- std::transform(X2->begin(), X2->end(),
- spectrum_buffer_->buffer[position].begin(), X2->begin(),
- std::plus<float>());
+ for (const auto& channel_spectrum : spectrum_buffer_->buffer[position]) {
+ std::transform(X2->begin(), X2->end(), channel_spectrum.begin(),
+ X2->begin(), std::plus<float>());
+ }
position = spectrum_buffer_->IncIndex(position);
}
}
@@ -58,16 +59,20 @@
int position = spectrum_buffer_->read;
size_t j = 0;
for (; j < num_spectra_shorter; ++j) {
- std::transform(X2_shorter->begin(), X2_shorter->end(),
- spectrum_buffer_->buffer[position].begin(),
- X2_shorter->begin(), std::plus<float>());
+ for (const auto& channel_spectrum : spectrum_buffer_->buffer[position]) {
+ std::transform(X2_shorter->begin(), X2_shorter->end(),
+ channel_spectrum.begin(), X2_shorter->begin(),
+ std::plus<float>());
+ }
position = spectrum_buffer_->IncIndex(position);
}
std::copy(X2_shorter->begin(), X2_shorter->end(), X2_longer->begin());
for (; j < num_spectra_longer; ++j) {
- std::transform(X2_longer->begin(), X2_longer->end(),
- spectrum_buffer_->buffer[position].begin(),
- X2_longer->begin(), std::plus<float>());
+ for (const auto& channel_spectrum : spectrum_buffer_->buffer[position]) {
+ std::transform(X2_longer->begin(), X2_longer->end(),
+ channel_spectrum.begin(), X2_longer->begin(),
+ std::plus<float>());
+ }
position = spectrum_buffer_->IncIndex(position);
}
}
diff --git a/modules/audio_processing/aec3/render_buffer.h b/modules/audio_processing/aec3/render_buffer.h
index 8759760..7c70e1b 100644
--- a/modules/audio_processing/aec3/render_buffer.h
+++ b/modules/audio_processing/aec3/render_buffer.h
@@ -44,10 +44,11 @@
}
// Get the spectrum from one of the FFTs in the buffer.
- rtc::ArrayView<const float> Spectrum(int buffer_offset_ffts) const {
+ rtc::ArrayView<const float> Spectrum(int buffer_offset_ffts,
+ size_t channel) const {
int position = spectrum_buffer_->OffsetIndex(spectrum_buffer_->read,
buffer_offset_ffts);
- return spectrum_buffer_->buffer[position];
+ return spectrum_buffer_->buffer[position][channel];
}
// Returns the circular fft buffer.
diff --git a/modules/audio_processing/aec3/render_buffer_unittest.cc b/modules/audio_processing/aec3/render_buffer_unittest.cc
index 4437178..e7e964c 100644
--- a/modules/audio_processing/aec3/render_buffer_unittest.cc
+++ b/modules/audio_processing/aec3/render_buffer_unittest.cc
@@ -23,7 +23,7 @@
// Verifies the check for non-null fft buffer.
TEST(RenderBuffer, NullExternalFftBuffer) {
MatrixBuffer block_buffer(10, 3, 1, kBlockSize);
- VectorBuffer spectrum_buffer(10, kFftLengthBy2Plus1);
+ VectorBuffer spectrum_buffer(10, 1, kFftLengthBy2Plus1);
EXPECT_DEATH(RenderBuffer(&block_buffer, &spectrum_buffer, nullptr), "");
}
@@ -37,7 +37,7 @@
// Verifies the check for non-null block buffer.
TEST(RenderBuffer, NullExternalBlockBuffer) {
FftBuffer fft_buffer(10);
- VectorBuffer spectrum_buffer(10, kFftLengthBy2Plus1);
+ VectorBuffer spectrum_buffer(10, 1, kFftLengthBy2Plus1);
EXPECT_DEATH(RenderBuffer(nullptr, &spectrum_buffer, &fft_buffer), "");
}
diff --git a/modules/audio_processing/aec3/render_delay_buffer.cc b/modules/audio_processing/aec3/render_delay_buffer.cc
index 379f5a1..9d47bec 100644
--- a/modules/audio_processing/aec3/render_delay_buffer.cc
+++ b/modules/audio_processing/aec3/render_delay_buffer.cc
@@ -127,7 +127,7 @@
NumBandsForRate(sample_rate_hz),
num_render_channels,
kBlockSize),
- spectra_(blocks_.buffer.size(), kFftLengthBy2Plus1),
+ spectra_(blocks_.buffer.size(), num_render_channels, kFftLengthBy2Plus1),
ffts_(blocks_.buffer.size()),
delay_(config_.delay.default_delay),
echo_remover_buffer_(&blocks_, &spectra_, &ffts_),
@@ -381,7 +381,9 @@
std::copy(ds.rbegin(), ds.rend(), lr.buffer.begin() + lr.write);
fft_.PaddedFft(block[0][0], b.buffer[previous_write][0][0],
&f.buffer[f.write]);
- f.buffer[f.write].Spectrum(optimization_, s.buffer[s.write]);
+ // TODO(http://bugs.webrtc.org/10913): Loop over all channels when FftBuffer
+ // supports multi-channel.
+ f.buffer[f.write].Spectrum(optimization_, s.buffer[s.write][/*channel=*/0]);
}
bool RenderDelayBufferImpl::DetectActiveRender(
diff --git a/modules/audio_processing/aec3/render_reverb_model.cc b/modules/audio_processing/aec3/render_reverb_model.cc
index 8ad54c0..72f6506 100644
--- a/modules/audio_processing/aec3/render_reverb_model.cc
+++ b/modules/audio_processing/aec3/render_reverb_model.cc
@@ -34,11 +34,12 @@
int idx_at_delay =
spectrum_buffer.OffsetIndex(spectrum_buffer.read, delay_blocks);
int idx_past = spectrum_buffer.IncIndex(idx_at_delay);
- const auto& X2 = spectrum_buffer.buffer[idx_at_delay];
+ const auto& X2 = spectrum_buffer.buffer[idx_at_delay][/*channel=*/0];
RTC_DCHECK_EQ(X2.size(), reverb_power_spectrum.size());
std::copy(X2.begin(), X2.end(), reverb_power_spectrum.begin());
- render_reverb_.AddReverbNoFreqShaping(spectrum_buffer.buffer[idx_past], 1.0f,
- reverb_decay, reverb_power_spectrum);
+ render_reverb_.AddReverbNoFreqShaping(
+ spectrum_buffer.buffer[idx_past][/*channel=*/0], 1.0f, reverb_decay,
+ reverb_power_spectrum);
}
} // namespace webrtc
diff --git a/modules/audio_processing/aec3/render_signal_analyzer.cc b/modules/audio_processing/aec3/render_signal_analyzer.cc
index 88bacaf..8db874c 100644
--- a/modules/audio_processing/aec3/render_signal_analyzer.cc
+++ b/modules/audio_processing/aec3/render_signal_analyzer.cc
@@ -34,7 +34,8 @@
return;
}
- rtc::ArrayView<const float> X2 = render_buffer.Spectrum(*delay_partitions);
+ rtc::ArrayView<const float> X2 =
+ render_buffer.Spectrum(*delay_partitions, /*channel=*/0);
RTC_DCHECK_EQ(kFftLengthBy2Plus1, X2.size());
for (size_t k = 1; k < (X2.size() - 1); ++k) {
@@ -49,7 +50,7 @@
int strong_peak_freeze_duration,
absl::optional<int>* narrow_peak_band,
size_t* narrow_peak_counter) {
- const auto X2_latest = render_buffer.Spectrum(0);
+ const auto X2_latest = render_buffer.Spectrum(0, /*channel=*/0);
// Identify the spectral peak.
const int peak_bin = static_cast<int>(
diff --git a/modules/audio_processing/aec3/residual_echo_estimator.cc b/modules/audio_processing/aec3/residual_echo_estimator.cc
index d863d8b..a278240 100644
--- a/modules/audio_processing/aec3/residual_echo_estimator.cc
+++ b/modules/audio_processing/aec3/residual_echo_estimator.cc
@@ -77,7 +77,8 @@
// Adds the estimated unmodelled echo power to the residual echo power
// estimate.
echo_reverb_.AddReverb(
- render_buffer.Spectrum(aec_state.FilterLengthBlocks() + 1),
+ render_buffer.Spectrum(aec_state.FilterLengthBlocks() + 1,
+ /*channel=*/0),
aec_state.GetReverbFrequencyResponse(), aec_state.ReverbDecay(), *R2);
} else {
// Estimate the echo generating signal power.
@@ -108,7 +109,8 @@
if (!(aec_state.TransparentMode())) {
echo_reverb_.AddReverbNoFreqShaping(
- render_buffer.Spectrum(aec_state.FilterDelayBlocks() + 1),
+ render_buffer.Spectrum(aec_state.FilterDelayBlocks() + 1,
+ /*channel=*/0),
echo_path_gain * echo_path_gain, aec_state.ReverbDecay(), *R2);
}
}
@@ -171,7 +173,8 @@
X2->fill(0.f);
for (int k = idx_start; k != idx_stop; k = spectrum_buffer.IncIndex(k)) {
- std::transform(X2->begin(), X2->end(), spectrum_buffer.buffer[k].begin(),
+ std::transform(X2->begin(), X2->end(),
+ spectrum_buffer.buffer[k][/*channel=*/0].begin(),
X2->begin(),
[](float a, float b) { return std::max(a, b); });
}
@@ -194,7 +197,7 @@
RTC_DCHECK(X2_noise_floor);
RTC_DCHECK(X2_noise_floor_counter);
- const auto render_power = render_buffer.Spectrum(0);
+ const auto render_power = render_buffer.Spectrum(0, /*channel=*/0);
RTC_DCHECK_EQ(X2_noise_floor->size(), render_power.size());
RTC_DCHECK_EQ(X2_noise_floor_counter->size(), render_power.size());
diff --git a/modules/audio_processing/aec3/signal_dependent_erle_estimator.cc b/modules/audio_processing/aec3/signal_dependent_erle_estimator.cc
index dbe8e48..b16b70e 100644
--- a/modules/audio_processing/aec3/signal_dependent_erle_estimator.cc
+++ b/modules/audio_processing/aec3/signal_dependent_erle_estimator.cc
@@ -332,9 +332,10 @@
filter_frequency_response.size());
for (size_t block = section_boundaries_blocks_[section];
block < block_limit; ++block) {
- std::transform(X2_section.begin(), X2_section.end(),
- spectrum_render_buffer.buffer[idx_render].begin(),
- X2_section.begin(), std::plus<float>());
+ std::transform(
+ X2_section.begin(), X2_section.end(),
+ spectrum_render_buffer.buffer[idx_render][/*channel=*/0].begin(),
+ X2_section.begin(), std::plus<float>());
std::transform(H2_section.begin(), H2_section.end(),
filter_frequency_response[block].begin(),
H2_section.begin(), std::plus<float>());
diff --git a/modules/audio_processing/aec3/signal_dependent_erle_estimator_unittest.cc b/modules/audio_processing/aec3/signal_dependent_erle_estimator_unittest.cc
index b8c83f7..51a2857 100644
--- a/modules/audio_processing/aec3/signal_dependent_erle_estimator_unittest.cc
+++ b/modules/audio_processing/aec3/signal_dependent_erle_estimator_unittest.cc
@@ -99,8 +99,8 @@
render_buffer_->GetSpectrumBuffer();
size_t idx = render_buffer_->Position();
size_t prev_idx = spectrum_render_buffer.OffsetIndex(idx, 1);
- auto& X2 = spectrum_render_buffer.buffer[idx];
- auto& X2_prev = spectrum_render_buffer.buffer[prev_idx];
+ auto& X2 = spectrum_render_buffer.buffer[idx][/*channel=*/0];
+ auto& X2_prev = spectrum_render_buffer.buffer[prev_idx][/*channel=*/0];
std::copy(X2.begin(), X2.end(), X2_.begin());
RTC_DCHECK_EQ(X2.size(), Y2_.size());
for (size_t k = 0; k < X2.size(); ++k) {
diff --git a/modules/audio_processing/aec3/stationarity_estimator.cc b/modules/audio_processing/aec3/stationarity_estimator.cc
index 47f04bf..f6a7d99 100644
--- a/modules/audio_processing/aec3/stationarity_estimator.cc
+++ b/modules/audio_processing/aec3/stationarity_estimator.cc
@@ -105,7 +105,7 @@
constexpr float kThrStationarity = 10.f;
float acum_power = 0.f;
for (auto idx : indexes) {
- acum_power += spectrum_buffer.buffer[idx][band];
+ acum_power += spectrum_buffer.buffer[idx][/*channel=*/0][band];
}
acum_power += reverb[band];
float noise = kWindowLength * GetStationarityPowerBand(band);
diff --git a/modules/audio_processing/aec3/vector_buffer.cc b/modules/audio_processing/aec3/vector_buffer.cc
index 0682885..74b1261 100644
--- a/modules/audio_processing/aec3/vector_buffer.cc
+++ b/modules/audio_processing/aec3/vector_buffer.cc
@@ -14,11 +14,18 @@
namespace webrtc {
-VectorBuffer::VectorBuffer(size_t size, size_t height)
+VectorBuffer::VectorBuffer(size_t size,
+ size_t num_channels,
+ size_t spectrum_length)
: size(static_cast<int>(size)),
- buffer(size, std::vector<float>(height, 0.f)) {
- for (auto& c : buffer) {
- std::fill(c.begin(), c.end(), 0.f);
+ buffer(size,
+ std::vector<std::vector<float>>(
+ num_channels,
+ std::vector<float>(spectrum_length, 0.f))) {
+ for (auto& channel : buffer) {
+ for (auto& c : channel) {
+ std::fill(c.begin(), c.end(), 0.f);
+ }
}
}
diff --git a/modules/audio_processing/aec3/vector_buffer.h b/modules/audio_processing/aec3/vector_buffer.h
index 9d1539f..36dfa80 100644
--- a/modules/audio_processing/aec3/vector_buffer.h
+++ b/modules/audio_processing/aec3/vector_buffer.h
@@ -21,8 +21,9 @@
// Struct for bundling a circular buffer of one dimensional vector objects
// together with the read and write indices.
+// TODO(peah): Change name of this class to be more specific to what it does.
struct VectorBuffer {
- VectorBuffer(size_t size, size_t height);
+ VectorBuffer(size_t size, size_t num_channels, size_t spectrum_length);
~VectorBuffer();
int IncIndex(int index) const {
@@ -50,7 +51,7 @@
void DecReadIndex() { read = DecIndex(read); }
const int size;
- std::vector<std::vector<float>> buffer;
+ std::vector<std::vector<std::vector<float>>> buffer;
int write = 0;
int read = 0;
};