blob: 00677f7c68d0e4afa6335110bfca0280872d7d8c [file] [log] [blame]
peah522d71b2017-02-23 05:16:26 -08001/*
2 * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020011#include "modules/audio_processing/aec3/aec_state.h"
peah522d71b2017-02-23 05:16:26 -080012
13#include <math.h>
Raphael Kubo da Costa07438142017-10-16 17:00:02 +020014
peah522d71b2017-02-23 05:16:26 -080015#include <numeric>
16#include <vector>
17
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020018#include "api/array_view.h"
19#include "modules/audio_processing/logging/apm_data_dumper.h"
20#include "rtc_base/atomicops.h"
21#include "rtc_base/checks.h"
peah522d71b2017-02-23 05:16:26 -080022
23namespace webrtc {
24namespace {
25
peah86afe9d2017-04-06 15:45:32 -070026// Computes delay of the adaptive filter.
Per Åhgren40659c32017-10-17 12:56:21 +020027int EstimateFilterDelay(
peah522d71b2017-02-23 05:16:26 -080028 const std::vector<std::array<float, kFftLengthBy2Plus1>>&
peah86afe9d2017-04-06 15:45:32 -070029 adaptive_filter_frequency_response) {
30 const auto& H2 = adaptive_filter_frequency_response;
peah522d71b2017-02-23 05:16:26 -080031 constexpr size_t kUpperBin = kFftLengthBy2 - 5;
Per Åhgren09a718a2017-12-11 22:28:45 +010032 RTC_DCHECK_GE(kMaxAdaptiveFilterLength, H2.size());
33 std::array<int, kMaxAdaptiveFilterLength> delays;
Per Åhgren40659c32017-10-17 12:56:21 +020034 delays.fill(0);
peah522d71b2017-02-23 05:16:26 -080035 for (size_t k = 1; k < kUpperBin; ++k) {
peah86afe9d2017-04-06 15:45:32 -070036 // Find the maximum of H2[j].
Per Åhgren40659c32017-10-17 12:56:21 +020037 size_t peak = 0;
peah522d71b2017-02-23 05:16:26 -080038 for (size_t j = 0; j < H2.size(); ++j) {
39 if (H2[j][k] > H2[peak][k]) {
40 peak = j;
41 }
42 }
Per Åhgren40659c32017-10-17 12:56:21 +020043 ++delays[peak];
peah522d71b2017-02-23 05:16:26 -080044 }
peah522d71b2017-02-23 05:16:26 -080045
Per Åhgren40659c32017-10-17 12:56:21 +020046 return std::distance(delays.begin(),
47 std::max_element(delays.begin(), delays.end()));
peah522d71b2017-02-23 05:16:26 -080048}
49
peah522d71b2017-02-23 05:16:26 -080050} // namespace
51
52int AecState::instance_count_ = 0;
53
Gustaf Ullbergbd83b912017-10-18 12:32:42 +020054AecState::AecState(const EchoCanceller3Config& config)
peah522d71b2017-02-23 05:16:26 -080055 : data_dumper_(
56 new ApmDataDumper(rtc::AtomicOps::Increment(&instance_count_))),
Gustaf Ullbergbd83b912017-10-18 12:32:42 +020057 erle_estimator_(config.erle.min, config.erle.max_l, config.erle.max_h),
peah8cee56f2017-08-24 22:36:53 -070058 config_(config),
Per Åhgren08ea5892018-01-15 08:07:41 +010059 max_render_(config_.filter.main.length_blocks, 0.f),
Per Åhgren09a718a2017-12-11 22:28:45 +010060 reverb_decay_(config_.ep_strength.default_len) {}
peah522d71b2017-02-23 05:16:26 -080061
62AecState::~AecState() = default;
63
peah86afe9d2017-04-06 15:45:32 -070064void AecState::HandleEchoPathChange(
65 const EchoPathVariability& echo_path_variability) {
Per Åhgren8ba58612017-12-01 23:01:44 +010066 const auto full_reset = [&]() {
Per Åhgren63b494d2017-12-06 11:32:38 +010067 blocks_since_last_saturation_ = 0;
peah86afe9d2017-04-06 15:45:32 -070068 usable_linear_estimate_ = false;
69 echo_leakage_detected_ = false;
70 capture_signal_saturation_ = false;
71 echo_saturation_ = false;
Per Åhgren63b494d2017-12-06 11:32:38 +010072 previous_max_sample_ = 0.f;
Per Åhgren09a718a2017-12-11 22:28:45 +010073 std::fill(max_render_.begin(), max_render_.end(), 0.f);
Per Åhgren8ba58612017-12-01 23:01:44 +010074 force_zero_gain_counter_ = 0;
Per Åhgren4b3bc0f2017-12-20 15:26:13 +010075 blocks_with_proper_filter_adaptation_ = 0;
Per Åhgren8ba58612017-12-01 23:01:44 +010076 capture_block_counter_ = 0;
Per Åhgren4b3bc0f2017-12-20 15:26:13 +010077 filter_has_had_time_to_converge_ = false;
Per Åhgren8ba58612017-12-01 23:01:44 +010078 render_received_ = false;
79 force_zero_gain_ = true;
Per Åhgren4b3bc0f2017-12-20 15:26:13 +010080 blocks_with_active_render_ = 0;
Per Åhgrena98c8072018-01-15 19:17:16 +010081 initial_state_ = true;
Per Åhgren8ba58612017-12-01 23:01:44 +010082 };
peah6d822ad2017-04-10 13:52:14 -070083
Per Åhgren8ba58612017-12-01 23:01:44 +010084 // TODO(peah): Refine the reset scheme according to the type of gain and
85 // delay adjustment.
86 if (echo_path_variability.gain_change) {
87 full_reset();
88 }
89
90 if (echo_path_variability.delay_change !=
91 EchoPathVariability::DelayAdjustment::kBufferReadjustment) {
92 full_reset();
93 } else if (echo_path_variability.delay_change !=
94 EchoPathVariability::DelayAdjustment::kBufferFlush) {
95 full_reset();
96
97 } else if (echo_path_variability.delay_change !=
98 EchoPathVariability::DelayAdjustment::kDelayReset) {
99 full_reset();
100 } else if (echo_path_variability.delay_change !=
101 EchoPathVariability::DelayAdjustment::kNewDetectedDelay) {
102 full_reset();
103 } else if (echo_path_variability.gain_change) {
104 capture_block_counter_ = kNumBlocksPerSecond;
peah86afe9d2017-04-06 15:45:32 -0700105 }
106}
107
Per Åhgren09a718a2017-12-11 22:28:45 +0100108void AecState::Update(
109 const std::vector<std::array<float, kFftLengthBy2Plus1>>&
110 adaptive_filter_frequency_response,
111 const std::vector<float>& adaptive_filter_impulse_response,
112 bool converged_filter,
Per Åhgren09a718a2017-12-11 22:28:45 +0100113 const RenderBuffer& render_buffer,
114 const std::array<float, kFftLengthBy2Plus1>& E2_main,
115 const std::array<float, kFftLengthBy2Plus1>& Y2,
Per Åhgren09a718a2017-12-11 22:28:45 +0100116 const std::array<float, kBlockSize>& s,
117 bool echo_leakage_detected) {
peah86afe9d2017-04-06 15:45:32 -0700118 // Store input parameters.
119 echo_leakage_detected_ = echo_leakage_detected;
120
Per Åhgren0e6d2f52017-12-20 22:19:56 +0100121 // Estimate the filter delay.
122 filter_delay_ = EstimateFilterDelay(adaptive_filter_frequency_response);
123 const std::vector<float>& x = render_buffer.Block(-filter_delay_)[0];
124
peah86afe9d2017-04-06 15:45:32 -0700125 // Update counters.
Per Åhgren1b4059e2017-10-15 20:19:21 +0200126 ++capture_block_counter_;
Per Åhgren4b3bc0f2017-12-20 15:26:13 +0100127 const bool active_render_block = DetectActiveRender(x);
128 blocks_with_active_render_ += active_render_block ? 1 : 0;
129 blocks_with_proper_filter_adaptation_ +=
130 active_render_block && !SaturatedCapture() ? 1 : 0;
peah86afe9d2017-04-06 15:45:32 -0700131
peah6d822ad2017-04-10 13:52:14 -0700132 // Force zero echo suppression gain after an echo path change to allow at
133 // least some render data to be collected in order to avoid an initial echo
134 // burst.
Per Åhgren4b3bc0f2017-12-20 15:26:13 +0100135 force_zero_gain_ = ++force_zero_gain_counter_ < kNumBlocksPerSecond / 5;
peah6d822ad2017-04-10 13:52:14 -0700136
Per Åhgren4b3bc0f2017-12-20 15:26:13 +0100137
peah86afe9d2017-04-06 15:45:32 -0700138 // Update the ERL and ERLE measures.
Per Åhgren40659c32017-10-17 12:56:21 +0200139 if (converged_filter && capture_block_counter_ >= 2 * kNumBlocksPerSecond) {
Per Åhgren0e6d2f52017-12-20 22:19:56 +0100140 const auto& X2 = render_buffer.Spectrum(filter_delay_);
peah522d71b2017-02-23 05:16:26 -0800141 erle_estimator_.Update(X2, Y2, E2_main);
142 erl_estimator_.Update(X2, Y2);
peah522d71b2017-02-23 05:16:26 -0800143 }
peah86afe9d2017-04-06 15:45:32 -0700144
Per Åhgren1b4059e2017-10-15 20:19:21 +0200145 // Update the echo audibility evaluator.
146 echo_audibility_.Update(x, s, converged_filter);
147
Per Åhgren63b494d2017-12-06 11:32:38 +0100148 // Detect and flag echo saturation.
149 // TODO(peah): Add the delay in this computation to ensure that the render and
150 // capture signals are properly aligned.
Gustaf Ullbergbd83b912017-10-18 12:32:42 +0200151 if (config_.ep_strength.echo_can_saturate) {
Per Åhgren4b3bc0f2017-12-20 15:26:13 +0100152 echo_saturation_ = DetectEchoSaturation(x);
Per Åhgren1b4059e2017-10-15 20:19:21 +0200153 }
peah86afe9d2017-04-06 15:45:32 -0700154
Per Åhgren63b494d2017-12-06 11:32:38 +0100155 // TODO(peah): Move?
Per Åhgren4b3bc0f2017-12-20 15:26:13 +0100156 filter_has_had_time_to_converge_ =
157 blocks_with_proper_filter_adaptation_ >= 2 * kNumBlocksPerSecond;
158
Per Åhgrena98c8072018-01-15 19:17:16 +0100159 initial_state_ =
160 blocks_with_proper_filter_adaptation_ < 5 * kNumBlocksPerSecond;
161
Per Åhgren63b494d2017-12-06 11:32:38 +0100162 // Flag whether the linear filter estimate is usable.
163 usable_linear_estimate_ =
Per Åhgren4b3bc0f2017-12-20 15:26:13 +0100164 !echo_saturation_ &&
165 (converged_filter || filter_has_had_time_to_converge_) &&
Per Åhgrende22a172017-12-20 18:00:51 +0100166 capture_block_counter_ >= 2 * kNumBlocksPerSecond && !TransparentMode();
Per Åhgren63b494d2017-12-06 11:32:38 +0100167
168 // After an amount of active render samples for which an echo should have been
169 // detected in the capture signal if the ERL was not infinite, flag that a
170 // transparent mode should be entered.
Per Åhgren4b3bc0f2017-12-20 15:26:13 +0100171 transparent_mode_ =
172 !converged_filter &&
173 (blocks_with_active_render_ == 0 ||
174 blocks_with_proper_filter_adaptation_ >= 5 * kNumBlocksPerSecond);
peah29103572017-07-11 02:54:02 -0700175
176 // Update the room reverb estimate.
177 UpdateReverb(adaptive_filter_impulse_response);
178}
179
Per Åhgren09a718a2017-12-11 22:28:45 +0100180void AecState::UpdateReverb(const std::vector<float>& impulse_response) {
peah29103572017-07-11 02:54:02 -0700181 if ((!(filter_delay_ && usable_linear_estimate_)) ||
Per Åhgren08ea5892018-01-15 08:07:41 +0100182 (filter_delay_ >
183 static_cast<int>(config_.filter.main.length_blocks) - 4)) {
peah29103572017-07-11 02:54:02 -0700184 return;
185 }
186
187 // Form the data to match against by squaring the impulse response
188 // coefficients.
Per Åhgren09a718a2017-12-11 22:28:45 +0100189 std::array<float, GetTimeDomainLength(kMaxAdaptiveFilterLength)>
190 matching_data_data;
Per Åhgren08ea5892018-01-15 08:07:41 +0100191 RTC_DCHECK_LE(GetTimeDomainLength(config_.filter.main.length_blocks),
Per Åhgren09a718a2017-12-11 22:28:45 +0100192 matching_data_data.size());
193 rtc::ArrayView<float> matching_data(
194 matching_data_data.data(),
Per Åhgren08ea5892018-01-15 08:07:41 +0100195 GetTimeDomainLength(config_.filter.main.length_blocks));
peah29103572017-07-11 02:54:02 -0700196 std::transform(impulse_response.begin(), impulse_response.end(),
197 matching_data.begin(), [](float a) { return a * a; });
198
199 // Avoid matching against noise in the model by subtracting an estimate of the
200 // model noise power.
201 constexpr size_t kTailLength = 64;
Per Åhgren09a718a2017-12-11 22:28:45 +0100202 const size_t tail_index =
Per Åhgren08ea5892018-01-15 08:07:41 +0100203 GetTimeDomainLength(config_.filter.main.length_blocks) - kTailLength;
peah29103572017-07-11 02:54:02 -0700204 const float tail_power = *std::max_element(matching_data.begin() + tail_index,
205 matching_data.end());
206 std::for_each(matching_data.begin(), matching_data.begin() + tail_index,
207 [tail_power](float& a) { a = std::max(0.f, a - tail_power); });
208
209 // Identify the peak index of the impulse response.
210 const size_t peak_index = *std::max_element(
211 matching_data.begin(), matching_data.begin() + tail_index);
212
213 if (peak_index + 128 < tail_index) {
214 size_t start_index = peak_index + 64;
215 // Compute the matching residual error for the current candidate to match.
216 float residual_sqr_sum = 0.f;
217 float d_k = reverb_decay_to_test_;
218 for (size_t k = start_index; k < tail_index; ++k) {
219 if (matching_data[start_index + 1] == 0.f) {
220 break;
221 }
222
223 float residual = matching_data[k] - matching_data[peak_index] * d_k;
224 residual_sqr_sum += residual * residual;
225 d_k *= reverb_decay_to_test_;
226 }
227
228 // If needed, update the best candidate for the reverb decay.
229 if (reverb_decay_candidate_residual_ < 0.f ||
230 residual_sqr_sum < reverb_decay_candidate_residual_) {
231 reverb_decay_candidate_residual_ = residual_sqr_sum;
232 reverb_decay_candidate_ = reverb_decay_to_test_;
233 }
234 }
235
236 // Compute the next reverb candidate to evaluate such that all candidates will
237 // be evaluated within one second.
238 reverb_decay_to_test_ += (0.9965f - 0.9f) / (5 * kNumBlocksPerSecond);
239
240 // If all reverb candidates have been evaluated, choose the best one as the
241 // reverb decay.
242 if (reverb_decay_to_test_ >= 0.9965f) {
243 if (reverb_decay_candidate_residual_ < 0.f) {
244 // Transform the decay to be in the unit of blocks.
245 reverb_decay_ = powf(reverb_decay_candidate_, kFftLengthBy2);
246
247 // Limit the estimated reverb_decay_ to the maximum one needed in practice
248 // to minimize the impact of incorrect estimates.
Gustaf Ullbergbd83b912017-10-18 12:32:42 +0200249 reverb_decay_ = std::min(config_.ep_strength.default_len, reverb_decay_);
peah29103572017-07-11 02:54:02 -0700250 }
251 reverb_decay_to_test_ = 0.9f;
252 reverb_decay_candidate_residual_ = -1.f;
253 }
254
255 // For noisy impulse responses, assume a fixed tail length.
256 if (tail_power > 0.0005f) {
Gustaf Ullbergbd83b912017-10-18 12:32:42 +0200257 reverb_decay_ = config_.ep_strength.default_len;
peah29103572017-07-11 02:54:02 -0700258 }
259 data_dumper_->DumpRaw("aec3_reverb_decay", reverb_decay_);
260 data_dumper_->DumpRaw("aec3_tail_power", tail_power);
261}
262
Per Åhgren4b3bc0f2017-12-20 15:26:13 +0100263bool AecState::DetectActiveRender(rtc::ArrayView<const float> x) const {
264 const float x_energy = std::inner_product(x.begin(), x.end(), x.begin(), 0.f);
265 return x_energy > (config_.render_levels.active_render_limit *
266 config_.render_levels.active_render_limit) *
267 kFftLengthBy2;
268}
269
270bool AecState::DetectEchoSaturation(rtc::ArrayView<const float> x) {
271 RTC_DCHECK_LT(0, x.size());
272 const float max_sample = fabs(*std::max_element(
273 x.begin(), x.end(), [](float a, float b) { return a * a < b * b; }));
274 previous_max_sample_ = max_sample;
275
276 // Set flag for potential presence of saturated echo
277 blocks_since_last_saturation_ =
278 previous_max_sample_ > 200.f && SaturatedCapture()
279 ? 0
280 : blocks_since_last_saturation_ + 1;
281
282 return blocks_since_last_saturation_ < 20;
283}
284
peah29103572017-07-11 02:54:02 -0700285void AecState::EchoAudibility::Update(rtc::ArrayView<const float> x,
Per Åhgren1b4059e2017-10-15 20:19:21 +0200286 const std::array<float, kBlockSize>& s,
287 bool converged_filter) {
peah29103572017-07-11 02:54:02 -0700288 auto result_x = std::minmax_element(x.begin(), x.end());
289 auto result_s = std::minmax_element(s.begin(), s.end());
290 const float x_abs =
Raphael Kubo da Costa07438142017-10-16 17:00:02 +0200291 std::max(fabsf(*result_x.first), fabsf(*result_x.second));
peah29103572017-07-11 02:54:02 -0700292 const float s_abs =
Raphael Kubo da Costa07438142017-10-16 17:00:02 +0200293 std::max(fabsf(*result_s.first), fabsf(*result_s.second));
peah29103572017-07-11 02:54:02 -0700294
Per Åhgren1b4059e2017-10-15 20:19:21 +0200295 if (converged_filter) {
296 if (x_abs < 20.f) {
297 ++low_farend_counter_;
298 } else {
299 low_farend_counter_ = 0;
300 }
peah29103572017-07-11 02:54:02 -0700301 } else {
Per Åhgren1b4059e2017-10-15 20:19:21 +0200302 if (x_abs < 100.f) {
303 ++low_farend_counter_;
304 } else {
305 low_farend_counter_ = 0;
306 }
peah29103572017-07-11 02:54:02 -0700307 }
308
309 // The echo is deemed as not audible if the echo estimate is on the level of
310 // the quantization noise in the FFTs and the nearend level is sufficiently
311 // strong to mask that by ensuring that the playout and AGC gains do not boost
312 // any residual echo that is below the quantization noise level. Furthermore,
313 // cases where the render signal is very close to zero are also identified as
314 // not producing audible echo.
Per Åhgren1b4059e2017-10-15 20:19:21 +0200315 inaudible_echo_ = (max_nearend_ > 500 && s_abs < 30.f) ||
316 (!converged_filter && x_abs < 500);
peah29103572017-07-11 02:54:02 -0700317 inaudible_echo_ = inaudible_echo_ || low_farend_counter_ > 20;
318}
319
320void AecState::EchoAudibility::UpdateWithOutput(rtc::ArrayView<const float> e) {
321 const float e_max = *std::max_element(e.begin(), e.end());
322 const float e_min = *std::min_element(e.begin(), e.end());
Raphael Kubo da Costa07438142017-10-16 17:00:02 +0200323 const float e_abs = std::max(fabsf(e_max), fabsf(e_min));
peah29103572017-07-11 02:54:02 -0700324
325 if (max_nearend_ < e_abs) {
326 max_nearend_ = e_abs;
327 max_nearend_counter_ = 0;
328 } else {
329 if (++max_nearend_counter_ > 5 * kNumBlocksPerSecond) {
330 max_nearend_ *= 0.995f;
331 }
332 }
peah522d71b2017-02-23 05:16:26 -0800333}
334
335} // namespace webrtc