blob: 2bfaa951d84af6b9b38f4c144cbc2d9dbb4c7fc7 [file] [log] [blame]
peah69221db2017-01-27 03:28:19 -08001/*
2 * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020010#include "modules/audio_processing/aec3/echo_remover.h"
peah69221db2017-01-27 03:28:19 -080011
peah86afe9d2017-04-06 15:45:32 -070012#include <math.h>
Yves Gerey988cc082018-10-23 12:03:01 +020013#include <stddef.h>
Jonas Olssona4d87372019-07-05 19:08:33 +020014
peah69221db2017-01-27 03:28:19 -080015#include <algorithm>
Yves Gerey988cc082018-10-23 12:03:01 +020016#include <array>
Mirko Bonadeidbce0902019-03-15 07:39:02 +010017#include <cmath>
peah522d71b2017-02-23 05:16:26 -080018#include <memory>
peah69221db2017-01-27 03:28:19 -080019
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020020#include "api/array_view.h"
21#include "modules/audio_processing/aec3/aec3_common.h"
Yves Gerey988cc082018-10-23 12:03:01 +020022#include "modules/audio_processing/aec3/aec3_fft.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020023#include "modules/audio_processing/aec3/aec_state.h"
24#include "modules/audio_processing/aec3/comfort_noise_generator.h"
25#include "modules/audio_processing/aec3/echo_path_variability.h"
26#include "modules/audio_processing/aec3/echo_remover_metrics.h"
27#include "modules/audio_processing/aec3/fft_data.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020028#include "modules/audio_processing/aec3/render_buffer.h"
Yves Gerey988cc082018-10-23 12:03:01 +020029#include "modules/audio_processing/aec3/render_signal_analyzer.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020030#include "modules/audio_processing/aec3/residual_echo_estimator.h"
31#include "modules/audio_processing/aec3/subtractor.h"
Yves Gerey988cc082018-10-23 12:03:01 +020032#include "modules/audio_processing/aec3/subtractor_output.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020033#include "modules/audio_processing/aec3/suppression_filter.h"
34#include "modules/audio_processing/aec3/suppression_gain.h"
35#include "modules/audio_processing/logging/apm_data_dumper.h"
Steve Anton10542f22019-01-11 09:11:00 -080036#include "rtc_base/atomic_ops.h"
Yves Gerey988cc082018-10-23 12:03:01 +020037#include "rtc_base/checks.h"
Per Åhgren88cf0502018-07-16 17:08:41 +020038#include "rtc_base/logging.h"
peah69221db2017-01-27 03:28:19 -080039
40namespace webrtc {
41
42namespace {
peah522d71b2017-02-23 05:16:26 -080043
Per Åhgrenf6aa5722019-09-10 18:05:17 +020044// Maximum number of channels for which the capture channel data is stored on
45// the stack. If the number of channels are larger than this, they are stored
46// using scratch memory that is pre-allocated on the heap. The reason for this
47// partitioning is not to waste heap space for handling the more common numbers
48// of channels, while at the same time not limiting the support for higher
49// numbers of channels by enforcing the capture channel data to be stored on the
50// stack using a fixed maximum value.
51constexpr size_t kMaxNumChannelsOnStack = 2;
52
53// Chooses the number of channels to store on the heap when that is required due
54// to the number of capture channels being larger than the pre-defined number
55// of channels to store on the stack.
56size_t NumChannelsOnHeap(size_t num_capture_channels) {
57 return num_capture_channels > kMaxNumChannelsOnStack ? num_capture_channels
58 : 0;
59}
60
peah522d71b2017-02-23 05:16:26 -080061void LinearEchoPower(const FftData& E,
62 const FftData& Y,
63 std::array<float, kFftLengthBy2Plus1>* S2) {
64 for (size_t k = 0; k < E.re.size(); ++k) {
65 (*S2)[k] = (Y.re[k] - E.re[k]) * (Y.re[k] - E.re[k]) +
66 (Y.im[k] - E.im[k]) * (Y.im[k] - E.im[k]);
67 }
68}
69
Per Åhgren22754392018-08-10 18:37:38 +020070// Fades between two input signals using a fix-sized transition.
71void SignalTransition(rtc::ArrayView<const float> from,
72 rtc::ArrayView<const float> to,
73 rtc::ArrayView<float> out) {
Gustaf Ullberg7911d372019-09-24 16:31:01 +020074 if (from == to) {
75 RTC_DCHECK_EQ(to.size(), out.size());
76 std::copy(to.begin(), to.end(), out.begin());
77 } else {
78 constexpr size_t kTransitionSize = 30;
79 constexpr float kOneByTransitionSizePlusOne = 1.f / (kTransitionSize + 1);
Per Åhgren22754392018-08-10 18:37:38 +020080
Gustaf Ullberg7911d372019-09-24 16:31:01 +020081 RTC_DCHECK_EQ(from.size(), to.size());
82 RTC_DCHECK_EQ(from.size(), out.size());
83 RTC_DCHECK_LE(kTransitionSize, out.size());
Per Åhgren22754392018-08-10 18:37:38 +020084
Gustaf Ullberg7911d372019-09-24 16:31:01 +020085 for (size_t k = 0; k < kTransitionSize; ++k) {
86 float a = (k + 1) * kOneByTransitionSizePlusOne;
87 out[k] = a * to[k] + (1.f - a) * from[k];
88 }
89
90 std::copy(to.begin() + kTransitionSize, to.end(),
91 out.begin() + kTransitionSize);
Per Åhgren22754392018-08-10 18:37:38 +020092 }
Per Åhgren22754392018-08-10 18:37:38 +020093}
94
Per Åhgren169c7fd2018-04-27 12:04:03 +020095// Computes a windowed (square root Hanning) padded FFT and updates the related
96// memory.
97void WindowedPaddedFft(const Aec3Fft& fft,
98 rtc::ArrayView<const float> v,
99 rtc::ArrayView<float> v_old,
100 FftData* V) {
101 fft.PaddedFft(v, v_old, Aec3Fft::Window::kSqrtHanning, V);
102 std::copy(v.begin(), v.end(), v_old.begin());
103}
104
peah522d71b2017-02-23 05:16:26 -0800105// Class for removing the echo from the capture signal.
peah69221db2017-01-27 03:28:19 -0800106class EchoRemoverImpl final : public EchoRemover {
107 public:
Per Åhgrence202a02019-09-02 17:01:19 +0200108 EchoRemoverImpl(const EchoCanceller3Config& config,
109 int sample_rate_hz,
110 size_t num_render_channels,
111 size_t num_capture_channels);
peah69221db2017-01-27 03:28:19 -0800112 ~EchoRemoverImpl() override;
Per Åhgrenf6aa5722019-09-10 18:05:17 +0200113 EchoRemoverImpl(const EchoRemoverImpl&) = delete;
114 EchoRemoverImpl& operator=(const EchoRemoverImpl&) = delete;
peah69221db2017-01-27 03:28:19 -0800115
Gustaf Ullberg332150d2017-11-22 14:17:39 +0100116 void GetMetrics(EchoControl::Metrics* metrics) const override;
117
peah522d71b2017-02-23 05:16:26 -0800118 // Removes the echo from a block of samples from the capture signal. The
119 // supplied render signal is assumed to be pre-aligned with the capture
120 // signal.
Per Åhgrence202a02019-09-02 17:01:19 +0200121 void ProcessCapture(
122 EchoPathVariability echo_path_variability,
123 bool capture_signal_saturation,
124 const absl::optional<DelayEstimate>& external_delay,
125 RenderBuffer* render_buffer,
Per Åhgrenc20a19c2019-11-13 11:12:29 +0100126 std::vector<std::vector<std::vector<float>>>* linear_output,
Per Åhgrence202a02019-09-02 17:01:19 +0200127 std::vector<std::vector<std::vector<float>>>* capture) override;
peah69221db2017-01-27 03:28:19 -0800128
peah522d71b2017-02-23 05:16:26 -0800129 // Updates the status on whether echo leakage is detected in the output of the
130 // echo remover.
131 void UpdateEchoLeakageStatus(bool leakage_detected) override {
132 echo_leakage_detected_ = leakage_detected;
133 }
peah69221db2017-01-27 03:28:19 -0800134
Per Åhgren8ee1ec82021-03-11 06:33:45 +0000135 void SetCaptureOutputUsage(bool capture_output_used) override {
136 capture_output_used_ = capture_output_used;
137 }
138
peah69221db2017-01-27 03:28:19 -0800139 private:
Per Åhgren9d661982020-03-20 11:26:48 +0100140 // Selects which of the coarse and refined linear filter outputs that is most
Per Åhgren22754392018-08-10 18:37:38 +0200141 // appropriate to pass to the suppressor and forms the linear filter output by
142 // smoothly transition between those.
Gustaf Ullberg68d6d442019-01-29 10:08:15 +0100143 void FormLinearFilterOutput(const SubtractorOutput& subtractor_output,
Per Åhgren22754392018-08-10 18:37:38 +0200144 rtc::ArrayView<float> output);
Per Åhgren78026752018-08-01 16:24:08 +0200145
peah522d71b2017-02-23 05:16:26 -0800146 static int instance_count_;
Gustaf Ullbergbd83b912017-10-18 12:32:42 +0200147 const EchoCanceller3Config config_;
peah522d71b2017-02-23 05:16:26 -0800148 const Aec3Fft fft_;
149 std::unique_ptr<ApmDataDumper> data_dumper_;
150 const Aec3Optimization optimization_;
peah69221db2017-01-27 03:28:19 -0800151 const int sample_rate_hz_;
Per Åhgrence202a02019-09-02 17:01:19 +0200152 const size_t num_render_channels_;
153 const size_t num_capture_channels_;
Per Åhgren9d661982020-03-20 11:26:48 +0100154 const bool use_coarse_filter_output_;
Per Åhgren7bdf0732019-09-25 14:53:30 +0200155 Subtractor subtractor_;
Gustaf Ullberg5ea57492019-11-05 15:19:02 +0100156 SuppressionGain suppression_gain_;
Gustaf Ullbergcaaa9e72019-10-31 14:10:24 +0100157 ComfortNoiseGenerator cng_;
peah522d71b2017-02-23 05:16:26 -0800158 SuppressionFilter suppression_filter_;
peah522d71b2017-02-23 05:16:26 -0800159 RenderSignalAnalyzer render_signal_analyzer_;
Per Åhgrenb4161d32019-10-08 12:35:47 +0200160 ResidualEchoEstimator residual_echo_estimator_;
peah522d71b2017-02-23 05:16:26 -0800161 bool echo_leakage_detected_ = false;
Per Åhgren8ee1ec82021-03-11 06:33:45 +0000162 bool capture_output_used_ = true;
peah522d71b2017-02-23 05:16:26 -0800163 AecState aec_state_;
peahe985b3f2017-02-28 22:08:53 -0800164 EchoRemoverMetrics metrics_;
Gustaf Ullberga99b89b2019-09-23 16:03:12 +0200165 std::vector<std::array<float, kFftLengthBy2>> e_old_;
166 std::vector<std::array<float, kFftLengthBy2>> y_old_;
Per Åhgren88cf0502018-07-16 17:08:41 +0200167 size_t block_counter_ = 0;
168 int gain_change_hangover_ = 0;
Per Åhgrenff045112020-03-20 11:20:39 +0100169 bool refined_filter_output_last_selected_ = true;
peah69221db2017-01-27 03:28:19 -0800170
Per Åhgren0e3b1ff2019-09-25 12:09:37 +0200171 std::vector<std::array<float, kFftLengthBy2>> e_heap_;
Per Åhgrenf6aa5722019-09-10 18:05:17 +0200172 std::vector<std::array<float, kFftLengthBy2Plus1>> Y2_heap_;
173 std::vector<std::array<float, kFftLengthBy2Plus1>> E2_heap_;
174 std::vector<std::array<float, kFftLengthBy2Plus1>> R2_heap_;
Gustaf Ullberga63d1522021-06-11 14:02:53 +0200175 std::vector<std::array<float, kFftLengthBy2Plus1>> R2_unbounded_heap_;
Per Åhgrenf6aa5722019-09-10 18:05:17 +0200176 std::vector<std::array<float, kFftLengthBy2Plus1>> S2_linear_heap_;
177 std::vector<FftData> Y_heap_;
178 std::vector<FftData> E_heap_;
179 std::vector<FftData> comfort_noise_heap_;
180 std::vector<FftData> high_band_comfort_noise_heap_;
181 std::vector<SubtractorOutput> subtractor_output_heap_;
peah69221db2017-01-27 03:28:19 -0800182};
183
peah522d71b2017-02-23 05:16:26 -0800184int EchoRemoverImpl::instance_count_ = 0;
185
Gustaf Ullbergbd83b912017-10-18 12:32:42 +0200186EchoRemoverImpl::EchoRemoverImpl(const EchoCanceller3Config& config,
Per Åhgrence202a02019-09-02 17:01:19 +0200187 int sample_rate_hz,
188 size_t num_render_channels,
189 size_t num_capture_channels)
peah8cee56f2017-08-24 22:36:53 -0700190 : config_(config),
191 fft_(),
aleloi88b82b52017-02-23 06:27:03 -0800192 data_dumper_(
peah522d71b2017-02-23 05:16:26 -0800193 new ApmDataDumper(rtc::AtomicOps::Increment(&instance_count_))),
194 optimization_(DetectOptimization()),
195 sample_rate_hz_(sample_rate_hz),
Per Åhgrence202a02019-09-02 17:01:19 +0200196 num_render_channels_(num_render_channels),
197 num_capture_channels_(num_capture_channels),
Per Åhgren9d661982020-03-20 11:26:48 +0100198 use_coarse_filter_output_(
199 config_.filter.enable_coarse_filter_output_usage),
Per Åhgren7bdf0732019-09-25 14:53:30 +0200200 subtractor_(config,
201 num_render_channels_,
202 num_capture_channels_,
203 data_dumper_.get(),
204 optimization_),
Gustaf Ullberg5ea57492019-11-05 15:19:02 +0100205 suppression_gain_(config_,
206 optimization_,
207 sample_rate_hz,
208 num_capture_channels),
Per Åhgrena388b752020-03-25 07:31:47 +0100209 cng_(config_, optimization_, num_capture_channels_),
Gustaf Ullbergaf3fdc02019-09-24 15:05:04 +0200210 suppression_filter_(optimization_,
211 sample_rate_hz_,
212 num_capture_channels_),
Per Åhgren971de072018-03-14 23:23:47 +0100213 render_signal_analyzer_(config_),
Per Åhgrenb4161d32019-10-08 12:35:47 +0200214 residual_echo_estimator_(config_, num_render_channels),
Sam Zackrisson8f736c02019-10-01 12:47:53 +0200215 aec_state_(config_, num_capture_channels_),
Gustaf Ullberg5ea57492019-11-05 15:19:02 +0100216 e_old_(num_capture_channels_, {0.f}),
217 y_old_(num_capture_channels_, {0.f}),
218 e_heap_(NumChannelsOnHeap(num_capture_channels_), {0.f}),
Per Åhgrenf6aa5722019-09-10 18:05:17 +0200219 Y2_heap_(NumChannelsOnHeap(num_capture_channels_)),
220 E2_heap_(NumChannelsOnHeap(num_capture_channels_)),
221 R2_heap_(NumChannelsOnHeap(num_capture_channels_)),
Gustaf Ullberga63d1522021-06-11 14:02:53 +0200222 R2_unbounded_heap_(NumChannelsOnHeap(num_capture_channels_)),
Per Åhgrenf6aa5722019-09-10 18:05:17 +0200223 S2_linear_heap_(NumChannelsOnHeap(num_capture_channels_)),
224 Y_heap_(NumChannelsOnHeap(num_capture_channels_)),
225 E_heap_(NumChannelsOnHeap(num_capture_channels_)),
226 comfort_noise_heap_(NumChannelsOnHeap(num_capture_channels_)),
227 high_band_comfort_noise_heap_(NumChannelsOnHeap(num_capture_channels_)),
228 subtractor_output_heap_(NumChannelsOnHeap(num_capture_channels_)) {
peah522d71b2017-02-23 05:16:26 -0800229 RTC_DCHECK(ValidFullBandRate(sample_rate_hz));
peah69221db2017-01-27 03:28:19 -0800230}
231
232EchoRemoverImpl::~EchoRemoverImpl() = default;
233
Gustaf Ullberg332150d2017-11-22 14:17:39 +0100234void EchoRemoverImpl::GetMetrics(EchoControl::Metrics* metrics) const {
235 // Echo return loss (ERL) is inverted to go from gain to attenuation.
Mirko Bonadeidbce0902019-03-15 07:39:02 +0100236 metrics->echo_return_loss = -10.0 * std::log10(aec_state_.ErlTimeDomain());
Gustaf Ullberg332150d2017-11-22 14:17:39 +0100237 metrics->echo_return_loss_enhancement =
Jesús de Vicente Peñae9a7e902018-09-27 11:49:39 +0200238 Log2TodB(aec_state_.FullBandErleLog2());
Gustaf Ullberg332150d2017-11-22 14:17:39 +0100239}
240
peahcf02cf12017-04-05 14:18:07 -0700241void EchoRemoverImpl::ProcessCapture(
Per Åhgren88cf0502018-07-16 17:08:41 +0200242 EchoPathVariability echo_path_variability,
peah69221db2017-01-27 03:28:19 -0800243 bool capture_signal_saturation,
Danil Chapovalovdb9f7ab2018-06-19 10:50:11 +0200244 const absl::optional<DelayEstimate>& external_delay,
Per Åhgrenc59a5762017-12-11 21:34:19 +0100245 RenderBuffer* render_buffer,
Per Åhgrenc20a19c2019-11-13 11:12:29 +0100246 std::vector<std::vector<std::vector<float>>>* linear_output,
Per Åhgrence202a02019-09-02 17:01:19 +0200247 std::vector<std::vector<std::vector<float>>>* capture) {
Per Åhgren88cf0502018-07-16 17:08:41 +0200248 ++block_counter_;
Per Åhgrence202a02019-09-02 17:01:19 +0200249 const std::vector<std::vector<std::vector<float>>>& x =
250 render_buffer->Block(0);
251 std::vector<std::vector<std::vector<float>>>* y = capture;
Per Åhgrenc59a5762017-12-11 21:34:19 +0100252 RTC_DCHECK(render_buffer);
peah522d71b2017-02-23 05:16:26 -0800253 RTC_DCHECK(y);
254 RTC_DCHECK_EQ(x.size(), NumBandsForRate(sample_rate_hz_));
255 RTC_DCHECK_EQ(y->size(), NumBandsForRate(sample_rate_hz_));
Per Åhgrence202a02019-09-02 17:01:19 +0200256 RTC_DCHECK_EQ(x[0].size(), num_render_channels_);
257 RTC_DCHECK_EQ((*y)[0].size(), num_capture_channels_);
258 RTC_DCHECK_EQ(x[0][0].size(), kBlockSize);
259 RTC_DCHECK_EQ((*y)[0][0].size(), kBlockSize);
Per Åhgrenf6aa5722019-09-10 18:05:17 +0200260
261 // Stack allocated data to use when the number of channels is low.
Per Åhgren0e3b1ff2019-09-25 12:09:37 +0200262 std::array<std::array<float, kFftLengthBy2>, kMaxNumChannelsOnStack> e_stack;
Per Åhgrenf6aa5722019-09-10 18:05:17 +0200263 std::array<std::array<float, kFftLengthBy2Plus1>, kMaxNumChannelsOnStack>
264 Y2_stack;
265 std::array<std::array<float, kFftLengthBy2Plus1>, kMaxNumChannelsOnStack>
266 E2_stack;
267 std::array<std::array<float, kFftLengthBy2Plus1>, kMaxNumChannelsOnStack>
268 R2_stack;
269 std::array<std::array<float, kFftLengthBy2Plus1>, kMaxNumChannelsOnStack>
Gustaf Ullberga63d1522021-06-11 14:02:53 +0200270 R2_unbounded_stack;
271 std::array<std::array<float, kFftLengthBy2Plus1>, kMaxNumChannelsOnStack>
Per Åhgrenf6aa5722019-09-10 18:05:17 +0200272 S2_linear_stack;
273 std::array<FftData, kMaxNumChannelsOnStack> Y_stack;
274 std::array<FftData, kMaxNumChannelsOnStack> E_stack;
275 std::array<FftData, kMaxNumChannelsOnStack> comfort_noise_stack;
276 std::array<FftData, kMaxNumChannelsOnStack> high_band_comfort_noise_stack;
277 std::array<SubtractorOutput, kMaxNumChannelsOnStack> subtractor_output_stack;
278
Per Åhgren0e3b1ff2019-09-25 12:09:37 +0200279 rtc::ArrayView<std::array<float, kFftLengthBy2>> e(e_stack.data(),
280 num_capture_channels_);
Per Åhgrenf6aa5722019-09-10 18:05:17 +0200281 rtc::ArrayView<std::array<float, kFftLengthBy2Plus1>> Y2(
282 Y2_stack.data(), num_capture_channels_);
283 rtc::ArrayView<std::array<float, kFftLengthBy2Plus1>> E2(
284 E2_stack.data(), num_capture_channels_);
285 rtc::ArrayView<std::array<float, kFftLengthBy2Plus1>> R2(
286 R2_stack.data(), num_capture_channels_);
Gustaf Ullberga63d1522021-06-11 14:02:53 +0200287 rtc::ArrayView<std::array<float, kFftLengthBy2Plus1>> R2_unbounded(
288 R2_unbounded_stack.data(), num_capture_channels_);
Per Åhgrenf6aa5722019-09-10 18:05:17 +0200289 rtc::ArrayView<std::array<float, kFftLengthBy2Plus1>> S2_linear(
290 S2_linear_stack.data(), num_capture_channels_);
291 rtc::ArrayView<FftData> Y(Y_stack.data(), num_capture_channels_);
292 rtc::ArrayView<FftData> E(E_stack.data(), num_capture_channels_);
293 rtc::ArrayView<FftData> comfort_noise(comfort_noise_stack.data(),
294 num_capture_channels_);
295 rtc::ArrayView<FftData> high_band_comfort_noise(
296 high_band_comfort_noise_stack.data(), num_capture_channels_);
297 rtc::ArrayView<SubtractorOutput> subtractor_output(
298 subtractor_output_stack.data(), num_capture_channels_);
299 if (NumChannelsOnHeap(num_capture_channels_) > 0) {
300 // If the stack-allocated space is too small, use the heap for storing the
301 // microphone data.
Per Åhgren0e3b1ff2019-09-25 12:09:37 +0200302 e = rtc::ArrayView<std::array<float, kFftLengthBy2>>(e_heap_.data(),
303 num_capture_channels_);
Per Åhgrenf6aa5722019-09-10 18:05:17 +0200304 Y2 = rtc::ArrayView<std::array<float, kFftLengthBy2Plus1>>(
305 Y2_heap_.data(), num_capture_channels_);
306 E2 = rtc::ArrayView<std::array<float, kFftLengthBy2Plus1>>(
307 E2_heap_.data(), num_capture_channels_);
308 R2 = rtc::ArrayView<std::array<float, kFftLengthBy2Plus1>>(
309 R2_heap_.data(), num_capture_channels_);
Gustaf Ullberga63d1522021-06-11 14:02:53 +0200310 R2_unbounded = rtc::ArrayView<std::array<float, kFftLengthBy2Plus1>>(
311 R2_unbounded_heap_.data(), num_capture_channels_);
Per Åhgrenf6aa5722019-09-10 18:05:17 +0200312 S2_linear = rtc::ArrayView<std::array<float, kFftLengthBy2Plus1>>(
313 S2_linear_heap_.data(), num_capture_channels_);
314 Y = rtc::ArrayView<FftData>(Y_heap_.data(), num_capture_channels_);
315 E = rtc::ArrayView<FftData>(E_heap_.data(), num_capture_channels_);
316 comfort_noise = rtc::ArrayView<FftData>(comfort_noise_heap_.data(),
317 num_capture_channels_);
318 high_band_comfort_noise = rtc::ArrayView<FftData>(
319 high_band_comfort_noise_heap_.data(), num_capture_channels_);
320 subtractor_output = rtc::ArrayView<SubtractorOutput>(
321 subtractor_output_heap_.data(), num_capture_channels_);
322 }
323
Per Åhgren119e2192019-10-18 08:50:50 +0200324 data_dumper_->DumpWav("aec3_echo_remover_capture_input", kBlockSize,
325 &(*y)[0][0][0], 16000, 1);
326 data_dumper_->DumpWav("aec3_echo_remover_render_input", kBlockSize,
327 &x[0][0][0], 16000, 1);
328 data_dumper_->DumpRaw("aec3_echo_remover_capture_input", (*y)[0][0]);
329 data_dumper_->DumpRaw("aec3_echo_remover_render_input", x[0][0]);
peah522d71b2017-02-23 05:16:26 -0800330
331 aec_state_.UpdateCaptureSaturation(capture_signal_saturation);
332
333 if (echo_path_variability.AudioPathChanged()) {
Per Åhgren88cf0502018-07-16 17:08:41 +0200334 // Ensure that the gain change is only acted on once per frame.
335 if (echo_path_variability.gain_change) {
336 if (gain_change_hangover_ == 0) {
337 constexpr int kMaxBlocksPerFrame = 3;
338 gain_change_hangover_ = kMaxBlocksPerFrame;
Sam Zackrissonffc84522019-10-15 13:43:02 +0200339 rtc::LoggingSeverity log_level =
340 config_.delay.log_warning_on_delay_changes ? rtc::LS_WARNING
Sam Zackrissonecc5b932020-01-14 12:58:12 +0100341 : rtc::LS_VERBOSE;
Sam Zackrissonffc84522019-10-15 13:43:02 +0200342 RTC_LOG_V(log_level)
343 << "Gain change detected at block " << block_counter_;
Per Åhgren88cf0502018-07-16 17:08:41 +0200344 } else {
345 echo_path_variability.gain_change = false;
346 }
347 }
348
Per Åhgren7bdf0732019-09-25 14:53:30 +0200349 subtractor_.HandleEchoPathChange(echo_path_variability);
peah86afe9d2017-04-06 15:45:32 -0700350 aec_state_.HandleEchoPathChange(echo_path_variability);
Per Åhgren88cf0502018-07-16 17:08:41 +0200351
352 if (echo_path_variability.delay_change !=
353 EchoPathVariability::DelayAdjustment::kNone) {
Gustaf Ullberg5ea57492019-11-05 15:19:02 +0100354 suppression_gain_.SetInitialState(true);
Per Åhgren88cf0502018-07-16 17:08:41 +0200355 }
356 }
357 if (gain_change_hangover_ > 0) {
358 --gain_change_hangover_;
peah522d71b2017-02-23 05:16:26 -0800359 }
360
peah522d71b2017-02-23 05:16:26 -0800361 // Analyze the render signal.
Per Åhgren5c532d32018-03-22 00:29:25 +0100362 render_signal_analyzer_.Update(*render_buffer,
Per Åhgren8718afb2019-10-15 10:31:35 +0200363 aec_state_.MinDirectPathFilterDelay());
peah522d71b2017-02-23 05:16:26 -0800364
Per Åhgren7bdf0732019-09-25 14:53:30 +0200365 // State transition.
Jesús de Vicente Peña02e9e442018-08-29 13:34:07 +0200366 if (aec_state_.TransitionTriggered()) {
Per Åhgren7bdf0732019-09-25 14:53:30 +0200367 subtractor_.ExitInitialState();
Gustaf Ullberg5ea57492019-11-05 15:19:02 +0100368 suppression_gain_.SetInitialState(false);
Per Åhgrena98c8072018-01-15 19:17:16 +0100369 }
Per Åhgren5c532d32018-03-22 00:29:25 +0100370
Per Åhgren7bdf0732019-09-25 14:53:30 +0200371 // Perform linear echo cancellation.
372 subtractor_.Process(*render_buffer, (*y)[0], render_signal_analyzer_,
373 aec_state_, subtractor_output);
374
Per Åhgren119e2192019-10-18 08:50:50 +0200375 // Compute spectra.
Gustaf Ullberga99b89b2019-09-23 16:03:12 +0200376 for (size_t ch = 0; ch < num_capture_channels_; ++ch) {
Per Åhgren0e3b1ff2019-09-25 12:09:37 +0200377 FormLinearFilterOutput(subtractor_output[ch], e[ch]);
Per Åhgren119e2192019-10-18 08:50:50 +0200378 WindowedPaddedFft(fft_, (*y)[0][ch], y_old_[ch], &Y[ch]);
Per Åhgren0e3b1ff2019-09-25 12:09:37 +0200379 WindowedPaddedFft(fft_, e[ch], e_old_[ch], &E[ch]);
Gustaf Ullberga99b89b2019-09-23 16:03:12 +0200380 LinearEchoPower(E[ch], Y[ch], &S2_linear[ch]);
381 Y[ch].Spectrum(optimization_, Y2[ch]);
382 E[ch].Spectrum(optimization_, E2[ch]);
383 }
peah522d71b2017-02-23 05:16:26 -0800384
Per Åhgrenc20a19c2019-11-13 11:12:29 +0100385 // Optionally return the linear filter output.
386 if (linear_output) {
387 RTC_DCHECK_GE(1, linear_output->size());
388 RTC_DCHECK_EQ(num_capture_channels_, linear_output[0].size());
389 for (size_t ch = 0; ch < num_capture_channels_; ++ch) {
390 RTC_DCHECK_EQ(kBlockSize, (*linear_output)[0][ch].size());
391 std::copy(e[ch].begin(), e[ch].end(), (*linear_output)[0][ch].begin());
392 }
393 }
394
peah522d71b2017-02-23 05:16:26 -0800395 // Update the AEC state information.
Per Åhgren119e2192019-10-18 08:50:50 +0200396 aec_state_.Update(external_delay, subtractor_.FilterFrequencyResponses(),
397 subtractor_.FilterImpulseResponses(), *render_buffer, E2,
398 Y2, subtractor_output);
Per Åhgren169c7fd2018-04-27 12:04:03 +0200399
peah522d71b2017-02-23 05:16:26 -0800400 // Choose the linear output.
Gustaf Ullbergaf3fdc02019-09-24 15:05:04 +0200401 const auto& Y_fft = aec_state_.UseLinearFilterOutput() ? E : Y;
Gustaf Ullberga99b89b2019-09-23 16:03:12 +0200402
Per Åhgren119e2192019-10-18 08:50:50 +0200403 data_dumper_->DumpWav("aec3_output_linear", kBlockSize, &(*y)[0][0][0], 16000,
404 1);
Per Åhgren0e3b1ff2019-09-25 12:09:37 +0200405 data_dumper_->DumpWav("aec3_output_linear2", kBlockSize, &e[0][0], 16000, 1);
peah522d71b2017-02-23 05:16:26 -0800406
Gustaf Ullbergcaaa9e72019-10-31 14:10:24 +0100407 // Estimate the comfort noise.
408 cng_.Compute(aec_state_.SaturatedCapture(), Y2, comfort_noise,
409 high_band_comfort_noise);
Jesús de Vicente Peña0faf0822018-09-24 12:48:28 +0200410
Per Åhgren8ee1ec82021-03-11 06:33:45 +0000411 // Only do the below processing if the output of the audio processing module
412 // is used.
Gustaf Ullberg5ea57492019-11-05 15:19:02 +0100413 std::array<float, kFftLengthBy2Plus1> G;
Per Åhgren8ee1ec82021-03-11 06:33:45 +0000414 if (capture_output_used_) {
415 // Estimate the residual echo power.
416 residual_echo_estimator_.Estimate(aec_state_, *render_buffer, S2_linear, Y2,
Gustaf Ullberga63d1522021-06-11 14:02:53 +0200417 suppression_gain_.IsDominantNearend(), R2,
418 R2_unbounded);
Jesús de Vicente Peña0faf0822018-09-24 12:48:28 +0200419
Per Åhgren8ee1ec82021-03-11 06:33:45 +0000420 // Suppressor nearend estimate.
421 if (aec_state_.UsableLinearEstimate()) {
422 // E2 is bound by Y2.
423 for (size_t ch = 0; ch < num_capture_channels_; ++ch) {
424 std::transform(E2[ch].begin(), E2[ch].end(), Y2[ch].begin(),
425 E2[ch].begin(),
426 [](float a, float b) { return std::min(a, b); });
427 }
428 }
429 const auto& nearend_spectrum = aec_state_.UsableLinearEstimate() ? E2 : Y2;
430
431 // Suppressor echo estimate.
432 const auto& echo_spectrum =
433 aec_state_.UsableLinearEstimate() ? S2_linear : R2;
434
435 // Determine if the suppressor should assume clock drift.
436 const bool clock_drift = config_.echo_removal_control.has_clock_drift ||
437 echo_path_variability.clock_drift;
438
439 // Compute preferred gains.
440 float high_bands_gain;
Gustaf Ullberga63d1522021-06-11 14:02:53 +0200441 suppression_gain_.GetGain(nearend_spectrum, echo_spectrum, R2, R2_unbounded,
Per Åhgren8ee1ec82021-03-11 06:33:45 +0000442 cng_.NoiseSpectrum(), render_signal_analyzer_,
443 aec_state_, x, clock_drift, &high_bands_gain, &G);
444
445 suppression_filter_.ApplyGain(comfort_noise, high_band_comfort_noise, G,
446 high_bands_gain, Y_fft, y);
447
448 } else {
449 G.fill(0.f);
450 }
peah522d71b2017-02-23 05:16:26 -0800451
peahe985b3f2017-02-28 22:08:53 -0800452 // Update the metrics.
Gustaf Ullbergcaaa9e72019-10-31 14:10:24 +0100453 metrics_.Update(aec_state_, cng_.NoiseSpectrum()[0], G);
peahe985b3f2017-02-28 22:08:53 -0800454
peah522d71b2017-02-23 05:16:26 -0800455 // Debug outputs for the purpose of development and analysis.
peah29103572017-07-11 02:54:02 -0700456 data_dumper_->DumpWav("aec3_echo_estimate", kBlockSize,
Per Åhgrenff045112020-03-20 11:20:39 +0100457 &subtractor_output[0].s_refined[0], 16000, 1);
Per Åhgren119e2192019-10-18 08:50:50 +0200458 data_dumper_->DumpRaw("aec3_output", (*y)[0][0]);
peah14c11a42017-07-11 06:13:43 -0700459 data_dumper_->DumpRaw("aec3_narrow_render",
460 render_signal_analyzer_.NarrowPeakBand() ? 1 : 0);
Gustaf Ullbergcaaa9e72019-10-31 14:10:24 +0100461 data_dumper_->DumpRaw("aec3_N2", cng_.NoiseSpectrum()[0]);
peah522d71b2017-02-23 05:16:26 -0800462 data_dumper_->DumpRaw("aec3_suppressor_gain", G);
Per Åhgren119e2192019-10-18 08:50:50 +0200463 data_dumper_->DumpWav("aec3_output",
464 rtc::ArrayView<const float>(&(*y)[0][0][0], kBlockSize),
465 16000, 1);
Per Åhgrenf6aa5722019-09-10 18:05:17 +0200466 data_dumper_->DumpRaw("aec3_using_subtractor_output[0]",
Per Åhgren5c532d32018-03-22 00:29:25 +0100467 aec_state_.UseLinearFilterOutput() ? 1 : 0);
Per Åhgrenf6aa5722019-09-10 18:05:17 +0200468 data_dumper_->DumpRaw("aec3_E2", E2[0]);
469 data_dumper_->DumpRaw("aec3_S2_linear", S2_linear[0]);
470 data_dumper_->DumpRaw("aec3_Y2", Y2[0]);
Jesús de Vicente Peña7682c6e2018-03-22 14:53:23 +0100471 data_dumper_->DumpRaw(
Sam Zackrisson98872dc2019-10-18 08:20:09 +0200472 "aec3_X2", render_buffer->Spectrum(
473 aec_state_.MinDirectPathFilterDelay())[/*channel=*/0]);
Per Åhgrenf6aa5722019-09-10 18:05:17 +0200474 data_dumper_->DumpRaw("aec3_R2", R2[0]);
Per Åhgren8718afb2019-10-15 10:31:35 +0200475 data_dumper_->DumpRaw("aec3_filter_delay",
476 aec_state_.MinDirectPathFilterDelay());
peah522d71b2017-02-23 05:16:26 -0800477 data_dumper_->DumpRaw("aec3_capture_saturation",
478 aec_state_.SaturatedCapture() ? 1 : 0);
479}
peah69221db2017-01-27 03:28:19 -0800480
Per Åhgren22754392018-08-10 18:37:38 +0200481void EchoRemoverImpl::FormLinearFilterOutput(
Per Åhgren22754392018-08-10 18:37:38 +0200482 const SubtractorOutput& subtractor_output,
483 rtc::ArrayView<float> output) {
Per Åhgrenff045112020-03-20 11:20:39 +0100484 RTC_DCHECK_EQ(subtractor_output.e_refined.size(), output.size());
Per Åhgren9d661982020-03-20 11:26:48 +0100485 RTC_DCHECK_EQ(subtractor_output.e_coarse.size(), output.size());
Per Åhgrenff045112020-03-20 11:20:39 +0100486 bool use_refined_output = true;
Per Åhgren9d661982020-03-20 11:26:48 +0100487 if (use_coarse_filter_output_) {
Per Åhgrenff045112020-03-20 11:20:39 +0100488 // As the output of the refined adaptive filter generally should be better
Per Åhgren9d661982020-03-20 11:26:48 +0100489 // than the coarse filter output, add a margin and threshold for when
490 // choosing the coarse filter output.
491 if (subtractor_output.e2_coarse < 0.9f * subtractor_output.e2_refined &&
Per Åhgren22754392018-08-10 18:37:38 +0200492 subtractor_output.y2 > 30.f * 30.f * kBlockSize &&
Per Åhgrenff045112020-03-20 11:20:39 +0100493 (subtractor_output.s2_refined > 60.f * 60.f * kBlockSize ||
Per Åhgren9d661982020-03-20 11:26:48 +0100494 subtractor_output.s2_coarse > 60.f * 60.f * kBlockSize)) {
Per Åhgrenff045112020-03-20 11:20:39 +0100495 use_refined_output = false;
Per Åhgren22754392018-08-10 18:37:38 +0200496 } else {
Per Åhgrenff045112020-03-20 11:20:39 +0100497 // If the refined filter is diverged, choose the filter output that has
498 // the lowest power.
Per Åhgren9d661982020-03-20 11:26:48 +0100499 if (subtractor_output.e2_coarse < subtractor_output.e2_refined &&
Per Åhgrenff045112020-03-20 11:20:39 +0100500 subtractor_output.y2 < subtractor_output.e2_refined) {
501 use_refined_output = false;
Per Åhgren22754392018-08-10 18:37:38 +0200502 }
503 }
504 }
505
Per Åhgrenff045112020-03-20 11:20:39 +0100506 SignalTransition(refined_filter_output_last_selected_
507 ? subtractor_output.e_refined
Per Åhgren9d661982020-03-20 11:26:48 +0100508 : subtractor_output.e_coarse,
Per Åhgrenff045112020-03-20 11:20:39 +0100509 use_refined_output ? subtractor_output.e_refined
Per Åhgren9d661982020-03-20 11:26:48 +0100510 : subtractor_output.e_coarse,
Per Åhgrenff045112020-03-20 11:20:39 +0100511 output);
512 refined_filter_output_last_selected_ = use_refined_output;
Per Åhgren22754392018-08-10 18:37:38 +0200513}
514
peah69221db2017-01-27 03:28:19 -0800515} // namespace
516
Gustaf Ullbergbd83b912017-10-18 12:32:42 +0200517EchoRemover* EchoRemover::Create(const EchoCanceller3Config& config,
Per Åhgrence202a02019-09-02 17:01:19 +0200518 int sample_rate_hz,
519 size_t num_render_channels,
520 size_t num_capture_channels) {
521 return new EchoRemoverImpl(config, sample_rate_hz, num_render_channels,
522 num_capture_channels);
peah69221db2017-01-27 03:28:19 -0800523}
524
525} // namespace webrtc