peah | 69221db | 2017-01-27 03:28:19 -0800 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. |
| 3 | * |
| 4 | * Use of this source code is governed by a BSD-style license |
| 5 | * that can be found in the LICENSE file in the root of the source |
| 6 | * tree. An additional intellectual property rights grant can be found |
| 7 | * in the file PATENTS. All contributing project authors may |
| 8 | * be found in the AUTHORS file in the root of the source tree. |
| 9 | */ |
Mirko Bonadei | 92ea95e | 2017-09-15 06:47:31 +0200 | [diff] [blame] | 10 | #include "modules/audio_processing/aec3/echo_remover.h" |
peah | 69221db | 2017-01-27 03:28:19 -0800 | [diff] [blame] | 11 | |
peah | 86afe9d | 2017-04-06 15:45:32 -0700 | [diff] [blame] | 12 | #include <math.h> |
Yves Gerey | 988cc08 | 2018-10-23 12:03:01 +0200 | [diff] [blame] | 13 | #include <stddef.h> |
Jonas Olsson | a4d8737 | 2019-07-05 19:08:33 +0200 | [diff] [blame] | 14 | |
peah | 69221db | 2017-01-27 03:28:19 -0800 | [diff] [blame] | 15 | #include <algorithm> |
Yves Gerey | 988cc08 | 2018-10-23 12:03:01 +0200 | [diff] [blame] | 16 | #include <array> |
Mirko Bonadei | dbce090 | 2019-03-15 07:39:02 +0100 | [diff] [blame] | 17 | #include <cmath> |
peah | 522d71b | 2017-02-23 05:16:26 -0800 | [diff] [blame] | 18 | #include <memory> |
peah | 69221db | 2017-01-27 03:28:19 -0800 | [diff] [blame] | 19 | |
Mirko Bonadei | 92ea95e | 2017-09-15 06:47:31 +0200 | [diff] [blame] | 20 | #include "api/array_view.h" |
| 21 | #include "modules/audio_processing/aec3/aec3_common.h" |
Yves Gerey | 988cc08 | 2018-10-23 12:03:01 +0200 | [diff] [blame] | 22 | #include "modules/audio_processing/aec3/aec3_fft.h" |
Mirko Bonadei | 92ea95e | 2017-09-15 06:47:31 +0200 | [diff] [blame] | 23 | #include "modules/audio_processing/aec3/aec_state.h" |
| 24 | #include "modules/audio_processing/aec3/comfort_noise_generator.h" |
| 25 | #include "modules/audio_processing/aec3/echo_path_variability.h" |
| 26 | #include "modules/audio_processing/aec3/echo_remover_metrics.h" |
| 27 | #include "modules/audio_processing/aec3/fft_data.h" |
Mirko Bonadei | 92ea95e | 2017-09-15 06:47:31 +0200 | [diff] [blame] | 28 | #include "modules/audio_processing/aec3/render_buffer.h" |
Yves Gerey | 988cc08 | 2018-10-23 12:03:01 +0200 | [diff] [blame] | 29 | #include "modules/audio_processing/aec3/render_signal_analyzer.h" |
Mirko Bonadei | 92ea95e | 2017-09-15 06:47:31 +0200 | [diff] [blame] | 30 | #include "modules/audio_processing/aec3/residual_echo_estimator.h" |
| 31 | #include "modules/audio_processing/aec3/subtractor.h" |
Yves Gerey | 988cc08 | 2018-10-23 12:03:01 +0200 | [diff] [blame] | 32 | #include "modules/audio_processing/aec3/subtractor_output.h" |
Mirko Bonadei | 92ea95e | 2017-09-15 06:47:31 +0200 | [diff] [blame] | 33 | #include "modules/audio_processing/aec3/suppression_filter.h" |
| 34 | #include "modules/audio_processing/aec3/suppression_gain.h" |
| 35 | #include "modules/audio_processing/logging/apm_data_dumper.h" |
Steve Anton | 10542f2 | 2019-01-11 09:11:00 -0800 | [diff] [blame] | 36 | #include "rtc_base/atomic_ops.h" |
Yves Gerey | 988cc08 | 2018-10-23 12:03:01 +0200 | [diff] [blame] | 37 | #include "rtc_base/checks.h" |
Per Åhgren | 88cf050 | 2018-07-16 17:08:41 +0200 | [diff] [blame] | 38 | #include "rtc_base/logging.h" |
peah | 69221db | 2017-01-27 03:28:19 -0800 | [diff] [blame] | 39 | |
| 40 | namespace webrtc { |
| 41 | |
| 42 | namespace { |
peah | 522d71b | 2017-02-23 05:16:26 -0800 | [diff] [blame] | 43 | |
Per Åhgren | f6aa572 | 2019-09-10 18:05:17 +0200 | [diff] [blame] | 44 | // Maximum number of channels for which the capture channel data is stored on |
| 45 | // the stack. If the number of channels are larger than this, they are stored |
| 46 | // using scratch memory that is pre-allocated on the heap. The reason for this |
| 47 | // partitioning is not to waste heap space for handling the more common numbers |
| 48 | // of channels, while at the same time not limiting the support for higher |
| 49 | // numbers of channels by enforcing the capture channel data to be stored on the |
| 50 | // stack using a fixed maximum value. |
| 51 | constexpr size_t kMaxNumChannelsOnStack = 2; |
| 52 | |
| 53 | // Chooses the number of channels to store on the heap when that is required due |
| 54 | // to the number of capture channels being larger than the pre-defined number |
| 55 | // of channels to store on the stack. |
| 56 | size_t NumChannelsOnHeap(size_t num_capture_channels) { |
| 57 | return num_capture_channels > kMaxNumChannelsOnStack ? num_capture_channels |
| 58 | : 0; |
| 59 | } |
| 60 | |
peah | 522d71b | 2017-02-23 05:16:26 -0800 | [diff] [blame] | 61 | void LinearEchoPower(const FftData& E, |
| 62 | const FftData& Y, |
| 63 | std::array<float, kFftLengthBy2Plus1>* S2) { |
| 64 | for (size_t k = 0; k < E.re.size(); ++k) { |
| 65 | (*S2)[k] = (Y.re[k] - E.re[k]) * (Y.re[k] - E.re[k]) + |
| 66 | (Y.im[k] - E.im[k]) * (Y.im[k] - E.im[k]); |
| 67 | } |
| 68 | } |
| 69 | |
Per Åhgren | 2275439 | 2018-08-10 18:37:38 +0200 | [diff] [blame] | 70 | // Fades between two input signals using a fix-sized transition. |
| 71 | void SignalTransition(rtc::ArrayView<const float> from, |
| 72 | rtc::ArrayView<const float> to, |
| 73 | rtc::ArrayView<float> out) { |
Gustaf Ullberg | 7911d37 | 2019-09-24 16:31:01 +0200 | [diff] [blame] | 74 | if (from == to) { |
| 75 | RTC_DCHECK_EQ(to.size(), out.size()); |
| 76 | std::copy(to.begin(), to.end(), out.begin()); |
| 77 | } else { |
| 78 | constexpr size_t kTransitionSize = 30; |
| 79 | constexpr float kOneByTransitionSizePlusOne = 1.f / (kTransitionSize + 1); |
Per Åhgren | 2275439 | 2018-08-10 18:37:38 +0200 | [diff] [blame] | 80 | |
Gustaf Ullberg | 7911d37 | 2019-09-24 16:31:01 +0200 | [diff] [blame] | 81 | RTC_DCHECK_EQ(from.size(), to.size()); |
| 82 | RTC_DCHECK_EQ(from.size(), out.size()); |
| 83 | RTC_DCHECK_LE(kTransitionSize, out.size()); |
Per Åhgren | 2275439 | 2018-08-10 18:37:38 +0200 | [diff] [blame] | 84 | |
Gustaf Ullberg | 7911d37 | 2019-09-24 16:31:01 +0200 | [diff] [blame] | 85 | for (size_t k = 0; k < kTransitionSize; ++k) { |
| 86 | float a = (k + 1) * kOneByTransitionSizePlusOne; |
| 87 | out[k] = a * to[k] + (1.f - a) * from[k]; |
| 88 | } |
| 89 | |
| 90 | std::copy(to.begin() + kTransitionSize, to.end(), |
| 91 | out.begin() + kTransitionSize); |
Per Åhgren | 2275439 | 2018-08-10 18:37:38 +0200 | [diff] [blame] | 92 | } |
Per Åhgren | 2275439 | 2018-08-10 18:37:38 +0200 | [diff] [blame] | 93 | } |
| 94 | |
Per Åhgren | 169c7fd | 2018-04-27 12:04:03 +0200 | [diff] [blame] | 95 | // Computes a windowed (square root Hanning) padded FFT and updates the related |
| 96 | // memory. |
| 97 | void WindowedPaddedFft(const Aec3Fft& fft, |
| 98 | rtc::ArrayView<const float> v, |
| 99 | rtc::ArrayView<float> v_old, |
| 100 | FftData* V) { |
| 101 | fft.PaddedFft(v, v_old, Aec3Fft::Window::kSqrtHanning, V); |
| 102 | std::copy(v.begin(), v.end(), v_old.begin()); |
| 103 | } |
| 104 | |
peah | 522d71b | 2017-02-23 05:16:26 -0800 | [diff] [blame] | 105 | // Class for removing the echo from the capture signal. |
peah | 69221db | 2017-01-27 03:28:19 -0800 | [diff] [blame] | 106 | class EchoRemoverImpl final : public EchoRemover { |
| 107 | public: |
Per Åhgren | ce202a0 | 2019-09-02 17:01:19 +0200 | [diff] [blame] | 108 | EchoRemoverImpl(const EchoCanceller3Config& config, |
| 109 | int sample_rate_hz, |
| 110 | size_t num_render_channels, |
| 111 | size_t num_capture_channels); |
peah | 69221db | 2017-01-27 03:28:19 -0800 | [diff] [blame] | 112 | ~EchoRemoverImpl() override; |
Per Åhgren | f6aa572 | 2019-09-10 18:05:17 +0200 | [diff] [blame] | 113 | EchoRemoverImpl(const EchoRemoverImpl&) = delete; |
| 114 | EchoRemoverImpl& operator=(const EchoRemoverImpl&) = delete; |
peah | 69221db | 2017-01-27 03:28:19 -0800 | [diff] [blame] | 115 | |
Gustaf Ullberg | 332150d | 2017-11-22 14:17:39 +0100 | [diff] [blame] | 116 | void GetMetrics(EchoControl::Metrics* metrics) const override; |
| 117 | |
peah | 522d71b | 2017-02-23 05:16:26 -0800 | [diff] [blame] | 118 | // Removes the echo from a block of samples from the capture signal. The |
| 119 | // supplied render signal is assumed to be pre-aligned with the capture |
| 120 | // signal. |
Per Åhgren | ce202a0 | 2019-09-02 17:01:19 +0200 | [diff] [blame] | 121 | void ProcessCapture( |
| 122 | EchoPathVariability echo_path_variability, |
| 123 | bool capture_signal_saturation, |
| 124 | const absl::optional<DelayEstimate>& external_delay, |
| 125 | RenderBuffer* render_buffer, |
Per Åhgren | c20a19c | 2019-11-13 11:12:29 +0100 | [diff] [blame] | 126 | std::vector<std::vector<std::vector<float>>>* linear_output, |
Per Åhgren | ce202a0 | 2019-09-02 17:01:19 +0200 | [diff] [blame] | 127 | std::vector<std::vector<std::vector<float>>>* capture) override; |
peah | 69221db | 2017-01-27 03:28:19 -0800 | [diff] [blame] | 128 | |
peah | 522d71b | 2017-02-23 05:16:26 -0800 | [diff] [blame] | 129 | // Updates the status on whether echo leakage is detected in the output of the |
| 130 | // echo remover. |
| 131 | void UpdateEchoLeakageStatus(bool leakage_detected) override { |
| 132 | echo_leakage_detected_ = leakage_detected; |
| 133 | } |
peah | 69221db | 2017-01-27 03:28:19 -0800 | [diff] [blame] | 134 | |
Per Åhgren | 8ee1ec8 | 2021-03-11 06:33:45 +0000 | [diff] [blame] | 135 | void SetCaptureOutputUsage(bool capture_output_used) override { |
| 136 | capture_output_used_ = capture_output_used; |
| 137 | } |
| 138 | |
peah | 69221db | 2017-01-27 03:28:19 -0800 | [diff] [blame] | 139 | private: |
Per Åhgren | 9d66198 | 2020-03-20 11:26:48 +0100 | [diff] [blame] | 140 | // Selects which of the coarse and refined linear filter outputs that is most |
Per Åhgren | 2275439 | 2018-08-10 18:37:38 +0200 | [diff] [blame] | 141 | // appropriate to pass to the suppressor and forms the linear filter output by |
| 142 | // smoothly transition between those. |
Gustaf Ullberg | 68d6d44 | 2019-01-29 10:08:15 +0100 | [diff] [blame] | 143 | void FormLinearFilterOutput(const SubtractorOutput& subtractor_output, |
Per Åhgren | 2275439 | 2018-08-10 18:37:38 +0200 | [diff] [blame] | 144 | rtc::ArrayView<float> output); |
Per Åhgren | 7802675 | 2018-08-01 16:24:08 +0200 | [diff] [blame] | 145 | |
peah | 522d71b | 2017-02-23 05:16:26 -0800 | [diff] [blame] | 146 | static int instance_count_; |
Gustaf Ullberg | bd83b91 | 2017-10-18 12:32:42 +0200 | [diff] [blame] | 147 | const EchoCanceller3Config config_; |
peah | 522d71b | 2017-02-23 05:16:26 -0800 | [diff] [blame] | 148 | const Aec3Fft fft_; |
| 149 | std::unique_ptr<ApmDataDumper> data_dumper_; |
| 150 | const Aec3Optimization optimization_; |
peah | 69221db | 2017-01-27 03:28:19 -0800 | [diff] [blame] | 151 | const int sample_rate_hz_; |
Per Åhgren | ce202a0 | 2019-09-02 17:01:19 +0200 | [diff] [blame] | 152 | const size_t num_render_channels_; |
| 153 | const size_t num_capture_channels_; |
Per Åhgren | 9d66198 | 2020-03-20 11:26:48 +0100 | [diff] [blame] | 154 | const bool use_coarse_filter_output_; |
Per Åhgren | 7bdf073 | 2019-09-25 14:53:30 +0200 | [diff] [blame] | 155 | Subtractor subtractor_; |
Gustaf Ullberg | 5ea5749 | 2019-11-05 15:19:02 +0100 | [diff] [blame] | 156 | SuppressionGain suppression_gain_; |
Gustaf Ullberg | caaa9e7 | 2019-10-31 14:10:24 +0100 | [diff] [blame] | 157 | ComfortNoiseGenerator cng_; |
peah | 522d71b | 2017-02-23 05:16:26 -0800 | [diff] [blame] | 158 | SuppressionFilter suppression_filter_; |
peah | 522d71b | 2017-02-23 05:16:26 -0800 | [diff] [blame] | 159 | RenderSignalAnalyzer render_signal_analyzer_; |
Per Åhgren | b4161d3 | 2019-10-08 12:35:47 +0200 | [diff] [blame] | 160 | ResidualEchoEstimator residual_echo_estimator_; |
peah | 522d71b | 2017-02-23 05:16:26 -0800 | [diff] [blame] | 161 | bool echo_leakage_detected_ = false; |
Per Åhgren | 8ee1ec8 | 2021-03-11 06:33:45 +0000 | [diff] [blame] | 162 | bool capture_output_used_ = true; |
peah | 522d71b | 2017-02-23 05:16:26 -0800 | [diff] [blame] | 163 | AecState aec_state_; |
peah | e985b3f | 2017-02-28 22:08:53 -0800 | [diff] [blame] | 164 | EchoRemoverMetrics metrics_; |
Gustaf Ullberg | a99b89b | 2019-09-23 16:03:12 +0200 | [diff] [blame] | 165 | std::vector<std::array<float, kFftLengthBy2>> e_old_; |
| 166 | std::vector<std::array<float, kFftLengthBy2>> y_old_; |
Per Åhgren | 88cf050 | 2018-07-16 17:08:41 +0200 | [diff] [blame] | 167 | size_t block_counter_ = 0; |
| 168 | int gain_change_hangover_ = 0; |
Per Åhgren | ff04511 | 2020-03-20 11:20:39 +0100 | [diff] [blame] | 169 | bool refined_filter_output_last_selected_ = true; |
peah | 69221db | 2017-01-27 03:28:19 -0800 | [diff] [blame] | 170 | |
Per Åhgren | 0e3b1ff | 2019-09-25 12:09:37 +0200 | [diff] [blame] | 171 | std::vector<std::array<float, kFftLengthBy2>> e_heap_; |
Per Åhgren | f6aa572 | 2019-09-10 18:05:17 +0200 | [diff] [blame] | 172 | std::vector<std::array<float, kFftLengthBy2Plus1>> Y2_heap_; |
| 173 | std::vector<std::array<float, kFftLengthBy2Plus1>> E2_heap_; |
| 174 | std::vector<std::array<float, kFftLengthBy2Plus1>> R2_heap_; |
Gustaf Ullberg | a63d152 | 2021-06-11 14:02:53 +0200 | [diff] [blame^] | 175 | std::vector<std::array<float, kFftLengthBy2Plus1>> R2_unbounded_heap_; |
Per Åhgren | f6aa572 | 2019-09-10 18:05:17 +0200 | [diff] [blame] | 176 | std::vector<std::array<float, kFftLengthBy2Plus1>> S2_linear_heap_; |
| 177 | std::vector<FftData> Y_heap_; |
| 178 | std::vector<FftData> E_heap_; |
| 179 | std::vector<FftData> comfort_noise_heap_; |
| 180 | std::vector<FftData> high_band_comfort_noise_heap_; |
| 181 | std::vector<SubtractorOutput> subtractor_output_heap_; |
peah | 69221db | 2017-01-27 03:28:19 -0800 | [diff] [blame] | 182 | }; |
| 183 | |
peah | 522d71b | 2017-02-23 05:16:26 -0800 | [diff] [blame] | 184 | int EchoRemoverImpl::instance_count_ = 0; |
| 185 | |
Gustaf Ullberg | bd83b91 | 2017-10-18 12:32:42 +0200 | [diff] [blame] | 186 | EchoRemoverImpl::EchoRemoverImpl(const EchoCanceller3Config& config, |
Per Åhgren | ce202a0 | 2019-09-02 17:01:19 +0200 | [diff] [blame] | 187 | int sample_rate_hz, |
| 188 | size_t num_render_channels, |
| 189 | size_t num_capture_channels) |
peah | 8cee56f | 2017-08-24 22:36:53 -0700 | [diff] [blame] | 190 | : config_(config), |
| 191 | fft_(), |
aleloi | 88b82b5 | 2017-02-23 06:27:03 -0800 | [diff] [blame] | 192 | data_dumper_( |
peah | 522d71b | 2017-02-23 05:16:26 -0800 | [diff] [blame] | 193 | new ApmDataDumper(rtc::AtomicOps::Increment(&instance_count_))), |
| 194 | optimization_(DetectOptimization()), |
| 195 | sample_rate_hz_(sample_rate_hz), |
Per Åhgren | ce202a0 | 2019-09-02 17:01:19 +0200 | [diff] [blame] | 196 | num_render_channels_(num_render_channels), |
| 197 | num_capture_channels_(num_capture_channels), |
Per Åhgren | 9d66198 | 2020-03-20 11:26:48 +0100 | [diff] [blame] | 198 | use_coarse_filter_output_( |
| 199 | config_.filter.enable_coarse_filter_output_usage), |
Per Åhgren | 7bdf073 | 2019-09-25 14:53:30 +0200 | [diff] [blame] | 200 | subtractor_(config, |
| 201 | num_render_channels_, |
| 202 | num_capture_channels_, |
| 203 | data_dumper_.get(), |
| 204 | optimization_), |
Gustaf Ullberg | 5ea5749 | 2019-11-05 15:19:02 +0100 | [diff] [blame] | 205 | suppression_gain_(config_, |
| 206 | optimization_, |
| 207 | sample_rate_hz, |
| 208 | num_capture_channels), |
Per Åhgren | a388b75 | 2020-03-25 07:31:47 +0100 | [diff] [blame] | 209 | cng_(config_, optimization_, num_capture_channels_), |
Gustaf Ullberg | af3fdc0 | 2019-09-24 15:05:04 +0200 | [diff] [blame] | 210 | suppression_filter_(optimization_, |
| 211 | sample_rate_hz_, |
| 212 | num_capture_channels_), |
Per Åhgren | 971de07 | 2018-03-14 23:23:47 +0100 | [diff] [blame] | 213 | render_signal_analyzer_(config_), |
Per Åhgren | b4161d3 | 2019-10-08 12:35:47 +0200 | [diff] [blame] | 214 | residual_echo_estimator_(config_, num_render_channels), |
Sam Zackrisson | 8f736c0 | 2019-10-01 12:47:53 +0200 | [diff] [blame] | 215 | aec_state_(config_, num_capture_channels_), |
Gustaf Ullberg | 5ea5749 | 2019-11-05 15:19:02 +0100 | [diff] [blame] | 216 | e_old_(num_capture_channels_, {0.f}), |
| 217 | y_old_(num_capture_channels_, {0.f}), |
| 218 | e_heap_(NumChannelsOnHeap(num_capture_channels_), {0.f}), |
Per Åhgren | f6aa572 | 2019-09-10 18:05:17 +0200 | [diff] [blame] | 219 | Y2_heap_(NumChannelsOnHeap(num_capture_channels_)), |
| 220 | E2_heap_(NumChannelsOnHeap(num_capture_channels_)), |
| 221 | R2_heap_(NumChannelsOnHeap(num_capture_channels_)), |
Gustaf Ullberg | a63d152 | 2021-06-11 14:02:53 +0200 | [diff] [blame^] | 222 | R2_unbounded_heap_(NumChannelsOnHeap(num_capture_channels_)), |
Per Åhgren | f6aa572 | 2019-09-10 18:05:17 +0200 | [diff] [blame] | 223 | S2_linear_heap_(NumChannelsOnHeap(num_capture_channels_)), |
| 224 | Y_heap_(NumChannelsOnHeap(num_capture_channels_)), |
| 225 | E_heap_(NumChannelsOnHeap(num_capture_channels_)), |
| 226 | comfort_noise_heap_(NumChannelsOnHeap(num_capture_channels_)), |
| 227 | high_band_comfort_noise_heap_(NumChannelsOnHeap(num_capture_channels_)), |
| 228 | subtractor_output_heap_(NumChannelsOnHeap(num_capture_channels_)) { |
peah | 522d71b | 2017-02-23 05:16:26 -0800 | [diff] [blame] | 229 | RTC_DCHECK(ValidFullBandRate(sample_rate_hz)); |
peah | 69221db | 2017-01-27 03:28:19 -0800 | [diff] [blame] | 230 | } |
| 231 | |
| 232 | EchoRemoverImpl::~EchoRemoverImpl() = default; |
| 233 | |
Gustaf Ullberg | 332150d | 2017-11-22 14:17:39 +0100 | [diff] [blame] | 234 | void EchoRemoverImpl::GetMetrics(EchoControl::Metrics* metrics) const { |
| 235 | // Echo return loss (ERL) is inverted to go from gain to attenuation. |
Mirko Bonadei | dbce090 | 2019-03-15 07:39:02 +0100 | [diff] [blame] | 236 | metrics->echo_return_loss = -10.0 * std::log10(aec_state_.ErlTimeDomain()); |
Gustaf Ullberg | 332150d | 2017-11-22 14:17:39 +0100 | [diff] [blame] | 237 | metrics->echo_return_loss_enhancement = |
Jesús de Vicente Peña | e9a7e90 | 2018-09-27 11:49:39 +0200 | [diff] [blame] | 238 | Log2TodB(aec_state_.FullBandErleLog2()); |
Gustaf Ullberg | 332150d | 2017-11-22 14:17:39 +0100 | [diff] [blame] | 239 | } |
| 240 | |
peah | cf02cf1 | 2017-04-05 14:18:07 -0700 | [diff] [blame] | 241 | void EchoRemoverImpl::ProcessCapture( |
Per Åhgren | 88cf050 | 2018-07-16 17:08:41 +0200 | [diff] [blame] | 242 | EchoPathVariability echo_path_variability, |
peah | 69221db | 2017-01-27 03:28:19 -0800 | [diff] [blame] | 243 | bool capture_signal_saturation, |
Danil Chapovalov | db9f7ab | 2018-06-19 10:50:11 +0200 | [diff] [blame] | 244 | const absl::optional<DelayEstimate>& external_delay, |
Per Åhgren | c59a576 | 2017-12-11 21:34:19 +0100 | [diff] [blame] | 245 | RenderBuffer* render_buffer, |
Per Åhgren | c20a19c | 2019-11-13 11:12:29 +0100 | [diff] [blame] | 246 | std::vector<std::vector<std::vector<float>>>* linear_output, |
Per Åhgren | ce202a0 | 2019-09-02 17:01:19 +0200 | [diff] [blame] | 247 | std::vector<std::vector<std::vector<float>>>* capture) { |
Per Åhgren | 88cf050 | 2018-07-16 17:08:41 +0200 | [diff] [blame] | 248 | ++block_counter_; |
Per Åhgren | ce202a0 | 2019-09-02 17:01:19 +0200 | [diff] [blame] | 249 | const std::vector<std::vector<std::vector<float>>>& x = |
| 250 | render_buffer->Block(0); |
| 251 | std::vector<std::vector<std::vector<float>>>* y = capture; |
Per Åhgren | c59a576 | 2017-12-11 21:34:19 +0100 | [diff] [blame] | 252 | RTC_DCHECK(render_buffer); |
peah | 522d71b | 2017-02-23 05:16:26 -0800 | [diff] [blame] | 253 | RTC_DCHECK(y); |
| 254 | RTC_DCHECK_EQ(x.size(), NumBandsForRate(sample_rate_hz_)); |
| 255 | RTC_DCHECK_EQ(y->size(), NumBandsForRate(sample_rate_hz_)); |
Per Åhgren | ce202a0 | 2019-09-02 17:01:19 +0200 | [diff] [blame] | 256 | RTC_DCHECK_EQ(x[0].size(), num_render_channels_); |
| 257 | RTC_DCHECK_EQ((*y)[0].size(), num_capture_channels_); |
| 258 | RTC_DCHECK_EQ(x[0][0].size(), kBlockSize); |
| 259 | RTC_DCHECK_EQ((*y)[0][0].size(), kBlockSize); |
Per Åhgren | f6aa572 | 2019-09-10 18:05:17 +0200 | [diff] [blame] | 260 | |
| 261 | // Stack allocated data to use when the number of channels is low. |
Per Åhgren | 0e3b1ff | 2019-09-25 12:09:37 +0200 | [diff] [blame] | 262 | std::array<std::array<float, kFftLengthBy2>, kMaxNumChannelsOnStack> e_stack; |
Per Åhgren | f6aa572 | 2019-09-10 18:05:17 +0200 | [diff] [blame] | 263 | std::array<std::array<float, kFftLengthBy2Plus1>, kMaxNumChannelsOnStack> |
| 264 | Y2_stack; |
| 265 | std::array<std::array<float, kFftLengthBy2Plus1>, kMaxNumChannelsOnStack> |
| 266 | E2_stack; |
| 267 | std::array<std::array<float, kFftLengthBy2Plus1>, kMaxNumChannelsOnStack> |
| 268 | R2_stack; |
| 269 | std::array<std::array<float, kFftLengthBy2Plus1>, kMaxNumChannelsOnStack> |
Gustaf Ullberg | a63d152 | 2021-06-11 14:02:53 +0200 | [diff] [blame^] | 270 | R2_unbounded_stack; |
| 271 | std::array<std::array<float, kFftLengthBy2Plus1>, kMaxNumChannelsOnStack> |
Per Åhgren | f6aa572 | 2019-09-10 18:05:17 +0200 | [diff] [blame] | 272 | S2_linear_stack; |
| 273 | std::array<FftData, kMaxNumChannelsOnStack> Y_stack; |
| 274 | std::array<FftData, kMaxNumChannelsOnStack> E_stack; |
| 275 | std::array<FftData, kMaxNumChannelsOnStack> comfort_noise_stack; |
| 276 | std::array<FftData, kMaxNumChannelsOnStack> high_band_comfort_noise_stack; |
| 277 | std::array<SubtractorOutput, kMaxNumChannelsOnStack> subtractor_output_stack; |
| 278 | |
Per Åhgren | 0e3b1ff | 2019-09-25 12:09:37 +0200 | [diff] [blame] | 279 | rtc::ArrayView<std::array<float, kFftLengthBy2>> e(e_stack.data(), |
| 280 | num_capture_channels_); |
Per Åhgren | f6aa572 | 2019-09-10 18:05:17 +0200 | [diff] [blame] | 281 | rtc::ArrayView<std::array<float, kFftLengthBy2Plus1>> Y2( |
| 282 | Y2_stack.data(), num_capture_channels_); |
| 283 | rtc::ArrayView<std::array<float, kFftLengthBy2Plus1>> E2( |
| 284 | E2_stack.data(), num_capture_channels_); |
| 285 | rtc::ArrayView<std::array<float, kFftLengthBy2Plus1>> R2( |
| 286 | R2_stack.data(), num_capture_channels_); |
Gustaf Ullberg | a63d152 | 2021-06-11 14:02:53 +0200 | [diff] [blame^] | 287 | rtc::ArrayView<std::array<float, kFftLengthBy2Plus1>> R2_unbounded( |
| 288 | R2_unbounded_stack.data(), num_capture_channels_); |
Per Åhgren | f6aa572 | 2019-09-10 18:05:17 +0200 | [diff] [blame] | 289 | rtc::ArrayView<std::array<float, kFftLengthBy2Plus1>> S2_linear( |
| 290 | S2_linear_stack.data(), num_capture_channels_); |
| 291 | rtc::ArrayView<FftData> Y(Y_stack.data(), num_capture_channels_); |
| 292 | rtc::ArrayView<FftData> E(E_stack.data(), num_capture_channels_); |
| 293 | rtc::ArrayView<FftData> comfort_noise(comfort_noise_stack.data(), |
| 294 | num_capture_channels_); |
| 295 | rtc::ArrayView<FftData> high_band_comfort_noise( |
| 296 | high_band_comfort_noise_stack.data(), num_capture_channels_); |
| 297 | rtc::ArrayView<SubtractorOutput> subtractor_output( |
| 298 | subtractor_output_stack.data(), num_capture_channels_); |
| 299 | if (NumChannelsOnHeap(num_capture_channels_) > 0) { |
| 300 | // If the stack-allocated space is too small, use the heap for storing the |
| 301 | // microphone data. |
Per Åhgren | 0e3b1ff | 2019-09-25 12:09:37 +0200 | [diff] [blame] | 302 | e = rtc::ArrayView<std::array<float, kFftLengthBy2>>(e_heap_.data(), |
| 303 | num_capture_channels_); |
Per Åhgren | f6aa572 | 2019-09-10 18:05:17 +0200 | [diff] [blame] | 304 | Y2 = rtc::ArrayView<std::array<float, kFftLengthBy2Plus1>>( |
| 305 | Y2_heap_.data(), num_capture_channels_); |
| 306 | E2 = rtc::ArrayView<std::array<float, kFftLengthBy2Plus1>>( |
| 307 | E2_heap_.data(), num_capture_channels_); |
| 308 | R2 = rtc::ArrayView<std::array<float, kFftLengthBy2Plus1>>( |
| 309 | R2_heap_.data(), num_capture_channels_); |
Gustaf Ullberg | a63d152 | 2021-06-11 14:02:53 +0200 | [diff] [blame^] | 310 | R2_unbounded = rtc::ArrayView<std::array<float, kFftLengthBy2Plus1>>( |
| 311 | R2_unbounded_heap_.data(), num_capture_channels_); |
Per Åhgren | f6aa572 | 2019-09-10 18:05:17 +0200 | [diff] [blame] | 312 | S2_linear = rtc::ArrayView<std::array<float, kFftLengthBy2Plus1>>( |
| 313 | S2_linear_heap_.data(), num_capture_channels_); |
| 314 | Y = rtc::ArrayView<FftData>(Y_heap_.data(), num_capture_channels_); |
| 315 | E = rtc::ArrayView<FftData>(E_heap_.data(), num_capture_channels_); |
| 316 | comfort_noise = rtc::ArrayView<FftData>(comfort_noise_heap_.data(), |
| 317 | num_capture_channels_); |
| 318 | high_band_comfort_noise = rtc::ArrayView<FftData>( |
| 319 | high_band_comfort_noise_heap_.data(), num_capture_channels_); |
| 320 | subtractor_output = rtc::ArrayView<SubtractorOutput>( |
| 321 | subtractor_output_heap_.data(), num_capture_channels_); |
| 322 | } |
| 323 | |
Per Åhgren | 119e219 | 2019-10-18 08:50:50 +0200 | [diff] [blame] | 324 | data_dumper_->DumpWav("aec3_echo_remover_capture_input", kBlockSize, |
| 325 | &(*y)[0][0][0], 16000, 1); |
| 326 | data_dumper_->DumpWav("aec3_echo_remover_render_input", kBlockSize, |
| 327 | &x[0][0][0], 16000, 1); |
| 328 | data_dumper_->DumpRaw("aec3_echo_remover_capture_input", (*y)[0][0]); |
| 329 | data_dumper_->DumpRaw("aec3_echo_remover_render_input", x[0][0]); |
peah | 522d71b | 2017-02-23 05:16:26 -0800 | [diff] [blame] | 330 | |
| 331 | aec_state_.UpdateCaptureSaturation(capture_signal_saturation); |
| 332 | |
| 333 | if (echo_path_variability.AudioPathChanged()) { |
Per Åhgren | 88cf050 | 2018-07-16 17:08:41 +0200 | [diff] [blame] | 334 | // Ensure that the gain change is only acted on once per frame. |
| 335 | if (echo_path_variability.gain_change) { |
| 336 | if (gain_change_hangover_ == 0) { |
| 337 | constexpr int kMaxBlocksPerFrame = 3; |
| 338 | gain_change_hangover_ = kMaxBlocksPerFrame; |
Sam Zackrisson | ffc8452 | 2019-10-15 13:43:02 +0200 | [diff] [blame] | 339 | rtc::LoggingSeverity log_level = |
| 340 | config_.delay.log_warning_on_delay_changes ? rtc::LS_WARNING |
Sam Zackrisson | ecc5b93 | 2020-01-14 12:58:12 +0100 | [diff] [blame] | 341 | : rtc::LS_VERBOSE; |
Sam Zackrisson | ffc8452 | 2019-10-15 13:43:02 +0200 | [diff] [blame] | 342 | RTC_LOG_V(log_level) |
| 343 | << "Gain change detected at block " << block_counter_; |
Per Åhgren | 88cf050 | 2018-07-16 17:08:41 +0200 | [diff] [blame] | 344 | } else { |
| 345 | echo_path_variability.gain_change = false; |
| 346 | } |
| 347 | } |
| 348 | |
Per Åhgren | 7bdf073 | 2019-09-25 14:53:30 +0200 | [diff] [blame] | 349 | subtractor_.HandleEchoPathChange(echo_path_variability); |
peah | 86afe9d | 2017-04-06 15:45:32 -0700 | [diff] [blame] | 350 | aec_state_.HandleEchoPathChange(echo_path_variability); |
Per Åhgren | 88cf050 | 2018-07-16 17:08:41 +0200 | [diff] [blame] | 351 | |
| 352 | if (echo_path_variability.delay_change != |
| 353 | EchoPathVariability::DelayAdjustment::kNone) { |
Gustaf Ullberg | 5ea5749 | 2019-11-05 15:19:02 +0100 | [diff] [blame] | 354 | suppression_gain_.SetInitialState(true); |
Per Åhgren | 88cf050 | 2018-07-16 17:08:41 +0200 | [diff] [blame] | 355 | } |
| 356 | } |
| 357 | if (gain_change_hangover_ > 0) { |
| 358 | --gain_change_hangover_; |
peah | 522d71b | 2017-02-23 05:16:26 -0800 | [diff] [blame] | 359 | } |
| 360 | |
peah | 522d71b | 2017-02-23 05:16:26 -0800 | [diff] [blame] | 361 | // Analyze the render signal. |
Per Åhgren | 5c532d3 | 2018-03-22 00:29:25 +0100 | [diff] [blame] | 362 | render_signal_analyzer_.Update(*render_buffer, |
Per Åhgren | 8718afb | 2019-10-15 10:31:35 +0200 | [diff] [blame] | 363 | aec_state_.MinDirectPathFilterDelay()); |
peah | 522d71b | 2017-02-23 05:16:26 -0800 | [diff] [blame] | 364 | |
Per Åhgren | 7bdf073 | 2019-09-25 14:53:30 +0200 | [diff] [blame] | 365 | // State transition. |
Jesús de Vicente Peña | 02e9e44 | 2018-08-29 13:34:07 +0200 | [diff] [blame] | 366 | if (aec_state_.TransitionTriggered()) { |
Per Åhgren | 7bdf073 | 2019-09-25 14:53:30 +0200 | [diff] [blame] | 367 | subtractor_.ExitInitialState(); |
Gustaf Ullberg | 5ea5749 | 2019-11-05 15:19:02 +0100 | [diff] [blame] | 368 | suppression_gain_.SetInitialState(false); |
Per Åhgren | a98c807 | 2018-01-15 19:17:16 +0100 | [diff] [blame] | 369 | } |
Per Åhgren | 5c532d3 | 2018-03-22 00:29:25 +0100 | [diff] [blame] | 370 | |
Per Åhgren | 7bdf073 | 2019-09-25 14:53:30 +0200 | [diff] [blame] | 371 | // Perform linear echo cancellation. |
| 372 | subtractor_.Process(*render_buffer, (*y)[0], render_signal_analyzer_, |
| 373 | aec_state_, subtractor_output); |
| 374 | |
Per Åhgren | 119e219 | 2019-10-18 08:50:50 +0200 | [diff] [blame] | 375 | // Compute spectra. |
Gustaf Ullberg | a99b89b | 2019-09-23 16:03:12 +0200 | [diff] [blame] | 376 | for (size_t ch = 0; ch < num_capture_channels_; ++ch) { |
Per Åhgren | 0e3b1ff | 2019-09-25 12:09:37 +0200 | [diff] [blame] | 377 | FormLinearFilterOutput(subtractor_output[ch], e[ch]); |
Per Åhgren | 119e219 | 2019-10-18 08:50:50 +0200 | [diff] [blame] | 378 | WindowedPaddedFft(fft_, (*y)[0][ch], y_old_[ch], &Y[ch]); |
Per Åhgren | 0e3b1ff | 2019-09-25 12:09:37 +0200 | [diff] [blame] | 379 | WindowedPaddedFft(fft_, e[ch], e_old_[ch], &E[ch]); |
Gustaf Ullberg | a99b89b | 2019-09-23 16:03:12 +0200 | [diff] [blame] | 380 | LinearEchoPower(E[ch], Y[ch], &S2_linear[ch]); |
| 381 | Y[ch].Spectrum(optimization_, Y2[ch]); |
| 382 | E[ch].Spectrum(optimization_, E2[ch]); |
| 383 | } |
peah | 522d71b | 2017-02-23 05:16:26 -0800 | [diff] [blame] | 384 | |
Per Åhgren | c20a19c | 2019-11-13 11:12:29 +0100 | [diff] [blame] | 385 | // Optionally return the linear filter output. |
| 386 | if (linear_output) { |
| 387 | RTC_DCHECK_GE(1, linear_output->size()); |
| 388 | RTC_DCHECK_EQ(num_capture_channels_, linear_output[0].size()); |
| 389 | for (size_t ch = 0; ch < num_capture_channels_; ++ch) { |
| 390 | RTC_DCHECK_EQ(kBlockSize, (*linear_output)[0][ch].size()); |
| 391 | std::copy(e[ch].begin(), e[ch].end(), (*linear_output)[0][ch].begin()); |
| 392 | } |
| 393 | } |
| 394 | |
peah | 522d71b | 2017-02-23 05:16:26 -0800 | [diff] [blame] | 395 | // Update the AEC state information. |
Per Åhgren | 119e219 | 2019-10-18 08:50:50 +0200 | [diff] [blame] | 396 | aec_state_.Update(external_delay, subtractor_.FilterFrequencyResponses(), |
| 397 | subtractor_.FilterImpulseResponses(), *render_buffer, E2, |
| 398 | Y2, subtractor_output); |
Per Åhgren | 169c7fd | 2018-04-27 12:04:03 +0200 | [diff] [blame] | 399 | |
peah | 522d71b | 2017-02-23 05:16:26 -0800 | [diff] [blame] | 400 | // Choose the linear output. |
Gustaf Ullberg | af3fdc0 | 2019-09-24 15:05:04 +0200 | [diff] [blame] | 401 | const auto& Y_fft = aec_state_.UseLinearFilterOutput() ? E : Y; |
Gustaf Ullberg | a99b89b | 2019-09-23 16:03:12 +0200 | [diff] [blame] | 402 | |
Per Åhgren | 119e219 | 2019-10-18 08:50:50 +0200 | [diff] [blame] | 403 | data_dumper_->DumpWav("aec3_output_linear", kBlockSize, &(*y)[0][0][0], 16000, |
| 404 | 1); |
Per Åhgren | 0e3b1ff | 2019-09-25 12:09:37 +0200 | [diff] [blame] | 405 | data_dumper_->DumpWav("aec3_output_linear2", kBlockSize, &e[0][0], 16000, 1); |
peah | 522d71b | 2017-02-23 05:16:26 -0800 | [diff] [blame] | 406 | |
Gustaf Ullberg | caaa9e7 | 2019-10-31 14:10:24 +0100 | [diff] [blame] | 407 | // Estimate the comfort noise. |
| 408 | cng_.Compute(aec_state_.SaturatedCapture(), Y2, comfort_noise, |
| 409 | high_band_comfort_noise); |
Jesús de Vicente Peña | 0faf082 | 2018-09-24 12:48:28 +0200 | [diff] [blame] | 410 | |
Per Åhgren | 8ee1ec8 | 2021-03-11 06:33:45 +0000 | [diff] [blame] | 411 | // Only do the below processing if the output of the audio processing module |
| 412 | // is used. |
Gustaf Ullberg | 5ea5749 | 2019-11-05 15:19:02 +0100 | [diff] [blame] | 413 | std::array<float, kFftLengthBy2Plus1> G; |
Per Åhgren | 8ee1ec8 | 2021-03-11 06:33:45 +0000 | [diff] [blame] | 414 | if (capture_output_used_) { |
| 415 | // Estimate the residual echo power. |
| 416 | residual_echo_estimator_.Estimate(aec_state_, *render_buffer, S2_linear, Y2, |
Gustaf Ullberg | a63d152 | 2021-06-11 14:02:53 +0200 | [diff] [blame^] | 417 | suppression_gain_.IsDominantNearend(), R2, |
| 418 | R2_unbounded); |
Jesús de Vicente Peña | 0faf082 | 2018-09-24 12:48:28 +0200 | [diff] [blame] | 419 | |
Per Åhgren | 8ee1ec8 | 2021-03-11 06:33:45 +0000 | [diff] [blame] | 420 | // Suppressor nearend estimate. |
| 421 | if (aec_state_.UsableLinearEstimate()) { |
| 422 | // E2 is bound by Y2. |
| 423 | for (size_t ch = 0; ch < num_capture_channels_; ++ch) { |
| 424 | std::transform(E2[ch].begin(), E2[ch].end(), Y2[ch].begin(), |
| 425 | E2[ch].begin(), |
| 426 | [](float a, float b) { return std::min(a, b); }); |
| 427 | } |
| 428 | } |
| 429 | const auto& nearend_spectrum = aec_state_.UsableLinearEstimate() ? E2 : Y2; |
| 430 | |
| 431 | // Suppressor echo estimate. |
| 432 | const auto& echo_spectrum = |
| 433 | aec_state_.UsableLinearEstimate() ? S2_linear : R2; |
| 434 | |
| 435 | // Determine if the suppressor should assume clock drift. |
| 436 | const bool clock_drift = config_.echo_removal_control.has_clock_drift || |
| 437 | echo_path_variability.clock_drift; |
| 438 | |
| 439 | // Compute preferred gains. |
| 440 | float high_bands_gain; |
Gustaf Ullberg | a63d152 | 2021-06-11 14:02:53 +0200 | [diff] [blame^] | 441 | suppression_gain_.GetGain(nearend_spectrum, echo_spectrum, R2, R2_unbounded, |
Per Åhgren | 8ee1ec8 | 2021-03-11 06:33:45 +0000 | [diff] [blame] | 442 | cng_.NoiseSpectrum(), render_signal_analyzer_, |
| 443 | aec_state_, x, clock_drift, &high_bands_gain, &G); |
| 444 | |
| 445 | suppression_filter_.ApplyGain(comfort_noise, high_band_comfort_noise, G, |
| 446 | high_bands_gain, Y_fft, y); |
| 447 | |
| 448 | } else { |
| 449 | G.fill(0.f); |
| 450 | } |
peah | 522d71b | 2017-02-23 05:16:26 -0800 | [diff] [blame] | 451 | |
peah | e985b3f | 2017-02-28 22:08:53 -0800 | [diff] [blame] | 452 | // Update the metrics. |
Gustaf Ullberg | caaa9e7 | 2019-10-31 14:10:24 +0100 | [diff] [blame] | 453 | metrics_.Update(aec_state_, cng_.NoiseSpectrum()[0], G); |
peah | e985b3f | 2017-02-28 22:08:53 -0800 | [diff] [blame] | 454 | |
peah | 522d71b | 2017-02-23 05:16:26 -0800 | [diff] [blame] | 455 | // Debug outputs for the purpose of development and analysis. |
peah | 2910357 | 2017-07-11 02:54:02 -0700 | [diff] [blame] | 456 | data_dumper_->DumpWav("aec3_echo_estimate", kBlockSize, |
Per Åhgren | ff04511 | 2020-03-20 11:20:39 +0100 | [diff] [blame] | 457 | &subtractor_output[0].s_refined[0], 16000, 1); |
Per Åhgren | 119e219 | 2019-10-18 08:50:50 +0200 | [diff] [blame] | 458 | data_dumper_->DumpRaw("aec3_output", (*y)[0][0]); |
peah | 14c11a4 | 2017-07-11 06:13:43 -0700 | [diff] [blame] | 459 | data_dumper_->DumpRaw("aec3_narrow_render", |
| 460 | render_signal_analyzer_.NarrowPeakBand() ? 1 : 0); |
Gustaf Ullberg | caaa9e7 | 2019-10-31 14:10:24 +0100 | [diff] [blame] | 461 | data_dumper_->DumpRaw("aec3_N2", cng_.NoiseSpectrum()[0]); |
peah | 522d71b | 2017-02-23 05:16:26 -0800 | [diff] [blame] | 462 | data_dumper_->DumpRaw("aec3_suppressor_gain", G); |
Per Åhgren | 119e219 | 2019-10-18 08:50:50 +0200 | [diff] [blame] | 463 | data_dumper_->DumpWav("aec3_output", |
| 464 | rtc::ArrayView<const float>(&(*y)[0][0][0], kBlockSize), |
| 465 | 16000, 1); |
Per Åhgren | f6aa572 | 2019-09-10 18:05:17 +0200 | [diff] [blame] | 466 | data_dumper_->DumpRaw("aec3_using_subtractor_output[0]", |
Per Åhgren | 5c532d3 | 2018-03-22 00:29:25 +0100 | [diff] [blame] | 467 | aec_state_.UseLinearFilterOutput() ? 1 : 0); |
Per Åhgren | f6aa572 | 2019-09-10 18:05:17 +0200 | [diff] [blame] | 468 | data_dumper_->DumpRaw("aec3_E2", E2[0]); |
| 469 | data_dumper_->DumpRaw("aec3_S2_linear", S2_linear[0]); |
| 470 | data_dumper_->DumpRaw("aec3_Y2", Y2[0]); |
Jesús de Vicente Peña | 7682c6e | 2018-03-22 14:53:23 +0100 | [diff] [blame] | 471 | data_dumper_->DumpRaw( |
Sam Zackrisson | 98872dc | 2019-10-18 08:20:09 +0200 | [diff] [blame] | 472 | "aec3_X2", render_buffer->Spectrum( |
| 473 | aec_state_.MinDirectPathFilterDelay())[/*channel=*/0]); |
Per Åhgren | f6aa572 | 2019-09-10 18:05:17 +0200 | [diff] [blame] | 474 | data_dumper_->DumpRaw("aec3_R2", R2[0]); |
Per Åhgren | 8718afb | 2019-10-15 10:31:35 +0200 | [diff] [blame] | 475 | data_dumper_->DumpRaw("aec3_filter_delay", |
| 476 | aec_state_.MinDirectPathFilterDelay()); |
peah | 522d71b | 2017-02-23 05:16:26 -0800 | [diff] [blame] | 477 | data_dumper_->DumpRaw("aec3_capture_saturation", |
| 478 | aec_state_.SaturatedCapture() ? 1 : 0); |
| 479 | } |
peah | 69221db | 2017-01-27 03:28:19 -0800 | [diff] [blame] | 480 | |
Per Åhgren | 2275439 | 2018-08-10 18:37:38 +0200 | [diff] [blame] | 481 | void EchoRemoverImpl::FormLinearFilterOutput( |
Per Åhgren | 2275439 | 2018-08-10 18:37:38 +0200 | [diff] [blame] | 482 | const SubtractorOutput& subtractor_output, |
| 483 | rtc::ArrayView<float> output) { |
Per Åhgren | ff04511 | 2020-03-20 11:20:39 +0100 | [diff] [blame] | 484 | RTC_DCHECK_EQ(subtractor_output.e_refined.size(), output.size()); |
Per Åhgren | 9d66198 | 2020-03-20 11:26:48 +0100 | [diff] [blame] | 485 | RTC_DCHECK_EQ(subtractor_output.e_coarse.size(), output.size()); |
Per Åhgren | ff04511 | 2020-03-20 11:20:39 +0100 | [diff] [blame] | 486 | bool use_refined_output = true; |
Per Åhgren | 9d66198 | 2020-03-20 11:26:48 +0100 | [diff] [blame] | 487 | if (use_coarse_filter_output_) { |
Per Åhgren | ff04511 | 2020-03-20 11:20:39 +0100 | [diff] [blame] | 488 | // As the output of the refined adaptive filter generally should be better |
Per Åhgren | 9d66198 | 2020-03-20 11:26:48 +0100 | [diff] [blame] | 489 | // than the coarse filter output, add a margin and threshold for when |
| 490 | // choosing the coarse filter output. |
| 491 | if (subtractor_output.e2_coarse < 0.9f * subtractor_output.e2_refined && |
Per Åhgren | 2275439 | 2018-08-10 18:37:38 +0200 | [diff] [blame] | 492 | subtractor_output.y2 > 30.f * 30.f * kBlockSize && |
Per Åhgren | ff04511 | 2020-03-20 11:20:39 +0100 | [diff] [blame] | 493 | (subtractor_output.s2_refined > 60.f * 60.f * kBlockSize || |
Per Åhgren | 9d66198 | 2020-03-20 11:26:48 +0100 | [diff] [blame] | 494 | subtractor_output.s2_coarse > 60.f * 60.f * kBlockSize)) { |
Per Åhgren | ff04511 | 2020-03-20 11:20:39 +0100 | [diff] [blame] | 495 | use_refined_output = false; |
Per Åhgren | 2275439 | 2018-08-10 18:37:38 +0200 | [diff] [blame] | 496 | } else { |
Per Åhgren | ff04511 | 2020-03-20 11:20:39 +0100 | [diff] [blame] | 497 | // If the refined filter is diverged, choose the filter output that has |
| 498 | // the lowest power. |
Per Åhgren | 9d66198 | 2020-03-20 11:26:48 +0100 | [diff] [blame] | 499 | if (subtractor_output.e2_coarse < subtractor_output.e2_refined && |
Per Åhgren | ff04511 | 2020-03-20 11:20:39 +0100 | [diff] [blame] | 500 | subtractor_output.y2 < subtractor_output.e2_refined) { |
| 501 | use_refined_output = false; |
Per Åhgren | 2275439 | 2018-08-10 18:37:38 +0200 | [diff] [blame] | 502 | } |
| 503 | } |
| 504 | } |
| 505 | |
Per Åhgren | ff04511 | 2020-03-20 11:20:39 +0100 | [diff] [blame] | 506 | SignalTransition(refined_filter_output_last_selected_ |
| 507 | ? subtractor_output.e_refined |
Per Åhgren | 9d66198 | 2020-03-20 11:26:48 +0100 | [diff] [blame] | 508 | : subtractor_output.e_coarse, |
Per Åhgren | ff04511 | 2020-03-20 11:20:39 +0100 | [diff] [blame] | 509 | use_refined_output ? subtractor_output.e_refined |
Per Åhgren | 9d66198 | 2020-03-20 11:26:48 +0100 | [diff] [blame] | 510 | : subtractor_output.e_coarse, |
Per Åhgren | ff04511 | 2020-03-20 11:20:39 +0100 | [diff] [blame] | 511 | output); |
| 512 | refined_filter_output_last_selected_ = use_refined_output; |
Per Åhgren | 2275439 | 2018-08-10 18:37:38 +0200 | [diff] [blame] | 513 | } |
| 514 | |
peah | 69221db | 2017-01-27 03:28:19 -0800 | [diff] [blame] | 515 | } // namespace |
| 516 | |
Gustaf Ullberg | bd83b91 | 2017-10-18 12:32:42 +0200 | [diff] [blame] | 517 | EchoRemover* EchoRemover::Create(const EchoCanceller3Config& config, |
Per Åhgren | ce202a0 | 2019-09-02 17:01:19 +0200 | [diff] [blame] | 518 | int sample_rate_hz, |
| 519 | size_t num_render_channels, |
| 520 | size_t num_capture_channels) { |
| 521 | return new EchoRemoverImpl(config, sample_rate_hz, num_render_channels, |
| 522 | num_capture_channels); |
peah | 69221db | 2017-01-27 03:28:19 -0800 | [diff] [blame] | 523 | } |
| 524 | |
| 525 | } // namespace webrtc |