blob: 2a564fc233d93e7ebe30b75cf4c932dfd8c1f66f [file] [log] [blame]
ivoc9f4a4a02016-10-28 05:39:16 -07001/*
2 * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020011#include "modules/audio_processing/residual_echo_detector.h"
ivoc9f4a4a02016-10-28 05:39:16 -070012
ivocaf27ed02016-10-28 07:04:03 -070013#include <algorithm>
14#include <numeric>
15
Yves Gerey988cc082018-10-23 12:03:01 +020016#include "absl/types/optional.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020017#include "modules/audio_processing/logging/apm_data_dumper.h"
Yves Gerey988cc082018-10-23 12:03:01 +020018#include "rtc_base/checks.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020019#include "rtc_base/logging.h"
20#include "system_wrappers/include/metrics.h"
ivoc9f4a4a02016-10-28 05:39:16 -070021
ivocaf27ed02016-10-28 07:04:03 -070022namespace {
23
24float Power(rtc::ArrayView<const float> input) {
Alex Loiko890988c2017-08-31 10:25:48 +020025 if (input.empty()) {
peah9e6a2902017-05-15 07:19:21 -070026 return 0.f;
27 }
28 return std::inner_product(input.begin(), input.end(), input.begin(), 0.f) /
29 input.size();
ivocaf27ed02016-10-28 07:04:03 -070030}
31
32constexpr size_t kLookbackFrames = 650;
33// TODO(ivoc): Verify the size of this buffer.
34constexpr size_t kRenderBufferSize = 30;
ivocfbb374d2016-11-17 06:19:47 -080035constexpr float kAlpha = 0.001f;
ivoc4e477a12017-01-15 08:29:46 -080036// 10 seconds of data, updated every 10 ms.
37constexpr size_t kAggregationBufferSize = 10 * 100;
ivocaf27ed02016-10-28 07:04:03 -070038
39} // namespace
40
ivoc9f4a4a02016-10-28 05:39:16 -070041namespace webrtc {
42
Niels Möller7a669002022-06-27 09:47:02 +020043std::atomic<int> ResidualEchoDetector::instance_count_(0);
peah9e6a2902017-05-15 07:19:21 -070044
ivocaf27ed02016-10-28 07:04:03 -070045ResidualEchoDetector::ResidualEchoDetector()
Niels Möller7a669002022-06-27 09:47:02 +020046 : data_dumper_(new ApmDataDumper(instance_count_.fetch_add(1) + 1)),
peah9e6a2902017-05-15 07:19:21 -070047 render_buffer_(kRenderBufferSize),
ivocaf27ed02016-10-28 07:04:03 -070048 render_power_(kLookbackFrames),
49 render_power_mean_(kLookbackFrames),
50 render_power_std_dev_(kLookbackFrames),
ivoc4e477a12017-01-15 08:29:46 -080051 covariances_(kLookbackFrames),
52 recent_likelihood_max_(kAggregationBufferSize) {}
ivoc9f4a4a02016-10-28 05:39:16 -070053
54ResidualEchoDetector::~ResidualEchoDetector() = default;
55
56void ResidualEchoDetector::AnalyzeRenderAudio(
ivocaf27ed02016-10-28 07:04:03 -070057 rtc::ArrayView<const float> render_audio) {
peah9e6a2902017-05-15 07:19:21 -070058 // Dump debug data assuming 48 kHz sample rate (if this assumption is not
59 // valid the dumped audio will need to be converted offline accordingly).
60 data_dumper_->DumpWav("ed_render", render_audio.size(), render_audio.data(),
61 48000, 1);
62
ivocaf27ed02016-10-28 07:04:03 -070063 if (render_buffer_.Size() == 0) {
64 frames_since_zero_buffer_size_ = 0;
65 } else if (frames_since_zero_buffer_size_ >= kRenderBufferSize) {
66 // This can happen in a few cases: at the start of a call, due to a glitch
67 // or due to clock drift. The excess capture value will be ignored.
68 // TODO(ivoc): Include how often this happens in APM stats.
69 render_buffer_.Pop();
70 frames_since_zero_buffer_size_ = 0;
71 }
72 ++frames_since_zero_buffer_size_;
73 float power = Power(render_audio);
74 render_buffer_.Push(power);
ivoc9f4a4a02016-10-28 05:39:16 -070075}
76
77void ResidualEchoDetector::AnalyzeCaptureAudio(
ivocaf27ed02016-10-28 07:04:03 -070078 rtc::ArrayView<const float> capture_audio) {
peah9e6a2902017-05-15 07:19:21 -070079 // Dump debug data assuming 48 kHz sample rate (if this assumption is not
80 // valid the dumped audio will need to be converted offline accordingly).
81 data_dumper_->DumpWav("ed_capture", capture_audio.size(),
82 capture_audio.data(), 48000, 1);
83
ivocaf27ed02016-10-28 07:04:03 -070084 if (first_process_call_) {
85 // On the first process call (so the start of a call), we must flush the
86 // render buffer, otherwise the render data will be delayed.
87 render_buffer_.Clear();
88 first_process_call_ = false;
89 }
90
91 // Get the next render value.
Danil Chapovalovdb9f7ab2018-06-19 10:50:11 +020092 const absl::optional<float> buffered_render_power = render_buffer_.Pop();
ivocaf27ed02016-10-28 07:04:03 -070093 if (!buffered_render_power) {
94 // This can happen in a few cases: at the start of a call, due to a glitch
95 // or due to clock drift. The excess capture value will be ignored.
96 // TODO(ivoc): Include how often this happens in APM stats.
97 return;
98 }
99 // Update the render statistics, and store the statistics in circular buffers.
100 render_statistics_.Update(*buffered_render_power);
101 RTC_DCHECK_LT(next_insertion_index_, kLookbackFrames);
102 render_power_[next_insertion_index_] = *buffered_render_power;
103 render_power_mean_[next_insertion_index_] = render_statistics_.mean();
104 render_power_std_dev_[next_insertion_index_] =
105 render_statistics_.std_deviation();
106
107 // Get the next capture value, update capture statistics and add the relevant
108 // values to the buffers.
109 const float capture_power = Power(capture_audio);
110 capture_statistics_.Update(capture_power);
111 const float capture_mean = capture_statistics_.mean();
112 const float capture_std_deviation = capture_statistics_.std_deviation();
113
114 // Update the covariance values and determine the new echo likelihood.
115 echo_likelihood_ = 0.f;
peah94f6fa02017-05-16 07:25:06 -0700116 size_t read_index = next_insertion_index_;
ivoc1592c742017-05-17 09:53:02 -0700117
118 int best_delay = -1;
ivocaf27ed02016-10-28 07:04:03 -0700119 for (size_t delay = 0; delay < covariances_.size(); ++delay) {
ivocaf27ed02016-10-28 07:04:03 -0700120 RTC_DCHECK_LT(read_index, render_power_.size());
121 covariances_[delay].Update(capture_power, capture_mean,
122 capture_std_deviation, render_power_[read_index],
123 render_power_mean_[read_index],
124 render_power_std_dev_[read_index]);
peah94f6fa02017-05-16 07:25:06 -0700125 read_index = read_index > 0 ? read_index - 1 : kLookbackFrames - 1;
ivoc1592c742017-05-17 09:53:02 -0700126
127 if (covariances_[delay].normalized_cross_correlation() > echo_likelihood_) {
128 echo_likelihood_ = covariances_[delay].normalized_cross_correlation();
129 best_delay = static_cast<int>(delay);
130 }
131 }
132 // This is a temporary log message to help find the underlying cause for echo
133 // likelihoods > 1.0.
134 // TODO(ivoc): Remove once the issue is resolved.
135 if (echo_likelihood_ > 1.1f) {
136 // Make sure we don't spam the log.
137 if (log_counter_ < 5 && best_delay != -1) {
138 size_t read_index = kLookbackFrames + next_insertion_index_ - best_delay;
139 if (read_index >= kLookbackFrames) {
140 read_index -= kLookbackFrames;
141 }
142 RTC_DCHECK_LT(read_index, render_power_.size());
Yves Gerey665174f2018-06-19 15:03:05 +0200143 RTC_LOG_F(LS_ERROR) << "Echo detector internal state: {"
144 "Echo likelihood: "
145 << echo_likelihood_ << ", Best Delay: " << best_delay
146 << ", Covariance: "
147 << covariances_[best_delay].covariance()
148 << ", Last capture power: " << capture_power
149 << ", Capture mean: " << capture_mean
150 << ", Capture_standard deviation: "
151 << capture_std_deviation << ", Last render power: "
152 << render_power_[read_index]
153 << ", Render mean: " << render_power_mean_[read_index]
154 << ", Render standard deviation: "
155 << render_power_std_dev_[read_index]
156 << ", Reliability: " << reliability_ << "}";
ivoc1592c742017-05-17 09:53:02 -0700157 log_counter_++;
158 }
ivocaf27ed02016-10-28 07:04:03 -0700159 }
ivoc860249e2017-05-16 06:50:11 -0700160 RTC_DCHECK_LT(echo_likelihood_, 1.1f);
ivoc1592c742017-05-17 09:53:02 -0700161
ivocfbb374d2016-11-17 06:19:47 -0800162 reliability_ = (1.0f - kAlpha) * reliability_ + kAlpha * 1.0f;
163 echo_likelihood_ *= reliability_;
ivoc8f94cd32017-05-05 05:50:10 -0700164 // This is a temporary fix to prevent echo likelihood values > 1.0.
165 // TODO(ivoc): Find the root cause of this issue and fix it.
166 echo_likelihood_ = std::min(echo_likelihood_, 1.0f);
ivocef6cbae2016-11-10 08:21:04 -0800167 int echo_percentage = static_cast<int>(echo_likelihood_ * 100);
168 RTC_HISTOGRAM_COUNTS("WebRTC.Audio.ResidualEchoDetector.EchoLikelihood",
169 echo_percentage, 0, 100, 100 /* number of bins */);
ivocaf27ed02016-10-28 07:04:03 -0700170
ivoc4e477a12017-01-15 08:29:46 -0800171 // Update the buffer of recent likelihood values.
172 recent_likelihood_max_.Update(echo_likelihood_);
173
ivocaf27ed02016-10-28 07:04:03 -0700174 // Update the next insertion index.
peah94f6fa02017-05-16 07:25:06 -0700175 next_insertion_index_ = next_insertion_index_ < (kLookbackFrames - 1)
176 ? next_insertion_index_ + 1
177 : 0;
ivoc9f4a4a02016-10-28 05:39:16 -0700178}
179
Ivo Creusen647ef092018-03-14 17:13:48 +0100180void ResidualEchoDetector::Initialize(int /*capture_sample_rate_hz*/,
181 int /*num_capture_channels*/,
182 int /*render_sample_rate_hz*/,
183 int /*num_render_channels*/) {
ivocaf27ed02016-10-28 07:04:03 -0700184 render_buffer_.Clear();
185 std::fill(render_power_.begin(), render_power_.end(), 0.f);
186 std::fill(render_power_mean_.begin(), render_power_mean_.end(), 0.f);
187 std::fill(render_power_std_dev_.begin(), render_power_std_dev_.end(), 0.f);
188 render_statistics_.Clear();
189 capture_statistics_.Clear();
ivoc4e477a12017-01-15 08:29:46 -0800190 recent_likelihood_max_.Clear();
ivocaf27ed02016-10-28 07:04:03 -0700191 for (auto& cov : covariances_) {
192 cov.Clear();
193 }
194 echo_likelihood_ = 0.f;
195 next_insertion_index_ = 0;
ivocfbb374d2016-11-17 06:19:47 -0800196 reliability_ = 0.f;
ivoc9f4a4a02016-10-28 05:39:16 -0700197}
198
Ivo Creusen09fa4b02018-01-11 16:08:54 +0100199EchoDetector::Metrics ResidualEchoDetector::GetMetrics() const {
200 EchoDetector::Metrics metrics;
201 metrics.echo_likelihood = echo_likelihood_;
202 metrics.echo_likelihood_recent_max = recent_likelihood_max_.max();
203 return metrics;
204}
ivoc9f4a4a02016-10-28 05:39:16 -0700205} // namespace webrtc