modules/audio_processing/aec3/aec_state.cc

/*
 *  Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
 *
 *  Use of this source code is governed by a BSD-style license
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
 *  in the file PATENTS.  All contributing project authors may
 *  be found in the AUTHORS file in the root of the source tree.
 */

#include "modules/audio_processing/aec3/aec_state.h"

#include <math.h>
#include <algorithm>
#include <numeric>
#include <vector>

#include "absl/types/optional.h"
#include "api/array_view.h"
#include "modules/audio_processing/aec3/aec3_common.h"
#include "modules/audio_processing/logging/apm_data_dumper.h"
#include "rtc_base/atomic_ops.h"
#include "rtc_base/checks.h"

namespace webrtc {
namespace {

constexpr size_t kBlocksSinceConvergencedFilterInit = 10000;
constexpr size_t kBlocksSinceConsistentEstimateInit = 10000;

}  // namespace

int AecState::instance_count_ = 0;

void AecState::GetResidualEchoScaling(
    rtc::ArrayView<float> residual_scaling) const {
  bool filter_has_had_time_to_converge;
  if (config_.filter.conservative_initial_phase) {
    filter_has_had_time_to_converge =
        strong_not_saturated_render_blocks_ >= 1.5f * kNumBlocksPerSecond;
  } else {
    filter_has_had_time_to_converge =
        strong_not_saturated_render_blocks_ >= 0.8f * kNumBlocksPerSecond;
  }
  echo_audibility_.GetResidualEchoScaling(filter_has_had_time_to_converge,
                                          residual_scaling);
}

absl::optional<float> AecState::ErleUncertainty() const {
  if (SaturatedEcho()) {
    return 1.f;
  }

  return absl::nullopt;
}

AecState::AecState(const EchoCanceller3Config& config)
    : data_dumper_(
          new ApmDataDumper(rtc::AtomicOps::Increment(&instance_count_))),
      config_(config),
      initial_state_(config_),
      delay_state_(config_),
      transparent_state_(config_),
      filter_quality_state_(config_),
      legacy_filter_quality_state_(config_),
      legacy_saturation_detector_(config_),
      erl_estimator_(2 * kNumBlocksPerSecond),
      erle_estimator_(2 * kNumBlocksPerSecond, config_),
      filter_analyzer_(config_),
      echo_audibility_(
          config_.echo_audibility.use_stationarity_properties_at_init),
      reverb_model_estimator_(config_) {}

AecState::~AecState() = default;

void AecState::HandleEchoPathChange(
    const EchoPathVariability& echo_path_variability) {
  const auto full_reset = [&]() {
    filter_analyzer_.Reset();
    capture_signal_saturation_ = false;
    strong_not_saturated_render_blocks_ = 0;
    blocks_with_active_render_ = 0;
    initial_state_.Reset();
    transparent_state_.Reset();
      legacy_saturation_detector_.Reset();
    erle_estimator_.Reset(true);
    erl_estimator_.Reset();
      filter_quality_state_.Reset();
  };

  // TODO(peah): Refine the reset scheme according to the type of gain and
  // delay adjustment.

  if (echo_path_variability.delay_change !=
      EchoPathVariability::DelayAdjustment::kNone) {
    full_reset();
  } else if (echo_path_variability.gain_change) {
    erle_estimator_.Reset(false);
  }
  subtractor_output_analyzer_.HandleEchoPathChange();
}

void AecState::Update(
    const absl::optional<DelayEstimate>& external_delay,
    const std::vector<std::array<float, kFftLengthBy2Plus1>>&
        adaptive_filter_frequency_response,
    const std::vector<float>& adaptive_filter_impulse_response,
    const RenderBuffer& render_buffer,
    const std::array<float, kFftLengthBy2Plus1>& E2_main,
    const std::array<float, kFftLengthBy2Plus1>& Y2,
    const SubtractorOutput& subtractor_output,
    rtc::ArrayView<const float> y) {
  // Analyze the filter output.
  subtractor_output_analyzer_.Update(subtractor_output);

  // Analyze the properties of the filter.
  filter_analyzer_.Update(adaptive_filter_impulse_response, render_buffer);

  // Estimate the direct path delay of the filter.
  delay_state_.Update(filter_analyzer_, external_delay,
                      strong_not_saturated_render_blocks_);

  const std::vector<float>& aligned_render_block =
      render_buffer.Block(-delay_state_.DirectPathFilterDelay())[0];

  // Update render counters.
  const float render_energy = std::inner_product(
      aligned_render_block.begin(), aligned_render_block.end(),
      aligned_render_block.begin(), 0.f);
  const bool active_render =
      render_energy > (config_.render_levels.active_render_limit *
                       config_.render_levels.active_render_limit) *
                          kFftLengthBy2;
  blocks_with_active_render_ += active_render ? 1 : 0;
  strong_not_saturated_render_blocks_ +=
      active_render && !SaturatedCapture() ? 1 : 0;

  std::array<float, kFftLengthBy2Plus1> X2_reverb;
  render_reverb_.Apply(
      render_buffer.GetSpectrumBuffer(), delay_state_.DirectPathFilterDelay(),
      config_.ep_strength.reverb_based_on_render ? ReverbDecay() : 0.f,
      X2_reverb);

  if (config_.echo_audibility.use_stationary_properties) {
    // Update the echo audibility evaluator.
    echo_audibility_.Update(render_buffer,
                            render_reverb_.GetReverbContributionPowerSpectrum(),
                            delay_state_.DirectPathFilterDelay(),
                            delay_state_.ExternalDelayReported());
  }

  // Update the ERL and ERLE measures.
  if (initial_state_.TransitionTriggered()) {
    erle_estimator_.Reset(false);
  }

  const auto& X2 = render_buffer.Spectrum(delay_state_.DirectPathFilterDelay());
  const auto& X2_input_erle = X2_reverb;

  erle_estimator_.Update(render_buffer, adaptive_filter_frequency_response,
                         X2_input_erle, Y2, E2_main,
                         subtractor_output_analyzer_.ConvergedFilter(),
                         config_.erle.onset_detection);

  erl_estimator_.Update(subtractor_output_analyzer_.ConvergedFilter(), X2, Y2);

  // Detect and flag echo saturation.
  saturation_detector_.Update(aligned_render_block, SaturatedCapture(),
                              UsableLinearEstimate(), subtractor_output,
                              EchoPathGain());

  // Update the decision on whether to use the initial state parameter set.
  initial_state_.Update(active_render, SaturatedCapture());

  // Detect whether the transparent mode should be activated.
  transparent_state_.Update(delay_state_.DirectPathFilterDelay(),
                            filter_analyzer_.Consistent(),
                            subtractor_output_analyzer_.ConvergedFilter(),
                            subtractor_output_analyzer_.DivergedFilter(),
                            active_render, SaturatedCapture());

  // Analyze the quality of the filter.
  filter_quality_state_.Update(active_render, TransparentMode(),
                               SaturatedCapture(),
                               filter_analyzer_.Consistent(), external_delay,
                               subtractor_output_analyzer_.ConvergedFilter());

  // Update the reverb estimate.
  const bool stationary_block =
      config_.echo_audibility.use_stationary_properties &&
      echo_audibility_.IsBlockStationary();

  reverb_model_estimator_.Update(filter_analyzer_.GetAdjustedFilter(),
                                 adaptive_filter_frequency_response,
                                 erle_estimator_.GetInstLinearQualityEstimate(),
                                 delay_state_.DirectPathFilterDelay(),
                                 UsableLinearEstimate(), stationary_block);

  erle_estimator_.Dump(data_dumper_);
  reverb_model_estimator_.Dump(data_dumper_.get());
  data_dumper_->DumpRaw("aec3_erl", Erl());
  data_dumper_->DumpRaw("aec3_erl_time_domain", ErlTimeDomain());
  data_dumper_->DumpRaw("aec3_erle", Erle());
  data_dumper_->DumpRaw("aec3_usable_linear_estimate", UsableLinearEstimate());
  data_dumper_->DumpRaw("aec3_transparent_mode", TransparentMode());
  data_dumper_->DumpRaw("aec3_filter_delay", filter_analyzer_.DelayBlocks());

  data_dumper_->DumpRaw("aec3_consistent_filter",
                        filter_analyzer_.Consistent());
  data_dumper_->DumpRaw("aec3_initial_state",
                        initial_state_.InitialStateActive());
  data_dumper_->DumpRaw("aec3_capture_saturation", SaturatedCapture());
  data_dumper_->DumpRaw("aec3_echo_saturation", SaturatedEcho());
  data_dumper_->DumpRaw("aec3_converged_filter",
                        subtractor_output_analyzer_.ConvergedFilter());
  data_dumper_->DumpRaw("aec3_diverged_filter",
                        subtractor_output_analyzer_.DivergedFilter());

  data_dumper_->DumpRaw("aec3_external_delay_avaliable",
                        external_delay ? 1 : 0);
  data_dumper_->DumpRaw("aec3_filter_tail_freq_resp_est",
                        GetReverbFrequencyResponse());
}

AecState::InitialState::InitialState(const EchoCanceller3Config& config)
    : conservative_initial_phase_(config.filter.conservative_initial_phase),
      initial_state_seconds_(config.filter.initial_state_seconds) {
  Reset();
}
void AecState::InitialState::InitialState::Reset() {
  initial_state_ = true;
  strong_not_saturated_render_blocks_ = 0;
}
void AecState::InitialState::InitialState::Update(bool active_render,
                                                  bool saturated_capture) {
  strong_not_saturated_render_blocks_ +=
      active_render && !saturated_capture ? 1 : 0;

  // Flag whether the initial state is still active.
  bool prev_initial_state = initial_state_;
  if (conservative_initial_phase_) {
    initial_state_ =
        strong_not_saturated_render_blocks_ < 5 * kNumBlocksPerSecond;
  } else {
    initial_state_ = strong_not_saturated_render_blocks_ <
                     initial_state_seconds_ * kNumBlocksPerSecond;
  }

  // Flag whether the transition from the initial state has started.
  transition_triggered_ = !initial_state_ && prev_initial_state;
}

AecState::FilterDelay::FilterDelay(const EchoCanceller3Config& config)
    : delay_headroom_blocks_(config.delay.delay_headroom_blocks) {}

void AecState::FilterDelay::Update(
    const FilterAnalyzer& filter_analyzer,
    const absl::optional<DelayEstimate>& external_delay,
    size_t blocks_with_proper_filter_adaptation) {
  // Update the delay based on the external delay.
  if (external_delay &&
      (!external_delay_ || external_delay_->delay != external_delay->delay)) {
    external_delay_ = external_delay;
    external_delay_reported_ = true;
  }

  // Override the estimated delay if it is not certain that the filter has had
  // time to converge.
  const bool delay_estimator_may_not_have_converged =
      blocks_with_proper_filter_adaptation < 2 * kNumBlocksPerSecond;
  if (delay_estimator_may_not_have_converged && external_delay_) {
    filter_delay_blocks_ = delay_headroom_blocks_;
  } else {
    filter_delay_blocks_ = filter_analyzer.DelayBlocks();
  }
}

AecState::TransparentMode::TransparentMode(const EchoCanceller3Config& config)
    : bounded_erl_(config.ep_strength.bounded_erl),
      linear_and_stable_echo_path_(
          config.echo_removal_control.linear_and_stable_echo_path),
      active_blocks_since_sane_filter_(kBlocksSinceConsistentEstimateInit),
      non_converged_sequence_size_(kBlocksSinceConvergencedFilterInit) {}

void AecState::TransparentMode::Reset() {
  non_converged_sequence_size_ = kBlocksSinceConvergencedFilterInit;
  diverged_sequence_size_ = 0;
  strong_not_saturated_render_blocks_ = 0;
  if (linear_and_stable_echo_path_) {
    recent_convergence_during_activity_ = false;
  }
}

void AecState::TransparentMode::Update(int filter_delay_blocks,
                                       bool consistent_filter,
                                       bool converged_filter,
                                       bool diverged_filter,
                                       bool active_render,
                                       bool saturated_capture) {
  ++capture_block_counter_;
  strong_not_saturated_render_blocks_ +=
      active_render && !saturated_capture ? 1 : 0;

  if (consistent_filter && filter_delay_blocks < 5) {
    sane_filter_observed_ = true;
    active_blocks_since_sane_filter_ = 0;
  } else if (active_render) {
    ++active_blocks_since_sane_filter_;
  }

  bool sane_filter_recently_seen;
  if (!sane_filter_observed_) {
    sane_filter_recently_seen =
        capture_block_counter_ <= 5 * kNumBlocksPerSecond;
  } else {
    sane_filter_recently_seen =
        active_blocks_since_sane_filter_ <= 30 * kNumBlocksPerSecond;
  }

  if (converged_filter) {
    recent_convergence_during_activity_ = true;
    active_non_converged_sequence_size_ = 0;
    non_converged_sequence_size_ = 0;
    ++num_converged_blocks_;
  } else {
    if (++non_converged_sequence_size_ > 20 * kNumBlocksPerSecond) {
      num_converged_blocks_ = 0;
    }

    if (active_render &&
        ++active_non_converged_sequence_size_ > 60 * kNumBlocksPerSecond) {
      recent_convergence_during_activity_ = false;
    }
  }

  if (!diverged_filter) {
    diverged_sequence_size_ = 0;
  } else if (++diverged_sequence_size_ >= 60) {
    // TODO(peah): Change these lines to ensure proper triggering of usable
    // filter.
    non_converged_sequence_size_ = kBlocksSinceConvergencedFilterInit;
  }

  if (active_non_converged_sequence_size_ > 60 * kNumBlocksPerSecond) {
    finite_erl_recently_detected_ = false;
  }
  if (num_converged_blocks_ > 50) {
    finite_erl_recently_detected_ = true;
  }

  if (bounded_erl_) {
    transparency_activated_ = false;
  } else if (finite_erl_recently_detected_) {
    transparency_activated_ = false;
  } else if (sane_filter_recently_seen && recent_convergence_during_activity_) {
    transparency_activated_ = false;
  } else {
    const bool filter_should_have_converged =
        strong_not_saturated_render_blocks_ > 6 * kNumBlocksPerSecond;
    transparency_activated_ = filter_should_have_converged;
  }
}

AecState::FilteringQualityAnalyzer::FilteringQualityAnalyzer(
    const EchoCanceller3Config& config) {}

void AecState::FilteringQualityAnalyzer::Reset() {
  usable_linear_estimate_ = false;
  filter_update_blocks_since_reset_ = 0;
}

void AecState::FilteringQualityAnalyzer::Update(
    bool active_render,
    bool transparent_mode,
    bool saturated_capture,
    bool consistent_estimate_,
    const absl::optional<DelayEstimate>& external_delay,
    bool converged_filter) {
  // Update blocks counter.
  const bool filter_update = active_render && !saturated_capture;
  filter_update_blocks_since_reset_ += filter_update ? 1 : 0;
  filter_update_blocks_since_start_ += filter_update ? 1 : 0;

  // Store convergence flag when observed.
  convergence_seen_ = convergence_seen_ || converged_filter;

  // Verify requirements for achieving a decent filter. The requirements for
  // filter adaptation at call startup are more restrictive than after an
  // in-call reset.
  const bool sufficient_data_to_converge_at_startup =
      filter_update_blocks_since_start_ > kNumBlocksPerSecond * 0.4f;
  const bool sufficient_data_to_converge_at_reset =
      sufficient_data_to_converge_at_startup &&
      filter_update_blocks_since_reset_ > kNumBlocksPerSecond * 0.2f;

  // The linear filter can only be used it has had time to converge.
  usable_linear_estimate_ = sufficient_data_to_converge_at_startup &&
                            sufficient_data_to_converge_at_reset;

  // The linear filter can only be used if an external delay or convergence have
  // been identified
  usable_linear_estimate_ =
      usable_linear_estimate_ && (external_delay || convergence_seen_);

  // If transparent mode is on, deactivate usign the linear filter.
  usable_linear_estimate_ = usable_linear_estimate_ && !transparent_mode;
}

AecState::LegacyFilteringQualityAnalyzer::LegacyFilteringQualityAnalyzer(
    const EchoCanceller3Config& config)
    : conservative_initial_phase_(config.filter.conservative_initial_phase),
      required_blocks_for_convergence_(
          kNumBlocksPerSecond * (conservative_initial_phase_ ? 1.5f : 0.8f)),
      linear_and_stable_echo_path_(
          config.echo_removal_control.linear_and_stable_echo_path),
      non_converged_sequence_size_(kBlocksSinceConvergencedFilterInit) {}

void AecState::LegacyFilteringQualityAnalyzer::Reset() {
  usable_linear_estimate_ = false;
  strong_not_saturated_render_blocks_ = 0;
  if (linear_and_stable_echo_path_) {
    recent_convergence_during_activity_ = false;
  }
  diverged_sequence_size_ = 0;
  // TODO(peah): Change to ensure proper triggering of usable filter.
  non_converged_sequence_size_ = 10000;
  recent_convergence_ = true;
}

void AecState::LegacyFilteringQualityAnalyzer::Update(
    bool saturated_echo,
    bool active_render,
    bool saturated_capture,
    bool transparent_mode,
    const absl::optional<DelayEstimate>& external_delay,
    bool converged_filter,
    bool diverged_filter) {
  diverged_sequence_size_ = diverged_filter ? diverged_sequence_size_ + 1 : 0;
  if (diverged_sequence_size_ >= 60) {
    // TODO(peah): Change these lines to ensure proper triggering of usable
    // filter.
    non_converged_sequence_size_ = 10000;
    recent_convergence_ = true;
  }

  if (converged_filter) {
    non_converged_sequence_size_ = 0;
    recent_convergence_ = true;
    active_non_converged_sequence_size_ = 0;
    recent_convergence_during_activity_ = true;
  } else {
    if (++non_converged_sequence_size_ >= 60 * kNumBlocksPerSecond) {
      recent_convergence_ = false;
    }

    if (active_render &&
        ++active_non_converged_sequence_size_ > 60 * kNumBlocksPerSecond) {
      recent_convergence_during_activity_ = false;
    }
  }

  strong_not_saturated_render_blocks_ +=
      active_render && !saturated_capture ? 1 : 0;
  const bool filter_has_had_time_to_converge =
      strong_not_saturated_render_blocks_ > required_blocks_for_convergence_;

  usable_linear_estimate_ = filter_has_had_time_to_converge && external_delay;

  if (!conservative_initial_phase_ && recent_convergence_during_activity_) {
    usable_linear_estimate_ = true;
  }

  if (!linear_and_stable_echo_path_ && !recent_convergence_) {
    usable_linear_estimate_ = false;
  }

  if (saturated_echo || transparent_mode) {
    usable_linear_estimate_ = false;
  }
}

void AecState::SaturationDetector::Update(
    rtc::ArrayView<const float> x,
    bool saturated_capture,
    bool usable_linear_estimate,
    const SubtractorOutput& subtractor_output,
    float echo_path_gain) {
  saturated_echo_ = saturated_capture;
  if (usable_linear_estimate) {
    constexpr float kSaturationThreshold = 20000.f;
    saturated_echo_ =
        saturated_echo_ &&
        (subtractor_output.s_main_max_abs > kSaturationThreshold ||
         subtractor_output.s_shadow_max_abs > kSaturationThreshold);
  } else {
    const float max_sample = fabs(*std::max_element(
        x.begin(), x.end(), [](float a, float b) { return a * a < b * b; }));

    const float kMargin = 10.f;
    float peak_echo_amplitude = max_sample * echo_path_gain * kMargin;
    saturated_echo_ = saturated_echo_ && peak_echo_amplitude > 32000;
  }
}

AecState::LegacySaturationDetector::LegacySaturationDetector(
    const EchoCanceller3Config& config)
    : echo_can_saturate_(config.ep_strength.echo_can_saturate),
      not_saturated_sequence_size_(1000) {}

void AecState::LegacySaturationDetector::Reset() {
  not_saturated_sequence_size_ = 0;
}

void AecState::LegacySaturationDetector::Update(rtc::ArrayView<const float> x,
                                                bool saturated_capture,
                                                float echo_path_gain) {
  if (!echo_can_saturate_) {
    saturated_echo_ = false;
    return;
  }

  RTC_DCHECK_LT(0, x.size());
  if (saturated_capture) {
    const float max_sample = fabs(*std::max_element(
        x.begin(), x.end(), [](float a, float b) { return a * a < b * b; }));

    // Set flag for potential presence of saturated echo
    const float kMargin = 10.f;
    float peak_echo_amplitude = max_sample * echo_path_gain * kMargin;
    if (peak_echo_amplitude > 32000) {
      not_saturated_sequence_size_ = 0;
      saturated_echo_ = true;
      return;
    }
  }

  saturated_echo_ = ++not_saturated_sequence_size_ < 5;
}

}  // namespace webrtc