modules/audio_processing/include/audio_processing.h

/*
 *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
 *
 *  Use of this source code is governed by a BSD-style license
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
 *  in the file PATENTS.  All contributing project authors may
 *  be found in the AUTHORS file in the root of the source tree.
 */

#ifndef MODULES_AUDIO_PROCESSING_INCLUDE_AUDIO_PROCESSING_H_
#define MODULES_AUDIO_PROCESSING_INCLUDE_AUDIO_PROCESSING_H_

// MSVC++ requires this to be set before any other includes to get M_PI.
#ifndef _USE_MATH_DEFINES
#define _USE_MATH_DEFINES
#endif

#include <math.h>
#include <stddef.h>  // size_t
#include <stdio.h>   // FILE
#include <string.h>

#include <vector>

#include "absl/types/optional.h"
#include "api/array_view.h"
#include "api/audio/echo_canceller3_config.h"
#include "api/audio/echo_control.h"
#include "api/scoped_refptr.h"
#include "modules/audio_processing/include/audio_processing_statistics.h"
#include "modules/audio_processing/include/config.h"
#include "rtc_base/arraysize.h"
#include "rtc_base/constructor_magic.h"
#include "rtc_base/ref_count.h"
#include "rtc_base/system/file_wrapper.h"
#include "rtc_base/system/rtc_export.h"

namespace rtc {
class TaskQueue;
}  // namespace rtc

namespace webrtc {

class AecDump;
class AudioBuffer;

class StreamConfig;
class ProcessingConfig;

class EchoDetector;
class CustomAudioAnalyzer;
class CustomProcessing;

// Use to enable experimental gain control (AGC). At startup the experimental
// AGC moves the microphone volume up to `startup_min_volume` if the current
// microphone volume is set too low. The value is clamped to its operating range
// [12, 255]. Here, 255 maps to 100%.
//
// Must be provided through AudioProcessingBuilder().Create(config).
#if defined(WEBRTC_CHROMIUM_BUILD)
static constexpr int kAgcStartupMinVolume = 85;
#else
static constexpr int kAgcStartupMinVolume = 0;
#endif  // defined(WEBRTC_CHROMIUM_BUILD)
static constexpr int kClippedLevelMin = 70;

// The Audio Processing Module (APM) provides a collection of voice processing
// components designed for real-time communications software.
//
// APM operates on two audio streams on a frame-by-frame basis. Frames of the
// primary stream, on which all processing is applied, are passed to
// `ProcessStream()`. Frames of the reverse direction stream are passed to
// `ProcessReverseStream()`. On the client-side, this will typically be the
// near-end (capture) and far-end (render) streams, respectively. APM should be
// placed in the signal chain as close to the audio hardware abstraction layer
// (HAL) as possible.
//
// On the server-side, the reverse stream will normally not be used, with
// processing occurring on each incoming stream.
//
// Component interfaces follow a similar pattern and are accessed through
// corresponding getters in APM. All components are disabled at create-time,
// with default settings that are recommended for most situations. New settings
// can be applied without enabling a component. Enabling a component triggers
// memory allocation and initialization to allow it to start processing the
// streams.
//
// Thread safety is provided with the following assumptions to reduce locking
// overhead:
//   1. The stream getters and setters are called from the same thread as
//      ProcessStream(). More precisely, stream functions are never called
//      concurrently with ProcessStream().
//   2. Parameter getters are never called concurrently with the corresponding
//      setter.
//
// APM accepts only linear PCM audio data in chunks of 10 ms. The int16
// interfaces use interleaved data, while the float interfaces use deinterleaved
// data.
//
// Usage example, omitting error checking:
// AudioProcessing* apm = AudioProcessingBuilder().Create();
//
// AudioProcessing::Config config;
// config.echo_canceller.enabled = true;
// config.echo_canceller.mobile_mode = false;
//
// config.gain_controller1.enabled = true;
// config.gain_controller1.mode =
// AudioProcessing::Config::GainController1::kAdaptiveAnalog;
// config.gain_controller1.analog_level_minimum = 0;
// config.gain_controller1.analog_level_maximum = 255;
//
// config.gain_controller2.enabled = true;
//
// config.high_pass_filter.enabled = true;
//
// config.voice_detection.enabled = true;
//
// apm->ApplyConfig(config)
//
// apm->noise_reduction()->set_level(kHighSuppression);
// apm->noise_reduction()->Enable(true);
//
// // Start a voice call...
//
// // ... Render frame arrives bound for the audio HAL ...
// apm->ProcessReverseStream(render_frame);
//
// // ... Capture frame arrives from the audio HAL ...
// // Call required set_stream_ functions.
// apm->set_stream_delay_ms(delay_ms);
// apm->set_stream_analog_level(analog_level);
//
// apm->ProcessStream(capture_frame);
//
// // Call required stream_ functions.
// analog_level = apm->recommended_stream_analog_level();
// has_voice = apm->stream_has_voice();
//
// // Repeat render and capture processing for the duration of the call...
// // Start a new call...
// apm->Initialize();
//
// // Close the application...
// delete apm;
//
class RTC_EXPORT AudioProcessing : public rtc::RefCountInterface {
 public:
  // The struct below constitutes the new parameter scheme for the audio
  // processing. It is being introduced gradually and until it is fully
  // introduced, it is prone to change.
  // TODO(peah): Remove this comment once the new config scheme is fully rolled
  // out.
  //
  // The parameters and behavior of the audio processing module are controlled
  // by changing the default values in the AudioProcessing::Config struct.
  // The config is applied by passing the struct to the ApplyConfig method.
  //
  // This config is intended to be used during setup, and to enable/disable
  // top-level processing effects. Use during processing may cause undesired
  // submodule resets, affecting the audio quality. Use the RuntimeSetting
  // construct for runtime configuration.
  struct RTC_EXPORT Config {

    // Sets the properties of the audio processing pipeline.
    struct RTC_EXPORT Pipeline {
      // Maximum allowed processing rate used internally. May only be set to
      // 32000 or 48000 and any differing values will be treated as 48000.
      int maximum_internal_processing_rate = 48000;
      // Allow multi-channel processing of render audio.
      bool multi_channel_render = false;
      // Allow multi-channel processing of capture audio when AEC3 is active
      // or a custom AEC is injected..
      bool multi_channel_capture = false;
    } pipeline;

    // Enabled the pre-amplifier. It amplifies the capture signal
    // before any other processing is done.
    // TODO(webrtc:5298): Deprecate and use the pre-gain functionality in
    // capture_level_adjustment instead.
    struct PreAmplifier {
      bool enabled = false;
      float fixed_gain_factor = 1.0f;
    } pre_amplifier;

    // Functionality for general level adjustment in the capture pipeline. This
    // should not be used together with the legacy PreAmplifier functionality.
    struct CaptureLevelAdjustment {
      bool operator==(const CaptureLevelAdjustment& rhs) const;
      bool operator!=(const CaptureLevelAdjustment& rhs) const {
        return !(*this == rhs);
      }
      bool enabled = false;
      // The `pre_gain_factor` scales the signal before any processing is done.
      float pre_gain_factor = 1.0f;
      // The `post_gain_factor` scales the signal after all processing is done.
      float post_gain_factor = 1.0f;
      struct AnalogMicGainEmulation {
        bool operator==(const AnalogMicGainEmulation& rhs) const;
        bool operator!=(const AnalogMicGainEmulation& rhs) const {
          return !(*this == rhs);
        }
        bool enabled = false;
        // Initial analog gain level to use for the emulated analog gain. Must
        // be in the range [0...255].
        int initial_level = 255;
      } analog_mic_gain_emulation;
    } capture_level_adjustment;

    struct HighPassFilter {
      bool enabled = false;
      bool apply_in_full_band = true;
    } high_pass_filter;

    struct EchoCanceller {
      bool enabled = false;
      bool mobile_mode = false;
      bool export_linear_aec_output = false;
      // Enforce the highpass filter to be on (has no effect for the mobile
      // mode).
      bool enforce_high_pass_filtering = true;
    } echo_canceller;

    // Enables background noise suppression.
    struct NoiseSuppression {
      bool enabled = false;
      enum Level { kLow, kModerate, kHigh, kVeryHigh };
      Level level = kModerate;
      bool analyze_linear_aec_output_when_available = false;
    } noise_suppression;

    // Enables transient suppression.
    struct TransientSuppression {
      bool enabled = false;
    } transient_suppression;

    // Enables reporting of `voice_detected` in webrtc::AudioProcessingStats.
    struct VoiceDetection {
      bool enabled = false;
    } voice_detection;

    // Enables automatic gain control (AGC) functionality.
    // The automatic gain control (AGC) component brings the signal to an
    // appropriate range. This is done by applying a digital gain directly and,
    // in the analog mode, prescribing an analog gain to be applied at the audio
    // HAL.
    // Recommended to be enabled on the client-side.
    struct RTC_EXPORT GainController1 {
      bool operator==(const GainController1& rhs) const;
      bool operator!=(const GainController1& rhs) const {
        return !(*this == rhs);
      }

      bool enabled = false;
      enum Mode {
        // Adaptive mode intended for use if an analog volume control is
        // available on the capture device. It will require the user to provide
        // coupling between the OS mixer controls and AGC through the
        // stream_analog_level() functions.
        // It consists of an analog gain prescription for the audio device and a
        // digital compression stage.
        kAdaptiveAnalog,
        // Adaptive mode intended for situations in which an analog volume
        // control is unavailable. It operates in a similar fashion to the
        // adaptive analog mode, but with scaling instead applied in the digital
        // domain. As with the analog mode, it additionally uses a digital
        // compression stage.
        kAdaptiveDigital,
        // Fixed mode which enables only the digital compression stage also used
        // by the two adaptive modes.
        // It is distinguished from the adaptive modes by considering only a
        // short time-window of the input signal. It applies a fixed gain
        // through most of the input level range, and compresses (gradually
        // reduces gain with increasing level) the input signal at higher
        // levels. This mode is preferred on embedded devices where the capture
        // signal level is predictable, so that a known gain can be applied.
        kFixedDigital
      };
      Mode mode = kAdaptiveAnalog;
      // Sets the target peak level (or envelope) of the AGC in dBFs (decibels
      // from digital full-scale). The convention is to use positive values. For
      // instance, passing in a value of 3 corresponds to -3 dBFs, or a target
      // level 3 dB below full-scale. Limited to [0, 31].
      int target_level_dbfs = 3;
      // Sets the maximum gain the digital compression stage may apply, in dB. A
      // higher number corresponds to greater compression, while a value of 0
      // will leave the signal uncompressed. Limited to [0, 90].
      // For updates after APM setup, use a RuntimeSetting instead.
      int compression_gain_db = 9;
      // When enabled, the compression stage will hard limit the signal to the
      // target level. Otherwise, the signal will be compressed but not limited
      // above the target level.
      bool enable_limiter = true;
      // Sets the minimum and maximum analog levels of the audio capture device.
      // Must be set if an analog mode is used. Limited to [0, 65535].
      int analog_level_minimum = 0;
      int analog_level_maximum = 255;

      // Enables the analog gain controller functionality.
      struct AnalogGainController {
        bool enabled = true;
        int startup_min_volume = kAgcStartupMinVolume;
        // Lowest analog microphone level that will be applied in response to
        // clipping.
        int clipped_level_min = kClippedLevelMin;
        bool enable_digital_adaptive = true;
        // Amount the microphone level is lowered with every clipping event.
        // Limited to (0, 255].
        int clipped_level_step = 15;
        // Proportion of clipped samples required to declare a clipping event.
        // Limited to (0.f, 1.f).
        float clipped_ratio_threshold = 0.1f;
        // Time in frames to wait after a clipping event before checking again.
        // Limited to values higher than 0.
        int clipped_wait_frames = 300;

        // Enables clipping prediction functionality.
        struct ClippingPredictor {
          bool enabled = false;
          enum Mode {
            // Clipping event prediction mode with fixed step estimation.
            kClippingEventPrediction,
            // Clipped peak estimation mode with adaptive step estimation.
            kAdaptiveStepClippingPeakPrediction,
            // Clipped peak estimation mode with fixed step estimation.
            kFixedStepClippingPeakPrediction,
          };
          Mode mode = kClippingEventPrediction;
          // Number of frames in the sliding analysis window.
          int window_length = 5;
          // Number of frames in the sliding reference window.
          int reference_window_length = 5;
          // Reference window delay (unit: number of frames).
          int reference_window_delay = 5;
          // Clipping prediction threshold (dBFS).
          float clipping_threshold = -1.0f;
          // Crest factor drop threshold (dB).
          float crest_factor_margin = 3.0f;
          // If true, the recommended clipped level step is used to modify the
          // analog gain. Otherwise, the predictor runs without affecting the
          // analog gain.
          bool use_predicted_step = true;
        } clipping_predictor;
      } analog_gain_controller;
    } gain_controller1;

    // Enables the next generation AGC functionality. This feature replaces the
    // standard methods of gain control in the previous AGC. Enabling this
    // submodule enables an adaptive digital AGC followed by a limiter. By
    // setting `fixed_gain_db`, the limiter can be turned into a compressor that
    // first applies a fixed gain. The adaptive digital AGC can be turned off by
    // setting |adaptive_digital_mode=false|.
    struct RTC_EXPORT GainController2 {
      bool operator==(const GainController2& rhs) const;
      bool operator!=(const GainController2& rhs) const {
        return !(*this == rhs);
      }

      // TODO(crbug.com/webrtc/7494): Remove `LevelEstimator`.
      enum LevelEstimator { kRms, kPeak };
      enum NoiseEstimator { kStationaryNoise, kNoiseFloor };
      bool enabled = false;
      struct FixedDigital {
        float gain_db = 0.0f;
      } fixed_digital;
      struct RTC_EXPORT AdaptiveDigital {
        bool operator==(const AdaptiveDigital& rhs) const;
        bool operator!=(const AdaptiveDigital& rhs) const {
          return !(*this == rhs);
        }

        bool enabled = false;
        // Run the adaptive digital controller but the signal is not modified.
        bool dry_run = false;
        int vad_reset_period_ms = 1500;
        int adjacent_speech_frames_threshold = 12;
        float max_gain_change_db_per_second = 3.0f;
        float max_output_noise_level_dbfs = -50.0f;
        bool sse2_allowed = true;
        bool avx2_allowed = true;
        bool neon_allowed = true;
        // TODO(crbug.com/webrtc/7494): Remove deprecated settings below.
        NoiseEstimator noise_estimator = kNoiseFloor;
        float vad_probability_attack = 1.0f;
        LevelEstimator level_estimator = kRms;
        int level_estimator_adjacent_speech_frames_threshold = 12;
        bool use_saturation_protector = true;
        float initial_saturation_margin_db = 25.0f;
        float extra_saturation_margin_db = 5.0f;
        int gain_applier_adjacent_speech_frames_threshold = 12;
      } adaptive_digital;
    } gain_controller2;

    struct ResidualEchoDetector {
      bool enabled = true;
    } residual_echo_detector;

    // Enables reporting of `output_rms_dbfs` in webrtc::AudioProcessingStats.
    struct LevelEstimation {
      bool enabled = false;
    } level_estimation;

    std::string ToString() const;
  };

  // TODO(mgraczyk): Remove once all methods that use ChannelLayout are gone.
  enum ChannelLayout {
    kMono,
    // Left, right.
    kStereo,
    // Mono, keyboard, and mic.
    kMonoAndKeyboard,
    // Left, right, keyboard, and mic.
    kStereoAndKeyboard
  };

  // Specifies the properties of a setting to be passed to AudioProcessing at
  // runtime.
  class RuntimeSetting {
   public:
    enum class Type {
      kNotSpecified,
      kCapturePreGain,
      kCaptureCompressionGain,
      kCaptureFixedPostGain,
      kPlayoutVolumeChange,
      kCustomRenderProcessingRuntimeSetting,
      kPlayoutAudioDeviceChange,
      kCapturePostGain,
      kCaptureOutputUsed
    };

    // Play-out audio device properties.
    struct PlayoutAudioDeviceInfo {
      int id;          // Identifies the audio device.
      int max_volume;  // Maximum play-out volume.
    };

    RuntimeSetting() : type_(Type::kNotSpecified), value_(0.0f) {}
    ~RuntimeSetting() = default;

    static RuntimeSetting CreateCapturePreGain(float gain) {
      return {Type::kCapturePreGain, gain};
    }

    static RuntimeSetting CreateCapturePostGain(float gain) {
      return {Type::kCapturePostGain, gain};
    }

    // Corresponds to Config::GainController1::compression_gain_db, but for
    // runtime configuration.
    static RuntimeSetting CreateCompressionGainDb(int gain_db) {
      RTC_DCHECK_GE(gain_db, 0);
      RTC_DCHECK_LE(gain_db, 90);
      return {Type::kCaptureCompressionGain, static_cast<float>(gain_db)};
    }

    // Corresponds to Config::GainController2::fixed_digital::gain_db, but for
    // runtime configuration.
    static RuntimeSetting CreateCaptureFixedPostGain(float gain_db) {
      RTC_DCHECK_GE(gain_db, 0.0f);
      RTC_DCHECK_LE(gain_db, 90.0f);
      return {Type::kCaptureFixedPostGain, gain_db};
    }

    // Creates a runtime setting to notify play-out (aka render) audio device
    // changes.
    static RuntimeSetting CreatePlayoutAudioDeviceChange(
        PlayoutAudioDeviceInfo audio_device) {
      return {Type::kPlayoutAudioDeviceChange, audio_device};
    }

    // Creates a runtime setting to notify play-out (aka render) volume changes.
    // `volume` is the unnormalized volume, the maximum of which
    static RuntimeSetting CreatePlayoutVolumeChange(int volume) {
      return {Type::kPlayoutVolumeChange, volume};
    }

    static RuntimeSetting CreateCustomRenderSetting(float payload) {
      return {Type::kCustomRenderProcessingRuntimeSetting, payload};
    }

    static RuntimeSetting CreateCaptureOutputUsedSetting(
        bool capture_output_used) {
      return {Type::kCaptureOutputUsed, capture_output_used};
    }

    Type type() const { return type_; }
    // Getters do not return a value but instead modify the argument to protect
    // from implicit casting.
    void GetFloat(float* value) const {
      RTC_DCHECK(value);
      *value = value_.float_value;
    }
    void GetInt(int* value) const {
      RTC_DCHECK(value);
      *value = value_.int_value;
    }
    void GetBool(bool* value) const {
      RTC_DCHECK(value);
      *value = value_.bool_value;
    }
    void GetPlayoutAudioDeviceInfo(PlayoutAudioDeviceInfo* value) const {
      RTC_DCHECK(value);
      *value = value_.playout_audio_device_info;
    }

   private:
    RuntimeSetting(Type id, float value) : type_(id), value_(value) {}
    RuntimeSetting(Type id, int value) : type_(id), value_(value) {}
    RuntimeSetting(Type id, PlayoutAudioDeviceInfo value)
        : type_(id), value_(value) {}
    Type type_;
    union U {
      U() {}
      U(int value) : int_value(value) {}
      U(float value) : float_value(value) {}
      U(PlayoutAudioDeviceInfo value) : playout_audio_device_info(value) {}
      float float_value;
      int int_value;
      bool bool_value;
      PlayoutAudioDeviceInfo playout_audio_device_info;
    } value_;
  };

  ~AudioProcessing() override {}

  // Initializes internal states, while retaining all user settings. This
  // should be called before beginning to process a new audio stream. However,
  // it is not necessary to call before processing the first stream after
  // creation.
  //
  // It is also not necessary to call if the audio parameters (sample
  // rate and number of channels) have changed. Passing updated parameters
  // directly to `ProcessStream()` and `ProcessReverseStream()` is permissible.
  // If the parameters are known at init-time though, they may be provided.
  // TODO(webrtc:5298): Change to return void.
  virtual int Initialize() = 0;

  // The int16 interfaces require:
  //   - only `NativeRate`s be used
  //   - that the input, output and reverse rates must match
  //   - that `processing_config.output_stream()` matches
  //     `processing_config.input_stream()`.
  //
  // The float interfaces accept arbitrary rates and support differing input and
  // output layouts, but the output must have either one channel or the same
  // number of channels as the input.
  virtual int Initialize(const ProcessingConfig& processing_config) = 0;

  // Initialize with unpacked parameters. See Initialize() above for details.
  //
  // TODO(mgraczyk): Remove once clients are updated to use the new interface.
  virtual int Initialize(int capture_input_sample_rate_hz,
                         int capture_output_sample_rate_hz,
                         int render_sample_rate_hz,
                         ChannelLayout capture_input_layout,
                         ChannelLayout capture_output_layout,
                         ChannelLayout render_input_layout) = 0;

  // TODO(peah): This method is a temporary solution used to take control
  // over the parameters in the audio processing module and is likely to change.
  virtual void ApplyConfig(const Config& config) = 0;

  // TODO(ajm): Only intended for internal use. Make private and friend the
  // necessary classes?
  virtual int proc_sample_rate_hz() const = 0;
  virtual int proc_split_sample_rate_hz() const = 0;
  virtual size_t num_input_channels() const = 0;
  virtual size_t num_proc_channels() const = 0;
  virtual size_t num_output_channels() const = 0;
  virtual size_t num_reverse_channels() const = 0;

  // Set to true when the output of AudioProcessing will be muted or in some
  // other way not used. Ideally, the captured audio would still be processed,
  // but some components may change behavior based on this information.
  // Default false. This method takes a lock. To achieve this in a lock-less
  // manner the PostRuntimeSetting can instead be used.
  virtual void set_output_will_be_muted(bool muted) = 0;

  // Enqueues a runtime setting.
  virtual void SetRuntimeSetting(RuntimeSetting setting) = 0;

  // Enqueues a runtime setting. Returns a bool indicating whether the
  // enqueueing was successfull.
  virtual bool PostRuntimeSetting(RuntimeSetting setting) = 0;

  // Accepts and produces a 10 ms frame interleaved 16 bit integer audio as
  // specified in `input_config` and `output_config`. `src` and `dest` may use
  // the same memory, if desired.
  virtual int ProcessStream(const int16_t* const src,
                            const StreamConfig& input_config,
                            const StreamConfig& output_config,
                            int16_t* const dest) = 0;

  // Accepts deinterleaved float audio with the range [-1, 1]. Each element of
  // `src` points to a channel buffer, arranged according to `input_stream`. At
  // output, the channels will be arranged according to `output_stream` in
  // `dest`.
  //
  // The output must have one channel or as many channels as the input. `src`
  // and `dest` may use the same memory, if desired.
  virtual int ProcessStream(const float* const* src,
                            const StreamConfig& input_config,
                            const StreamConfig& output_config,
                            float* const* dest) = 0;

  // Accepts and produces a 10 ms frame of interleaved 16 bit integer audio for
  // the reverse direction audio stream as specified in `input_config` and
  // `output_config`. `src` and `dest` may use the same memory, if desired.
  virtual int ProcessReverseStream(const int16_t* const src,
                                   const StreamConfig& input_config,
                                   const StreamConfig& output_config,
                                   int16_t* const dest) = 0;

  // Accepts deinterleaved float audio with the range [-1, 1]. Each element of
  // `data` points to a channel buffer, arranged according to `reverse_config`.
  virtual int ProcessReverseStream(const float* const* src,
                                   const StreamConfig& input_config,
                                   const StreamConfig& output_config,
                                   float* const* dest) = 0;

  // Accepts deinterleaved float audio with the range [-1, 1]. Each element
  // of `data` points to a channel buffer, arranged according to
  // `reverse_config`.
  virtual int AnalyzeReverseStream(const float* const* data,
                                   const StreamConfig& reverse_config) = 0;

  // Returns the most recently produced 10 ms of the linear AEC output at a rate
  // of 16 kHz. If there is more than one capture channel, a mono representation
  // of the input is returned. Returns true/false to indicate whether an output
  // returned.
  virtual bool GetLinearAecOutput(
      rtc::ArrayView<std::array<float, 160>> linear_output) const = 0;

  // This must be called prior to ProcessStream() if and only if adaptive analog
  // gain control is enabled, to pass the current analog level from the audio
  // HAL. Must be within the range provided in Config::GainController1.
  virtual void set_stream_analog_level(int level) = 0;

  // When an analog mode is set, this should be called after ProcessStream()
  // to obtain the recommended new analog level for the audio HAL. It is the
  // user's responsibility to apply this level.
  virtual int recommended_stream_analog_level() const = 0;

  // This must be called if and only if echo processing is enabled.
  //
  // Sets the `delay` in ms between ProcessReverseStream() receiving a far-end
  // frame and ProcessStream() receiving a near-end frame containing the
  // corresponding echo. On the client-side this can be expressed as
  //   delay = (t_render - t_analyze) + (t_process - t_capture)
  // where,
  //   - t_analyze is the time a frame is passed to ProcessReverseStream() and
  //     t_render is the time the first sample of the same frame is rendered by
  //     the audio hardware.
  //   - t_capture is the time the first sample of a frame is captured by the
  //     audio hardware and t_process is the time the same frame is passed to
  //     ProcessStream().
  virtual int set_stream_delay_ms(int delay) = 0;
  virtual int stream_delay_ms() const = 0;

  // Call to signal that a key press occurred (true) or did not occur (false)
  // with this chunk of audio.
  virtual void set_stream_key_pressed(bool key_pressed) = 0;

  // Creates and attaches an webrtc::AecDump for recording debugging
  // information.
  // The `worker_queue` may not be null and must outlive the created
  // AecDump instance. |max_log_size_bytes == -1| means the log size
  // will be unlimited. `handle` may not be null. The AecDump takes
  // responsibility for `handle` and closes it in the destructor. A
  // return value of true indicates that the file has been
  // sucessfully opened, while a value of false indicates that
  // opening the file failed.
  virtual bool CreateAndAttachAecDump(const std::string& file_name,
                                      int64_t max_log_size_bytes,
                                      rtc::TaskQueue* worker_queue) = 0;
  virtual bool CreateAndAttachAecDump(FILE* handle,
                                      int64_t max_log_size_bytes,
                                      rtc::TaskQueue* worker_queue) = 0;

  // TODO(webrtc:5298) Deprecated variant.
  // Attaches provided webrtc::AecDump for recording debugging
  // information. Log file and maximum file size logic is supposed to
  // be handled by implementing instance of AecDump. Calling this
  // method when another AecDump is attached resets the active AecDump
  // with a new one. This causes the d-tor of the earlier AecDump to
  // be called. The d-tor call may block until all pending logging
  // tasks are completed.
  virtual void AttachAecDump(std::unique_ptr<AecDump> aec_dump) = 0;

  // If no AecDump is attached, this has no effect. If an AecDump is
  // attached, it's destructor is called. The d-tor may block until
  // all pending logging tasks are completed.
  virtual void DetachAecDump() = 0;

  // Get audio processing statistics.
  virtual AudioProcessingStats GetStatistics() = 0;
  // TODO(webrtc:5298) Deprecated variant. The `has_remote_tracks` argument
  // should be set if there are active remote tracks (this would usually be true
  // during a call). If there are no remote tracks some of the stats will not be
  // set by AudioProcessing, because they only make sense if there is at least
  // one remote track.
  virtual AudioProcessingStats GetStatistics(bool has_remote_tracks) = 0;

  // Returns the last applied configuration.
  virtual AudioProcessing::Config GetConfig() const = 0;

  enum Error {
    // Fatal errors.
    kNoError = 0,
    kUnspecifiedError = -1,
    kCreationFailedError = -2,
    kUnsupportedComponentError = -3,
    kUnsupportedFunctionError = -4,
    kNullPointerError = -5,
    kBadParameterError = -6,
    kBadSampleRateError = -7,
    kBadDataLengthError = -8,
    kBadNumberChannelsError = -9,
    kFileError = -10,
    kStreamParameterNotSetError = -11,
    kNotEnabledError = -12,

    // Warnings are non-fatal.
    // This results when a set_stream_ parameter is out of range. Processing
    // will continue, but the parameter may have been truncated.
    kBadStreamParameterWarning = -13
  };

  // Native rates supported by the integer interfaces.
  enum NativeRate {
    kSampleRate8kHz = 8000,
    kSampleRate16kHz = 16000,
    kSampleRate32kHz = 32000,
    kSampleRate48kHz = 48000
  };

  // TODO(kwiberg): We currently need to support a compiler (Visual C++) that
  // complains if we don't explicitly state the size of the array here. Remove
  // the size when that's no longer the case.
  static constexpr int kNativeSampleRatesHz[4] = {
      kSampleRate8kHz, kSampleRate16kHz, kSampleRate32kHz, kSampleRate48kHz};
  static constexpr size_t kNumNativeSampleRates =
      arraysize(kNativeSampleRatesHz);
  static constexpr int kMaxNativeSampleRateHz =
      kNativeSampleRatesHz[kNumNativeSampleRates - 1];

  static constexpr int kChunkSizeMs = 10;
};

class RTC_EXPORT AudioProcessingBuilder {
 public:
  AudioProcessingBuilder();
  ~AudioProcessingBuilder();
  // The AudioProcessingBuilder takes ownership of the echo_control_factory.
  AudioProcessingBuilder& SetEchoControlFactory(
      std::unique_ptr<EchoControlFactory> echo_control_factory) {
    echo_control_factory_ = std::move(echo_control_factory);
    return *this;
  }
  // The AudioProcessingBuilder takes ownership of the capture_post_processing.
  AudioProcessingBuilder& SetCapturePostProcessing(
      std::unique_ptr<CustomProcessing> capture_post_processing) {
    capture_post_processing_ = std::move(capture_post_processing);
    return *this;
  }
  // The AudioProcessingBuilder takes ownership of the render_pre_processing.
  AudioProcessingBuilder& SetRenderPreProcessing(
      std::unique_ptr<CustomProcessing> render_pre_processing) {
    render_pre_processing_ = std::move(render_pre_processing);
    return *this;
  }
  // The AudioProcessingBuilder takes ownership of the echo_detector.
  AudioProcessingBuilder& SetEchoDetector(
      rtc::scoped_refptr<EchoDetector> echo_detector) {
    echo_detector_ = std::move(echo_detector);
    return *this;
  }
  // The AudioProcessingBuilder takes ownership of the capture_analyzer.
  AudioProcessingBuilder& SetCaptureAnalyzer(
      std::unique_ptr<CustomAudioAnalyzer> capture_analyzer) {
    capture_analyzer_ = std::move(capture_analyzer);
    return *this;
  }
  // This creates an APM instance using the previously set components. Calling
  // the Create function resets the AudioProcessingBuilder to its initial state.
  rtc::scoped_refptr<AudioProcessing> Create();

 private:
  std::unique_ptr<EchoControlFactory> echo_control_factory_;
  std::unique_ptr<CustomProcessing> capture_post_processing_;
  std::unique_ptr<CustomProcessing> render_pre_processing_;
  rtc::scoped_refptr<EchoDetector> echo_detector_;
  std::unique_ptr<CustomAudioAnalyzer> capture_analyzer_;
  RTC_DISALLOW_COPY_AND_ASSIGN(AudioProcessingBuilder);
};

class StreamConfig {
 public:
  // sample_rate_hz: The sampling rate of the stream.
  //
  // num_channels: The number of audio channels in the stream, excluding the
  //               keyboard channel if it is present. When passing a
  //               StreamConfig with an array of arrays T*[N],
  //
  //                N == {num_channels + 1  if  has_keyboard
  //                     {num_channels      if  !has_keyboard
  //
  // has_keyboard: True if the stream has a keyboard channel. When has_keyboard
  //               is true, the last channel in any corresponding list of
  //               channels is the keyboard channel.
  StreamConfig(int sample_rate_hz = 0,
               size_t num_channels = 0,
               bool has_keyboard = false)
      : sample_rate_hz_(sample_rate_hz),
        num_channels_(num_channels),
        has_keyboard_(has_keyboard),
        num_frames_(calculate_frames(sample_rate_hz)) {}

  void set_sample_rate_hz(int value) {
    sample_rate_hz_ = value;
    num_frames_ = calculate_frames(value);
  }
  void set_num_channels(size_t value) { num_channels_ = value; }
  void set_has_keyboard(bool value) { has_keyboard_ = value; }

  int sample_rate_hz() const { return sample_rate_hz_; }

  // The number of channels in the stream, not including the keyboard channel if
  // present.
  size_t num_channels() const { return num_channels_; }

  bool has_keyboard() const { return has_keyboard_; }
  size_t num_frames() const { return num_frames_; }
  size_t num_samples() const { return num_channels_ * num_frames_; }

  bool operator==(const StreamConfig& other) const {
    return sample_rate_hz_ == other.sample_rate_hz_ &&
           num_channels_ == other.num_channels_ &&
           has_keyboard_ == other.has_keyboard_;
  }

  bool operator!=(const StreamConfig& other) const { return !(*this == other); }

 private:
  static size_t calculate_frames(int sample_rate_hz) {
    return static_cast<size_t>(AudioProcessing::kChunkSizeMs * sample_rate_hz /
                               1000);
  }

  int sample_rate_hz_;
  size_t num_channels_;
  bool has_keyboard_;
  size_t num_frames_;
};

class ProcessingConfig {
 public:
  enum StreamName {
    kInputStream,
    kOutputStream,
    kReverseInputStream,
    kReverseOutputStream,
    kNumStreamNames,
  };

  const StreamConfig& input_stream() const {
    return streams[StreamName::kInputStream];
  }
  const StreamConfig& output_stream() const {
    return streams[StreamName::kOutputStream];
  }
  const StreamConfig& reverse_input_stream() const {
    return streams[StreamName::kReverseInputStream];
  }
  const StreamConfig& reverse_output_stream() const {
    return streams[StreamName::kReverseOutputStream];
  }

  StreamConfig& input_stream() { return streams[StreamName::kInputStream]; }
  StreamConfig& output_stream() { return streams[StreamName::kOutputStream]; }
  StreamConfig& reverse_input_stream() {
    return streams[StreamName::kReverseInputStream];
  }
  StreamConfig& reverse_output_stream() {
    return streams[StreamName::kReverseOutputStream];
  }

  bool operator==(const ProcessingConfig& other) const {
    for (int i = 0; i < StreamName::kNumStreamNames; ++i) {
      if (this->streams[i] != other.streams[i]) {
        return false;
      }
    }
    return true;
  }

  bool operator!=(const ProcessingConfig& other) const {
    return !(*this == other);
  }

  StreamConfig streams[StreamName::kNumStreamNames];
};

// Experimental interface for a custom analysis submodule.
class CustomAudioAnalyzer {
 public:
  // (Re-) Initializes the submodule.
  virtual void Initialize(int sample_rate_hz, int num_channels) = 0;
  // Analyzes the given capture or render signal.
  virtual void Analyze(const AudioBuffer* audio) = 0;
  // Returns a string representation of the module state.
  virtual std::string ToString() const = 0;

  virtual ~CustomAudioAnalyzer() {}
};

// Interface for a custom processing submodule.
class CustomProcessing {
 public:
  // (Re-)Initializes the submodule.
  virtual void Initialize(int sample_rate_hz, int num_channels) = 0;
  // Processes the given capture or render signal.
  virtual void Process(AudioBuffer* audio) = 0;
  // Returns a string representation of the module state.
  virtual std::string ToString() const = 0;
  // Handles RuntimeSettings. TODO(webrtc:9262): make pure virtual
  // after updating dependencies.
  virtual void SetRuntimeSetting(AudioProcessing::RuntimeSetting setting);

  virtual ~CustomProcessing() {}
};

// Interface for an echo detector submodule.
class EchoDetector : public rtc::RefCountInterface {
 public:
  // (Re-)Initializes the submodule.
  virtual void Initialize(int capture_sample_rate_hz,
                          int num_capture_channels,
                          int render_sample_rate_hz,
                          int num_render_channels) = 0;

  // Analysis (not changing) of the render signal.
  virtual void AnalyzeRenderAudio(rtc::ArrayView<const float> render_audio) = 0;

  // Analysis (not changing) of the capture signal.
  virtual void AnalyzeCaptureAudio(
      rtc::ArrayView<const float> capture_audio) = 0;

  // Pack an AudioBuffer into a vector<float>.
  static void PackRenderAudioBuffer(AudioBuffer* audio,
                                    std::vector<float>* packed_buffer);

  struct Metrics {
    absl::optional<double> echo_likelihood;
    absl::optional<double> echo_likelihood_recent_max;
  };

  // Collect current metrics from the echo detector.
  virtual Metrics GetMetrics() const = 0;
};

}  // namespace webrtc

#endif  // MODULES_AUDIO_PROCESSING_INCLUDE_AUDIO_PROCESSING_H_