webrtc/modules/audio_mixer/audio_mixer_impl.cc

/*
 *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
 *
 *  Use of this source code is governed by a BSD-style license
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
 *  in the file PATENTS.  All contributing project authors may
 *  be found in the AUTHORS file in the root of the source tree.
 */

#include "webrtc/modules/audio_mixer/audio_mixer_impl.h"

#include <algorithm>
#include <functional>

#include "webrtc/modules/audio_mixer/audio_frame_manipulator.h"
#include "webrtc/modules/audio_mixer/audio_mixer_defines.h"
#include "webrtc/modules/audio_processing/include/audio_processing.h"
#include "webrtc/modules/utility/include/audio_frame_operations.h"
#include "webrtc/system_wrappers/include/critical_section_wrapper.h"
#include "webrtc/system_wrappers/include/trace.h"

namespace webrtc {
namespace {

class SourceFrame {
 public:
  SourceFrame(MixerAudioSource* p, AudioFrame* a, bool m, bool was_mixed_before)
      : audio_source_(p),
        audio_frame_(a),
        muted_(m),
        was_mixed_before_(was_mixed_before) {
    if (!muted_) {
      energy_ = NewMixerCalculateEnergy(*a);
    }
  }

  // a.shouldMixBefore(b) is used to select mixer participants.
  bool shouldMixBefore(const SourceFrame& other) const {
    if (muted_ != other.muted_) {
      return other.muted_;
    }

    auto our_activity = audio_frame_->vad_activity_;
    auto other_activity = other.audio_frame_->vad_activity_;

    if (our_activity != other_activity) {
      return our_activity == AudioFrame::kVadActive;
    }

    return energy_ > other.energy_;
  }

  MixerAudioSource* audio_source_;
  AudioFrame* audio_frame_;
  bool muted_;
  uint32_t energy_;
  bool was_mixed_before_;
};

// Remixes a frame between stereo and mono.
void RemixFrame(AudioFrame* frame, size_t number_of_channels) {
  RTC_DCHECK(number_of_channels == 1 || number_of_channels == 2);
  if (frame->num_channels_ == 1 && number_of_channels == 2) {
    AudioFrameOperations::MonoToStereo(frame);
  } else if (frame->num_channels_ == 2 && number_of_channels == 1) {
    AudioFrameOperations::StereoToMono(frame);
  }
}

// Mix |frame| into |mixed_frame|, with saturation protection and upmixing.
// These effects are applied to |frame| itself prior to mixing. Assumes that
// |mixed_frame| always has at least as many channels as |frame|. Supports
// stereo at most.
//
void MixFrames(AudioFrame* mixed_frame, AudioFrame* frame, bool use_limiter) {
  RTC_DCHECK_GE(mixed_frame->num_channels_, frame->num_channels_);
  if (use_limiter) {
    // Divide by two to avoid saturation in the mixing.
    // This is only meaningful if the limiter will be used.
    *frame >>= 1;
  }
  RTC_DCHECK_EQ(frame->num_channels_, mixed_frame->num_channels_);
  *mixed_frame += *frame;
}

}  // namespace

MixerAudioSource::MixerAudioSource() : _mixHistory(new NewMixHistory()) {}

MixerAudioSource::~MixerAudioSource() {
  delete _mixHistory;
}

bool MixerAudioSource::IsMixed() const {
  return _mixHistory->IsMixed();
}

NewMixHistory::NewMixHistory() : is_mixed_(0) {}

NewMixHistory::~NewMixHistory() {}

bool NewMixHistory::IsMixed() const {
  return is_mixed_;
}

bool NewMixHistory::WasMixed() const {
  // Was mixed is the same as is mixed depending on perspective. This function
  // is for the perspective of NewAudioConferenceMixerImpl.
  return IsMixed();
}

int32_t NewMixHistory::SetIsMixed(const bool mixed) {
  is_mixed_ = mixed;
  return 0;
}

void NewMixHistory::ResetMixedStatus() {
  is_mixed_ = false;
}

std::unique_ptr<AudioMixer> AudioMixer::Create(int id) {
  AudioMixerImpl* mixer = new AudioMixerImpl(id);
  if (!mixer->Init()) {
    delete mixer;
    return NULL;
  }
  return std::unique_ptr<AudioMixer>(mixer);
}

AudioMixerImpl::AudioMixerImpl(int id)
    : id_(id),
      output_frequency_(kDefaultFrequency),
      sample_size_(0),
      audio_source_list_(),
      additional_audio_source_list_(),
      num_mixed_audio_sources_(0),
      use_limiter_(true),
      time_stamp_(0) {
  thread_checker_.DetachFromThread();
}

AudioMixerImpl::~AudioMixerImpl() {}

bool AudioMixerImpl::Init() {
  crit_.reset(CriticalSectionWrapper::CreateCriticalSection());
  if (crit_.get() == NULL)
    return false;

  cb_crit_.reset(CriticalSectionWrapper::CreateCriticalSection());
  if (cb_crit_.get() == NULL)
    return false;

  Config config;
  config.Set<ExperimentalAgc>(new ExperimentalAgc(false));
  limiter_.reset(AudioProcessing::Create(config));
  if (!limiter_.get())
    return false;

  if (SetOutputFrequency(kDefaultFrequency) == -1)
    return false;

  if (limiter_->gain_control()->set_mode(GainControl::kFixedDigital) !=
      limiter_->kNoError)
    return false;

  // We smoothly limit the mixed frame to -7 dbFS. -6 would correspond to the
  // divide-by-2 but -7 is used instead to give a bit of headroom since the
  // AGC is not a hard limiter.
  if (limiter_->gain_control()->set_target_level_dbfs(7) != limiter_->kNoError)
    return false;

  if (limiter_->gain_control()->set_compression_gain_db(0) !=
      limiter_->kNoError)
    return false;

  if (limiter_->gain_control()->enable_limiter(true) != limiter_->kNoError)
    return false;

  if (limiter_->gain_control()->Enable(true) != limiter_->kNoError)
    return false;

  return true;
}

void AudioMixerImpl::Mix(int sample_rate,
                         size_t number_of_channels,
                         AudioFrame* audio_frame_for_mixing) {
  RTC_DCHECK(number_of_channels == 1 || number_of_channels == 2);
  RTC_DCHECK(thread_checker_.CalledOnValidThread());
  AudioFrameList mixList;
  AudioFrameList additionalFramesList;
  std::map<int, MixerAudioSource*> mixedAudioSourcesMap;
  {
    CriticalSectionScoped cs(cb_crit_.get());
    Frequency mixing_frequency;

    switch (sample_rate) {
      case 8000:
        mixing_frequency = kNbInHz;
        break;
      case 16000:
        mixing_frequency = kWbInHz;
        break;
      case 32000:
        mixing_frequency = kSwbInHz;
        break;
      case 48000:
        mixing_frequency = kFbInHz;
        break;
      default:
        RTC_NOTREACHED();
        return;
    }

    if (OutputFrequency() != mixing_frequency) {
      SetOutputFrequency(mixing_frequency);
    }

    mixList = UpdateToMix(kMaximumAmountOfMixedAudioSources);
    GetAdditionalAudio(&additionalFramesList);
  }

  for (FrameAndMuteInfo& frame_and_mute : mixList) {
    RemixFrame(frame_and_mute.frame, number_of_channels);
  }
  for (FrameAndMuteInfo& frame_and_mute : additionalFramesList) {
    RemixFrame(frame_and_mute.frame, number_of_channels);
  }

  audio_frame_for_mixing->UpdateFrame(
      -1, time_stamp_, NULL, 0, output_frequency_, AudioFrame::kNormalSpeech,
      AudioFrame::kVadPassive, number_of_channels);

  time_stamp_ += static_cast<uint32_t>(sample_size_);

  use_limiter_ = num_mixed_audio_sources_ > 1;

  // We only use the limiter if it supports the output sample rate and
  // we're actually mixing multiple streams.
  MixFromList(audio_frame_for_mixing, mixList, id_, use_limiter_);

  {
    CriticalSectionScoped cs(crit_.get());
    MixAnonomouslyFromList(audio_frame_for_mixing, additionalFramesList);

    if (audio_frame_for_mixing->samples_per_channel_ == 0) {
      // Nothing was mixed, set the audio samples to silence.
      audio_frame_for_mixing->samples_per_channel_ = sample_size_;
      audio_frame_for_mixing->Mute();
    } else {
      // Only call the limiter if we have something to mix.
      LimitMixedAudio(audio_frame_for_mixing);
    }
  }

  // Pass the final result to the level indicator.
  audio_level_.ComputeLevel(*audio_frame_for_mixing);

  return;
}

int32_t AudioMixerImpl::SetOutputFrequency(const Frequency& frequency) {
  CriticalSectionScoped cs(crit_.get());

  output_frequency_ = frequency;
  sample_size_ =
      static_cast<size_t>((output_frequency_ * kFrameDurationInMs) / 1000);

  return 0;
}

AudioMixer::Frequency AudioMixerImpl::OutputFrequency() const {
  CriticalSectionScoped cs(crit_.get());
  return output_frequency_;
}

int32_t AudioMixerImpl::SetMixabilityStatus(MixerAudioSource* audio_source,
                                            bool mixable) {
  if (!mixable) {
    // Anonymous audio sources are in a separate list. Make sure that the
    // audio source is in the _audioSourceList if it is being mixed.
    SetAnonymousMixabilityStatus(audio_source, false);
  }
  size_t numMixedAudioSources;
  {
    CriticalSectionScoped cs(cb_crit_.get());
    const bool isMixed = IsAudioSourceInList(*audio_source, audio_source_list_);
    // API must be called with a new state.
    if (!(mixable ^ isMixed)) {
      WEBRTC_TRACE(kTraceWarning, kTraceAudioMixerServer, id_,
                   "Mixable is aready %s", isMixed ? "ON" : "off");
      return -1;
    }
    bool success = false;
    if (mixable) {
      success = AddAudioSourceToList(audio_source, &audio_source_list_);
    } else {
      success = RemoveAudioSourceFromList(audio_source, &audio_source_list_);
    }
    if (!success) {
      WEBRTC_TRACE(kTraceError, kTraceAudioMixerServer, id_,
                   "failed to %s audio_source", mixable ? "add" : "remove");
      RTC_NOTREACHED();
      return -1;
    }

    size_t numMixedNonAnonymous = audio_source_list_.size();
    if (numMixedNonAnonymous > kMaximumAmountOfMixedAudioSources) {
      numMixedNonAnonymous = kMaximumAmountOfMixedAudioSources;
    }
    numMixedAudioSources =
        numMixedNonAnonymous + additional_audio_source_list_.size();
  }
  // A MixerAudioSource was added or removed. Make sure the scratch
  // buffer is updated if necessary.
  // Note: The scratch buffer may only be updated in Process().
  CriticalSectionScoped cs(crit_.get());
  num_mixed_audio_sources_ = numMixedAudioSources;
  return 0;
}

bool AudioMixerImpl::MixabilityStatus(
    const MixerAudioSource& audio_source) const {
  CriticalSectionScoped cs(cb_crit_.get());
  return IsAudioSourceInList(audio_source, audio_source_list_);
}

int32_t AudioMixerImpl::SetAnonymousMixabilityStatus(
    MixerAudioSource* audio_source,
    bool anonymous) {
  CriticalSectionScoped cs(cb_crit_.get());
  if (IsAudioSourceInList(*audio_source, additional_audio_source_list_)) {
    if (anonymous) {
      return 0;
    }
    if (!RemoveAudioSourceFromList(audio_source,
                                   &additional_audio_source_list_)) {
      WEBRTC_TRACE(kTraceError, kTraceAudioMixerServer, id_,
                   "unable to remove audio_source from anonymous list");
      RTC_NOTREACHED();
      return -1;
    }
    return AddAudioSourceToList(audio_source, &audio_source_list_) ? 0 : -1;
  }
  if (!anonymous) {
    return 0;
  }
  const bool mixable =
      RemoveAudioSourceFromList(audio_source, &audio_source_list_);
  if (!mixable) {
    WEBRTC_TRACE(
        kTraceWarning, kTraceAudioMixerServer, id_,
        "audio_source must be registered before turning it into anonymous");
    // Setting anonymous status is only possible if MixerAudioSource is
    // already registered.
    return -1;
  }
  return AddAudioSourceToList(audio_source, &additional_audio_source_list_)
             ? 0
             : -1;
}

bool AudioMixerImpl::AnonymousMixabilityStatus(
    const MixerAudioSource& audio_source) const {
  CriticalSectionScoped cs(cb_crit_.get());
  return IsAudioSourceInList(audio_source, additional_audio_source_list_);
}

AudioFrameList AudioMixerImpl::UpdateToMix(size_t maxAudioFrameCounter) const {
  AudioFrameList result;
  std::vector<SourceFrame> audioSourceMixingDataList;

  // Get audio source audio and put it in the struct vector.
  for (MixerAudioSource* audio_source : audio_source_list_) {
    auto audio_frame_with_info = audio_source->GetAudioFrameWithMuted(
        id_, static_cast<int>(output_frequency_));

    auto audio_frame_info = audio_frame_with_info.audio_frame_info;
    AudioFrame* audio_source_audio_frame = audio_frame_with_info.audio_frame;

    if (audio_frame_info == MixerAudioSource::AudioFrameInfo::kError) {
      WEBRTC_TRACE(kTraceWarning, kTraceAudioMixerServer, id_,
                   "failed to GetAudioFrameWithMuted() from participant");
      continue;
    }
    audioSourceMixingDataList.emplace_back(
        audio_source, audio_source_audio_frame,
        audio_frame_info == MixerAudioSource::AudioFrameInfo::kMuted,
        audio_source->_mixHistory->WasMixed());
  }

  // Sort frames by sorting function.
  std::sort(audioSourceMixingDataList.begin(), audioSourceMixingDataList.end(),
            std::mem_fn(&SourceFrame::shouldMixBefore));

  // Go through list in order and put things in mixList.
  for (SourceFrame& p : audioSourceMixingDataList) {
    // Filter muted.
    if (p.muted_) {
      p.audio_source_->_mixHistory->SetIsMixed(false);
      continue;
    }

    // Add frame to result vector for mixing.
    bool is_mixed = false;
    if (maxAudioFrameCounter > 0) {
      --maxAudioFrameCounter;
      if (!p.was_mixed_before_) {
        NewMixerRampIn(p.audio_frame_);
      }
      result.emplace_back(p.audio_frame_, false);
      is_mixed = true;
    }

    // Ramp out unmuted.
    if (p.was_mixed_before_ && !is_mixed) {
      NewMixerRampOut(p.audio_frame_);
      result.emplace_back(p.audio_frame_, false);
    }

    p.audio_source_->_mixHistory->SetIsMixed(is_mixed);
  }
  return result;
}

void AudioMixerImpl::GetAdditionalAudio(
    AudioFrameList* additionalFramesList) const {
  WEBRTC_TRACE(kTraceStream, kTraceAudioMixerServer, id_,
               "GetAdditionalAudio(additionalFramesList)");
  // The GetAudioFrameWithMuted() callback may result in the audio source being
  // removed from additionalAudioFramesList_. If that happens it will
  // invalidate any iterators. Create a copy of the audio sources list such
  // that the list of participants can be traversed safely.
  MixerAudioSourceList additionalAudioSourceList;
  additionalAudioSourceList.insert(additionalAudioSourceList.begin(),
                                   additional_audio_source_list_.begin(),
                                   additional_audio_source_list_.end());

  for (MixerAudioSourceList::const_iterator audio_source =
           additionalAudioSourceList.begin();
       audio_source != additionalAudioSourceList.end(); ++audio_source) {
    auto audio_frame_with_info =
        (*audio_source)->GetAudioFrameWithMuted(id_, output_frequency_);
    auto ret = audio_frame_with_info.audio_frame_info;
    AudioFrame* audio_frame = audio_frame_with_info.audio_frame;
    if (ret == MixerAudioSource::AudioFrameInfo::kError) {
      WEBRTC_TRACE(kTraceWarning, kTraceAudioMixerServer, id_,
                   "failed to GetAudioFrameWithMuted() from audio_source");
      continue;
    }
    if (audio_frame->samples_per_channel_ == 0) {
      // Empty frame. Don't use it.
      continue;
    }
    additionalFramesList->push_back(FrameAndMuteInfo(
        audio_frame, ret == MixerAudioSource::AudioFrameInfo::kMuted));
  }
}

bool AudioMixerImpl::IsAudioSourceInList(
    const MixerAudioSource& audio_source,
    const MixerAudioSourceList& audioSourceList) const {
  WEBRTC_TRACE(kTraceStream, kTraceAudioMixerServer, id_,
               "IsAudioSourceInList(audio_source,audioSourceList)");
  return std::find(audioSourceList.begin(), audioSourceList.end(),
                   &audio_source) != audioSourceList.end();
}

bool AudioMixerImpl::AddAudioSourceToList(
    MixerAudioSource* audio_source,
    MixerAudioSourceList* audioSourceList) const {
  WEBRTC_TRACE(kTraceStream, kTraceAudioMixerServer, id_,
               "AddAudioSourceToList(audio_source, audioSourceList)");
  audioSourceList->push_back(audio_source);
  // Make sure that the mixed status is correct for new MixerAudioSource.
  audio_source->_mixHistory->ResetMixedStatus();
  return true;
}

bool AudioMixerImpl::RemoveAudioSourceFromList(
    MixerAudioSource* audio_source,
    MixerAudioSourceList* audioSourceList) const {
  WEBRTC_TRACE(kTraceStream, kTraceAudioMixerServer, id_,
               "RemoveAudioSourceFromList(audio_source, audioSourceList)");
  auto iter =
      std::find(audioSourceList->begin(), audioSourceList->end(), audio_source);
  if (iter != audioSourceList->end()) {
    audioSourceList->erase(iter);
    // AudioSource is no longer mixed, reset to default.
    audio_source->_mixHistory->ResetMixedStatus();
    return true;
  } else {
    return false;
  }
}

int32_t AudioMixerImpl::MixFromList(AudioFrame* mixedAudio,
                                    const AudioFrameList& audioFrameList,
                                    int32_t id,
                                    bool use_limiter) {
  WEBRTC_TRACE(kTraceStream, kTraceAudioMixerServer, id,
               "MixFromList(mixedAudio, audioFrameList)");
  if (audioFrameList.empty())
    return 0;

  uint32_t position = 0;

  if (audioFrameList.size() == 1) {
    mixedAudio->timestamp_ = audioFrameList.front().frame->timestamp_;
    mixedAudio->elapsed_time_ms_ =
        audioFrameList.front().frame->elapsed_time_ms_;
  } else {
    // TODO(wu): Issue 3390.
    // Audio frame timestamp is only supported in one channel case.
    mixedAudio->timestamp_ = 0;
    mixedAudio->elapsed_time_ms_ = -1;
  }

  for (AudioFrameList::const_iterator iter = audioFrameList.begin();
       iter != audioFrameList.end(); ++iter) {
    if (!iter->muted) {
      MixFrames(mixedAudio, iter->frame, use_limiter);
    }

    position++;
  }

  return 0;
}

// TODO(andrew): consolidate this function with MixFromList.
int32_t AudioMixerImpl::MixAnonomouslyFromList(
    AudioFrame* mixedAudio,
    const AudioFrameList& audioFrameList) const {
  WEBRTC_TRACE(kTraceStream, kTraceAudioMixerServer, id_,
               "MixAnonomouslyFromList(mixedAudio, audioFrameList)");

  if (audioFrameList.empty())
    return 0;

  for (AudioFrameList::const_iterator iter = audioFrameList.begin();
       iter != audioFrameList.end(); ++iter) {
    if (!iter->muted) {
      MixFrames(mixedAudio, iter->frame, use_limiter_);
    }
  }
  return 0;
}

bool AudioMixerImpl::LimitMixedAudio(AudioFrame* mixedAudio) const {
  if (!use_limiter_) {
    return true;
  }

  // Smoothly limit the mixed frame.
  const int error = limiter_->ProcessStream(mixedAudio);

  // And now we can safely restore the level. This procedure results in
  // some loss of resolution, deemed acceptable.
  //
  // It's possible to apply the gain in the AGC (with a target level of 0 dbFS
  // and compression gain of 6 dB). However, in the transition frame when this
  // is enabled (moving from one to two audio sources) it has the potential to
  // create discontinuities in the mixed frame.
  //
  // Instead we double the frame (with addition since left-shifting a
  // negative value is undefined).
  *mixedAudio += *mixedAudio;

  if (error != limiter_->kNoError) {
    WEBRTC_TRACE(kTraceError, kTraceAudioMixerServer, id_,
                 "Error from AudioProcessing: %d", error);
    RTC_NOTREACHED();
    return false;
  }
  return true;
}

int AudioMixerImpl::GetOutputAudioLevel() {
  const int level = audio_level_.Level();
  WEBRTC_TRACE(kTraceStateInfo, kTraceAudioMixerServer, id_,
               "GetAudioOutputLevel() => level=%d", level);
  return level;
}

int AudioMixerImpl::GetOutputAudioLevelFullRange() {
  const int level = audio_level_.LevelFullRange();
  WEBRTC_TRACE(kTraceStateInfo, kTraceAudioMixerServer, id_,
               "GetAudioOutputLevelFullRange() => level=%d", level);
  return level;
}
}  // namespace webrtc