Moves channel-dependent audio input processing to separate encoder task queue.
First approach to remove parts of the heavy load done for encoding, and
preparation for sending, from native audio thread to separate task queue.
With this change we will give the native input audio thread more time to
"relax" between successive audio captures.
Separate profiling done on Android has verified that the change works well;
the load is now redistributed and the load of the native AudioRecordThread
is reduced. Similar conclusions should be valid for all other OS:es as well.
BUG=NONE
CQ_INCLUDE_TRYBOTS=master.tryserver.chromium.android:android_compile_dbg,linux_android_rel_ng
Review-Url: https://codereview.webrtc.org/2665693002
Cr-Commit-Position: refs/heads/master@{#17488}
diff --git a/webrtc/voice_engine/BUILD.gn b/webrtc/voice_engine/BUILD.gn
index cb604c6..198e62c 100644
--- a/webrtc/voice_engine/BUILD.gn
+++ b/webrtc/voice_engine/BUILD.gn
@@ -137,6 +137,7 @@
"../api/audio_codecs:builtin_audio_decoder_factory",
"../audio/utility:audio_frame_operations",
"../base:rtc_base_approved",
+ "../base:rtc_task_queue",
# TODO(nisse): Delete when declaration of RtpTransportController
# and related interfaces move to api/.
diff --git a/webrtc/voice_engine/channel.cc b/webrtc/voice_engine/channel.cc
index 1ab6971..6ff3685 100644
--- a/webrtc/voice_engine/channel.cc
+++ b/webrtc/voice_engine/channel.cc
@@ -21,6 +21,8 @@
#include "webrtc/base/location.h"
#include "webrtc/base/logging.h"
#include "webrtc/base/rate_limiter.h"
+#include "webrtc/base/task_queue.h"
+#include "webrtc/base/thread_checker.h"
#include "webrtc/base/timeutils.h"
#include "webrtc/call/rtp_transport_controller_send.h"
#include "webrtc/config.h"
@@ -409,12 +411,32 @@
RtcpBandwidthObserver* bandwidth_observer_ GUARDED_BY(crit_);
};
+class Channel::ProcessAndEncodeAudioTask : public rtc::QueuedTask {
+ public:
+ ProcessAndEncodeAudioTask(std::unique_ptr<AudioFrame> audio_frame,
+ Channel* channel)
+ : audio_frame_(std::move(audio_frame)), channel_(channel) {
+ RTC_DCHECK(channel_);
+ }
+
+ private:
+ bool Run() override {
+ RTC_DCHECK_RUN_ON(channel_->encoder_queue_);
+ channel_->ProcessAndEncodeAudioOnTaskQueue(audio_frame_.get());
+ return true;
+ }
+
+ std::unique_ptr<AudioFrame> audio_frame_;
+ Channel* const channel_;
+};
+
int32_t Channel::SendData(FrameType frameType,
uint8_t payloadType,
uint32_t timeStamp,
const uint8_t* payloadData,
size_t payloadSize,
const RTPFragmentationHeader* fragmentation) {
+ RTC_DCHECK_RUN_ON(encoder_queue_);
WEBRTC_TRACE(kTraceStream, kTraceVoice, VoEId(_instanceId, _channelId),
"Channel::SendData(frameType=%u, payloadType=%u, timeStamp=%u,"
" payloadSize=%" PRIuS ", fragmentation=0x%x)",
@@ -442,9 +464,6 @@
return -1;
}
- _lastLocalTimeStamp = timeStamp;
- _lastPayloadType = payloadType;
-
return 0;
}
@@ -779,11 +798,10 @@
return (highestNeeded);
}
-int32_t Channel::CreateChannel(
- Channel*& channel,
- int32_t channelId,
- uint32_t instanceId,
- const VoEBase::ChannelConfig& config) {
+int32_t Channel::CreateChannel(Channel*& channel,
+ int32_t channelId,
+ uint32_t instanceId,
+ const VoEBase::ChannelConfig& config) {
WEBRTC_TRACE(kTraceMemory, kTraceVoice, VoEId(instanceId, channelId),
"Channel::CreateChannel(channelId=%d, instanceId=%d)", channelId,
instanceId);
@@ -890,8 +908,6 @@
previous_frame_muted_(false),
_outputGain(1.0f),
_mixFileWithMicrophone(false),
- _lastLocalTimeStamp(0),
- _lastPayloadType(0),
_includeAudioLevelIndication(false),
transport_overhead_per_packet_(0),
rtp_overhead_per_packet_(0),
@@ -1125,7 +1141,10 @@
ProcessThread& moduleProcessThread,
AudioDeviceModule& audioDeviceModule,
VoiceEngineObserver* voiceEngineObserver,
- rtc::CriticalSection* callbackCritSect) {
+ rtc::CriticalSection* callbackCritSect,
+ rtc::TaskQueue* encoder_queue) {
+ RTC_DCHECK(encoder_queue);
+ RTC_DCHECK(!encoder_queue_);
WEBRTC_TRACE(kTraceInfo, kTraceVoice, VoEId(_instanceId, _channelId),
"Channel::SetEngineInformation()");
_engineStatisticsPtr = &engineStatistics;
@@ -1134,11 +1153,7 @@
_audioDeviceModulePtr = &audioDeviceModule;
_voiceEngineObserverPtr = voiceEngineObserver;
_callbackCritSectPtr = callbackCritSect;
- return 0;
-}
-
-int32_t Channel::UpdateLocalTimeStamp() {
- _timeStamp += static_cast<uint32_t>(_audioFrame.samples_per_channel_);
+ encoder_queue_ = encoder_queue;
return 0;
}
@@ -1222,14 +1237,25 @@
return 0;
}
-int32_t Channel::StopSend() {
+void Channel::StopSend() {
WEBRTC_TRACE(kTraceInfo, kTraceVoice, VoEId(_instanceId, _channelId),
"Channel::StopSend()");
if (!channel_state_.Get().sending) {
- return 0;
+ return;
}
channel_state_.SetSending(false);
+ // Post a task to the encoder thread which sets an event when the task is
+ // executed. We know that no more encoding tasks will be added to the task
+ // queue for this channel since sending is now deactivated. It means that,
+ // if we wait for the event to bet set, we know that no more pending tasks
+ // exists and it is therfore guaranteed that the task queue will never try
+ // to acccess and invalid channel object.
+ RTC_DCHECK(encoder_queue_);
+ rtc::Event flush(false, false);
+ encoder_queue_->PostTask([&flush]() { flush.Set(); });
+ flush.Wait(rtc::Event::kForever);
+
// Store the sequence number to be able to pick up the same sequence for
// the next StartSend(). This is needed for restarting device, otherwise
// it might cause libSRTP to complain about packets being replayed.
@@ -1246,8 +1272,6 @@
"StartSend() RTP/RTCP failed to stop sending");
}
_rtpRtcpModule->SetSendingMediaStatus(false);
-
- return 0;
}
int32_t Channel::RegisterVoiceEngineObserver(VoiceEngineObserver& observer) {
@@ -2648,90 +2672,73 @@
return _rtpRtcpModule->SendNACK(sequence_numbers, length);
}
-uint32_t Channel::Demultiplex(const AudioFrame& audioFrame) {
- WEBRTC_TRACE(kTraceStream, kTraceVoice, VoEId(_instanceId, _channelId),
- "Channel::Demultiplex()");
- _audioFrame.CopyFrom(audioFrame);
- _audioFrame.id_ = _channelId;
- return 0;
+void Channel::ProcessAndEncodeAudio(const AudioFrame& audio_input) {
+ RTC_DCHECK(channel_state_.Get().sending);
+ std::unique_ptr<AudioFrame> audio_frame(new AudioFrame());
+ // TODO(henrika): try to avoid copying by moving ownership of audio frame
+ // either into pool of frames or into the task itself.
+ audio_frame->CopyFrom(audio_input);
+ audio_frame->id_ = ChannelId();
+ encoder_queue_->PostTask(std::unique_ptr<rtc::QueuedTask>(
+ new ProcessAndEncodeAudioTask(std::move(audio_frame), this)));
}
-void Channel::Demultiplex(const int16_t* audio_data,
- int sample_rate,
- size_t number_of_frames,
- size_t number_of_channels) {
+void Channel::ProcessAndEncodeAudio(const int16_t* audio_data,
+ int sample_rate,
+ size_t number_of_frames,
+ size_t number_of_channels) {
+ RTC_DCHECK(channel_state_.Get().sending);
CodecInst codec;
GetSendCodec(codec);
-
- // Never upsample or upmix the capture signal here. This should be done at the
- // end of the send chain.
- _audioFrame.sample_rate_hz_ = std::min(codec.plfreq, sample_rate);
- _audioFrame.num_channels_ = std::min(number_of_channels, codec.channels);
+ std::unique_ptr<AudioFrame> audio_frame(new AudioFrame());
+ audio_frame->id_ = ChannelId();
+ audio_frame->sample_rate_hz_ = std::min(codec.plfreq, sample_rate);
+ audio_frame->num_channels_ = std::min(number_of_channels, codec.channels);
RemixAndResample(audio_data, number_of_frames, number_of_channels,
- sample_rate, &input_resampler_, &_audioFrame);
+ sample_rate, &input_resampler_, audio_frame.get());
+ encoder_queue_->PostTask(std::unique_ptr<rtc::QueuedTask>(
+ new ProcessAndEncodeAudioTask(std::move(audio_frame), this)));
}
-uint32_t Channel::PrepareEncodeAndSend(int mixingFrequency) {
- WEBRTC_TRACE(kTraceStream, kTraceVoice, VoEId(_instanceId, _channelId),
- "Channel::PrepareEncodeAndSend()");
-
- if (_audioFrame.samples_per_channel_ == 0) {
- WEBRTC_TRACE(kTraceWarning, kTraceVoice, VoEId(_instanceId, _channelId),
- "Channel::PrepareEncodeAndSend() invalid audio frame");
- return 0xFFFFFFFF;
- }
+void Channel::ProcessAndEncodeAudioOnTaskQueue(AudioFrame* audio_input) {
+ RTC_DCHECK_RUN_ON(encoder_queue_);
+ RTC_DCHECK_GT(audio_input->samples_per_channel_, 0);
+ RTC_DCHECK_LE(audio_input->num_channels_, 2);
+ RTC_DCHECK_EQ(audio_input->id_, ChannelId());
if (channel_state_.Get().input_file_playing) {
- MixOrReplaceAudioWithFile(mixingFrequency);
+ MixOrReplaceAudioWithFile(audio_input);
}
- bool is_muted = InputMute(); // Cache locally as InputMute() takes a lock.
- AudioFrameOperations::Mute(&_audioFrame, previous_frame_muted_, is_muted);
+ bool is_muted = InputMute();
+ AudioFrameOperations::Mute(audio_input, previous_frame_muted_, is_muted);
if (_includeAudioLevelIndication) {
size_t length =
- _audioFrame.samples_per_channel_ * _audioFrame.num_channels_;
- RTC_CHECK_LE(length, sizeof(_audioFrame.data_));
+ audio_input->samples_per_channel_ * audio_input->num_channels_;
+ RTC_CHECK_LE(length, sizeof(audio_input->data_));
if (is_muted && previous_frame_muted_) {
rms_level_.AnalyzeMuted(length);
} else {
rms_level_.Analyze(
- rtc::ArrayView<const int16_t>(_audioFrame.data_, length));
+ rtc::ArrayView<const int16_t>(audio_input->data_, length));
}
}
previous_frame_muted_ = is_muted;
- return 0;
-}
-
-uint32_t Channel::EncodeAndSend() {
- WEBRTC_TRACE(kTraceStream, kTraceVoice, VoEId(_instanceId, _channelId),
- "Channel::EncodeAndSend()");
-
- assert(_audioFrame.num_channels_ <= 2);
- if (_audioFrame.samples_per_channel_ == 0) {
- WEBRTC_TRACE(kTraceWarning, kTraceVoice, VoEId(_instanceId, _channelId),
- "Channel::EncodeAndSend() invalid audio frame");
- return 0xFFFFFFFF;
- }
-
- _audioFrame.id_ = _channelId;
-
- // --- Add 10ms of raw (PCM) audio data to the encoder @ 32kHz.
+ // Add 10ms of raw (PCM) audio data to the encoder @ 32kHz.
// The ACM resamples internally.
- _audioFrame.timestamp_ = _timeStamp;
+ audio_input->timestamp_ = _timeStamp;
// This call will trigger AudioPacketizationCallback::SendData if encoding
// is done and payload is ready for packetization and transmission.
// Otherwise, it will return without invoking the callback.
- if (audio_coding_->Add10MsData((AudioFrame&)_audioFrame) < 0) {
- WEBRTC_TRACE(kTraceError, kTraceVoice, VoEId(_instanceId, _channelId),
- "Channel::EncodeAndSend() ACM encoding failed");
- return 0xFFFFFFFF;
+ if (audio_coding_->Add10MsData(*audio_input) < 0) {
+ LOG(LS_ERROR) << "ACM::Add10MsData() failed for channel " << _channelId;
+ return;
}
- _timeStamp += static_cast<uint32_t>(_audioFrame.samples_per_channel_);
- return 0;
+ _timeStamp += static_cast<uint32_t>(audio_input->samples_per_channel_);
}
void Channel::set_associate_send_channel(const ChannelOwner& channel) {
@@ -2840,10 +2847,11 @@
// TODO(andrew): refactor Mix functions here and in transmit_mixer.cc to use
// a shared helper.
-int32_t Channel::MixOrReplaceAudioWithFile(int mixingFrequency) {
+int32_t Channel::MixOrReplaceAudioWithFile(AudioFrame* audio_input) {
+ RTC_DCHECK_RUN_ON(encoder_queue_);
std::unique_ptr<int16_t[]> fileBuffer(new int16_t[640]);
size_t fileSamples(0);
-
+ const int mixingFrequency = audio_input->sample_rate_hz_;
{
rtc::CritScope cs(&_fileCritSect);
@@ -2868,18 +2876,18 @@
}
}
- assert(_audioFrame.samples_per_channel_ == fileSamples);
+ RTC_DCHECK_EQ(audio_input->samples_per_channel_, fileSamples);
if (_mixFileWithMicrophone) {
// Currently file stream is always mono.
// TODO(xians): Change the code when FilePlayer supports real stereo.
- MixWithSat(_audioFrame.data_, _audioFrame.num_channels_, fileBuffer.get(),
+ MixWithSat(audio_input->data_, audio_input->num_channels_, fileBuffer.get(),
1, fileSamples);
} else {
// Replace ACM audio with file.
// Currently file stream is always mono.
// TODO(xians): Change the code when FilePlayer supports real stereo.
- _audioFrame.UpdateFrame(
+ audio_input->UpdateFrame(
_channelId, 0xFFFFFFFF, fileBuffer.get(), fileSamples, mixingFrequency,
AudioFrame::kNormalSpeech, AudioFrame::kVadUnknown, 1);
}
diff --git a/webrtc/voice_engine/channel.h b/webrtc/voice_engine/channel.h
index 24a7d6b..1d2b089 100644
--- a/webrtc/voice_engine/channel.h
+++ b/webrtc/voice_engine/channel.h
@@ -16,6 +16,7 @@
#include "webrtc/api/audio/audio_mixer.h"
#include "webrtc/api/call/audio_sink.h"
#include "webrtc/base/criticalsection.h"
+#include "webrtc/base/event.h"
#include "webrtc/base/optional.h"
#include "webrtc/base/thread_checker.h"
#include "webrtc/common_audio/resampler/include/push_resampler.h"
@@ -143,11 +144,10 @@
enum { KNumSocketThreads = 1 };
enum { KNumberOfSocketBuffers = 8 };
virtual ~Channel();
- static int32_t CreateChannel(
- Channel*& channel,
- int32_t channelId,
- uint32_t instanceId,
- const VoEBase::ChannelConfig& config);
+ static int32_t CreateChannel(Channel*& channel,
+ int32_t channelId,
+ uint32_t instanceId,
+ const VoEBase::ChannelConfig& config);
Channel(int32_t channelId,
uint32_t instanceId,
const VoEBase::ChannelConfig& config);
@@ -159,8 +159,8 @@
ProcessThread& moduleProcessThread,
AudioDeviceModule& audioDeviceModule,
VoiceEngineObserver* voiceEngineObserver,
- rtc::CriticalSection* callbackCritSect);
- int32_t UpdateLocalTimeStamp();
+ rtc::CriticalSection* callbackCritSect,
+ rtc::TaskQueue* encoder_queue);
void SetSink(std::unique_ptr<AudioSinkInterface> sink);
@@ -178,7 +178,7 @@
int32_t StartPlayout();
int32_t StopPlayout();
int32_t StartSend();
- int32_t StopSend();
+ void StopSend();
int32_t RegisterVoiceEngineObserver(VoiceEngineObserver& observer);
int32_t DeRegisterVoiceEngineObserver();
@@ -354,16 +354,27 @@
}
RtpRtcp* RtpRtcpModulePtr() const { return _rtpRtcpModule.get(); }
int8_t OutputEnergyLevel() const { return _outputAudioLevel.Level(); }
- uint32_t Demultiplex(const AudioFrame& audioFrame);
- // Demultiplex the data to the channel's |_audioFrame|. The difference
- // between this method and the overloaded method above is that |audio_data|
- // does not go through transmit_mixer and APM.
- void Demultiplex(const int16_t* audio_data,
- int sample_rate,
- size_t number_of_frames,
- size_t number_of_channels);
- uint32_t PrepareEncodeAndSend(int mixingFrequency);
- uint32_t EncodeAndSend();
+
+ // ProcessAndEncodeAudio() creates an audio frame copy and posts a task
+ // on the shared encoder task queue, wich in turn calls (on the queue)
+ // ProcessAndEncodeAudioOnTaskQueue() where the actual processing of the
+ // audio takes place. The processing mainly consists of encoding and preparing
+ // the result for sending by adding it to a send queue.
+ // The main reason for using a task queue here is to release the native,
+ // OS-specific, audio capture thread as soon as possible to ensure that it
+ // can go back to sleep and be prepared to deliver an new captured audio
+ // packet.
+ void ProcessAndEncodeAudio(const AudioFrame& audio_input);
+
+ // This version of ProcessAndEncodeAudio() is used by PushCaptureData() in
+ // VoEBase and the audio in |audio_data| has not been subject to any APM
+ // processing. Some extra steps are therfore needed when building up the
+ // audio frame copy before using the same task as in the default call to
+ // ProcessAndEncodeAudio(const AudioFrame& audio_input).
+ void ProcessAndEncodeAudio(const int16_t* audio_data,
+ int sample_rate,
+ size_t number_of_frames,
+ size_t number_of_channels);
// Associate to a send channel.
// Used for obtaining RTT for a receive-only channel.
@@ -389,8 +400,9 @@
void OnRecoverableUplinkPacketLossRate(float recoverable_packet_loss_rate);
private:
- void OnUplinkPacketLossRate(float packet_loss_rate);
+ class ProcessAndEncodeAudioTask;
+ void OnUplinkPacketLossRate(float packet_loss_rate);
bool InputMute() const;
bool OnRtpPacketWithHeader(const uint8_t* received_packet,
size_t length,
@@ -405,7 +417,7 @@
bool IsPacketInOrder(const RTPHeader& header) const;
bool IsPacketRetransmitted(const RTPHeader& header, bool in_order) const;
int ResendPackets(const uint16_t* sequence_numbers, int length);
- int32_t MixOrReplaceAudioWithFile(int mixingFrequency);
+ int32_t MixOrReplaceAudioWithFile(AudioFrame* audio_frame);
int32_t MixAudioWithFile(AudioFrame& audioFrame, int mixingFrequency);
void UpdatePlayoutTimestamp(bool rtcp);
void RegisterReceiveCodecsToRTPModule();
@@ -420,11 +432,16 @@
int GetRtpTimestampRateHz() const;
int64_t GetRTT(bool allow_associate_channel) const;
+ // Called on the encoder task queue when a new input audio frame is ready
+ // for encoding.
+ void ProcessAndEncodeAudioOnTaskQueue(AudioFrame* audio_input);
+
+ uint32_t _instanceId;
+ int32_t _channelId;
+
rtc::CriticalSection _fileCritSect;
rtc::CriticalSection _callbackCritSect;
rtc::CriticalSection volume_settings_critsect_;
- uint32_t _instanceId;
- int32_t _channelId;
ChannelState channel_state_;
@@ -443,7 +460,6 @@
std::unique_ptr<AudioSinkInterface> audio_sink_;
AudioLevel _outputAudioLevel;
bool _externalTransport;
- AudioFrame _audioFrame;
// Downsamples to the codec rate if necessary.
PushResampler<int16_t> input_resampler_;
std::unique_ptr<FilePlayer> input_file_player_;
@@ -453,7 +469,7 @@
int _outputFilePlayerId;
int _outputFileRecorderId;
bool _outputFileRecording;
- uint32_t _timeStamp;
+ uint32_t _timeStamp ACCESS_ON(encoder_queue_);
RemoteNtpTimeEstimator ntp_estimator_ GUARDED_BY(ts_stats_lock_);
@@ -483,15 +499,15 @@
VoiceEngineObserver* _voiceEngineObserverPtr; // owned by base
rtc::CriticalSection* _callbackCritSectPtr; // owned by base
Transport* _transportPtr; // WebRtc socket or external transport
- RmsLevel rms_level_;
+ RmsLevel rms_level_ ACCESS_ON(encoder_queue_);
bool input_mute_ GUARDED_BY(volume_settings_critsect_);
- bool previous_frame_muted_; // Only accessed from PrepareEncodeAndSend().
+ bool previous_frame_muted_ ACCESS_ON(encoder_queue_);
float _outputGain GUARDED_BY(volume_settings_critsect_);
// VoEBase
bool _mixFileWithMicrophone;
// VoeRTP_RTCP
- uint32_t _lastLocalTimeStamp;
- int8_t _lastPayloadType;
+ // TODO(henrika): can today be accessed on the main thread and on the
+ // task queue; hence potential race.
bool _includeAudioLevelIndication;
size_t transport_overhead_per_packet_ GUARDED_BY(overhead_per_packet_lock_);
size_t rtp_overhead_per_packet_ GUARDED_BY(overhead_per_packet_lock_);
@@ -519,6 +535,8 @@
rtc::ThreadChecker construction_thread_;
const bool use_twcc_plr_for_ana_;
+
+ rtc::TaskQueue* encoder_queue_ = nullptr;
};
} // namespace voe
diff --git a/webrtc/voice_engine/shared_data.cc b/webrtc/voice_engine/shared_data.cc
index 57a1a59..e77f52a 100644
--- a/webrtc/voice_engine/shared_data.cc
+++ b/webrtc/voice_engine/shared_data.cc
@@ -27,19 +27,16 @@
_channelManager(_gInstanceCounter),
_engineStatistics(_gInstanceCounter),
_audioDevicePtr(NULL),
- _moduleProcessThreadPtr(
- ProcessThread::Create("VoiceProcessThread")) {
- Trace::CreateTrace();
- if (OutputMixer::Create(_outputMixerPtr, _gInstanceCounter) == 0)
- {
- _outputMixerPtr->SetEngineInformation(_engineStatistics);
- }
- if (TransmitMixer::Create(_transmitMixerPtr, _gInstanceCounter) == 0)
- {
- _transmitMixerPtr->SetEngineInformation(*_moduleProcessThreadPtr,
- _engineStatistics,
- _channelManager);
- }
+ _moduleProcessThreadPtr(ProcessThread::Create("VoiceProcessThread")),
+ encoder_queue_("AudioEncoderQueue") {
+ Trace::CreateTrace();
+ if (OutputMixer::Create(_outputMixerPtr, _gInstanceCounter) == 0) {
+ _outputMixerPtr->SetEngineInformation(_engineStatistics);
+ }
+ if (TransmitMixer::Create(_transmitMixerPtr, _gInstanceCounter) == 0) {
+ _transmitMixerPtr->SetEngineInformation(*_moduleProcessThreadPtr,
+ _engineStatistics, _channelManager);
+ }
}
SharedData::~SharedData()
@@ -53,6 +50,11 @@
Trace::ReturnTrace();
}
+rtc::TaskQueue* SharedData::encoder_queue() {
+ RTC_DCHECK_RUN_ON(&construction_thread_);
+ return &encoder_queue_;
+}
+
void SharedData::set_audio_device(
const rtc::scoped_refptr<AudioDeviceModule>& audio_device) {
_audioDevicePtr = audio_device;
diff --git a/webrtc/voice_engine/shared_data.h b/webrtc/voice_engine/shared_data.h
index e301419..1a91040 100644
--- a/webrtc/voice_engine/shared_data.h
+++ b/webrtc/voice_engine/shared_data.h
@@ -15,6 +15,9 @@
#include "webrtc/base/criticalsection.h"
#include "webrtc/base/scoped_ref_ptr.h"
+#include "webrtc/base/task_queue.h"
+#include "webrtc/base/thread_annotations.h"
+#include "webrtc/base/thread_checker.h"
#include "webrtc/modules/audio_device/include/audio_device.h"
#include "webrtc/modules/audio_processing/include/audio_processing.h"
#include "webrtc/modules/utility/include/process_thread.h"
@@ -46,6 +49,7 @@
OutputMixer* output_mixer() { return _outputMixerPtr; }
rtc::CriticalSection* crit_sec() { return &_apiCritPtr; }
ProcessThread* process_thread() { return _moduleProcessThreadPtr.get(); }
+ rtc::TaskQueue* encoder_queue();
int NumOfSendingChannels();
int NumOfPlayingChannels();
@@ -57,18 +61,22 @@
const char* msg) const;
protected:
- const uint32_t _instanceId;
- rtc::CriticalSection _apiCritPtr;
- ChannelManager _channelManager;
- Statistics _engineStatistics;
- rtc::scoped_refptr<AudioDeviceModule> _audioDevicePtr;
- OutputMixer* _outputMixerPtr;
- TransmitMixer* _transmitMixerPtr;
- std::unique_ptr<AudioProcessing> audioproc_;
- std::unique_ptr<ProcessThread> _moduleProcessThreadPtr;
+ rtc::ThreadChecker construction_thread_;
+ const uint32_t _instanceId;
+ rtc::CriticalSection _apiCritPtr;
+ ChannelManager _channelManager;
+ Statistics _engineStatistics;
+ rtc::scoped_refptr<AudioDeviceModule> _audioDevicePtr;
+ OutputMixer* _outputMixerPtr;
+ TransmitMixer* _transmitMixerPtr;
+ std::unique_ptr<AudioProcessing> audioproc_;
+ std::unique_ptr<ProcessThread> _moduleProcessThreadPtr;
+ // |encoder_queue| is defined last to ensure all pending tasks are cancelled
+ // and deleted before any other members.
+ rtc::TaskQueue encoder_queue_ ACCESS_ON(construction_thread_);
- SharedData();
- virtual ~SharedData();
+ SharedData();
+ virtual ~SharedData();
};
} // namespace voe
diff --git a/webrtc/voice_engine/transmit_mixer.cc b/webrtc/voice_engine/transmit_mixer.cc
index 07b47ec..e14b03f 100644
--- a/webrtc/voice_engine/transmit_mixer.cc
+++ b/webrtc/voice_engine/transmit_mixer.cc
@@ -311,66 +311,14 @@
return 0;
}
-int32_t
-TransmitMixer::DemuxAndMix()
-{
- WEBRTC_TRACE(kTraceStream, kTraceVoice, VoEId(_instanceId, -1),
- "TransmitMixer::DemuxAndMix()");
-
- for (ChannelManager::Iterator it(_channelManagerPtr); it.IsValid();
- it.Increment())
- {
- Channel* channelPtr = it.GetChannel();
- if (channelPtr->Sending())
- {
- // Demultiplex makes a copy of its input.
- channelPtr->Demultiplex(_audioFrame);
- channelPtr->PrepareEncodeAndSend(_audioFrame.sample_rate_hz_);
- }
+void TransmitMixer::ProcessAndEncodeAudio() {
+ RTC_DCHECK_GT(_audioFrame.samples_per_channel_, 0);
+ for (ChannelManager::Iterator it(_channelManagerPtr); it.IsValid();
+ it.Increment()) {
+ Channel* const channel = it.GetChannel();
+ if (channel->Sending()) {
+ channel->ProcessAndEncodeAudio(_audioFrame);
}
- return 0;
-}
-
-void TransmitMixer::DemuxAndMix(const int voe_channels[],
- size_t number_of_voe_channels) {
- for (size_t i = 0; i < number_of_voe_channels; ++i) {
- voe::ChannelOwner ch = _channelManagerPtr->GetChannel(voe_channels[i]);
- voe::Channel* channel_ptr = ch.channel();
- if (channel_ptr) {
- if (channel_ptr->Sending()) {
- // Demultiplex makes a copy of its input.
- channel_ptr->Demultiplex(_audioFrame);
- channel_ptr->PrepareEncodeAndSend(_audioFrame.sample_rate_hz_);
- }
- }
- }
-}
-
-int32_t
-TransmitMixer::EncodeAndSend()
-{
- WEBRTC_TRACE(kTraceStream, kTraceVoice, VoEId(_instanceId, -1),
- "TransmitMixer::EncodeAndSend()");
-
- for (ChannelManager::Iterator it(_channelManagerPtr); it.IsValid();
- it.Increment())
- {
- Channel* channelPtr = it.GetChannel();
- if (channelPtr->Sending())
- {
- channelPtr->EncodeAndSend();
- }
- }
- return 0;
-}
-
-void TransmitMixer::EncodeAndSend(const int voe_channels[],
- size_t number_of_voe_channels) {
- for (size_t i = 0; i < number_of_voe_channels; ++i) {
- voe::ChannelOwner ch = _channelManagerPtr->GetChannel(voe_channels[i]);
- voe::Channel* channel_ptr = ch.channel();
- if (channel_ptr && channel_ptr->Sending())
- channel_ptr->EncodeAndSend();
}
}
diff --git a/webrtc/voice_engine/transmit_mixer.h b/webrtc/voice_engine/transmit_mixer.h
index 2e728f8..a045cf8 100644
--- a/webrtc/voice_engine/transmit_mixer.h
+++ b/webrtc/voice_engine/transmit_mixer.h
@@ -32,7 +32,6 @@
#endif
namespace webrtc {
-
class AudioProcessing;
class ProcessThread;
@@ -64,16 +63,7 @@
uint16_t currentMicLevel,
bool keyPressed);
-
- int32_t DemuxAndMix();
- // Used by the Chrome to pass the recording data to the specific VoE
- // channels for demux.
- void DemuxAndMix(const int voe_channels[], size_t number_of_voe_channels);
-
- int32_t EncodeAndSend();
- // Used by the Chrome to pass the recording data to the specific VoE
- // channels for encoding and sending to the network.
- void EncodeAndSend(const int voe_channels[], size_t number_of_voe_channels);
+ void ProcessAndEncodeAudio();
// Must be called on the same thread as PrepareDemux().
uint32_t CaptureLevel() const;
diff --git a/webrtc/voice_engine/voe_base_impl.cc b/webrtc/voice_engine/voe_base_impl.cc
index d1981a4..8072cc8 100644
--- a/webrtc/voice_engine/voe_base_impl.cc
+++ b/webrtc/voice_engine/voe_base_impl.cc
@@ -77,19 +77,66 @@
}
}
-int32_t VoEBaseImpl::RecordedDataIsAvailable(const void* audioSamples,
- const size_t nSamples,
- const size_t nBytesPerSample,
- const size_t nChannels,
- const uint32_t samplesPerSec,
- const uint32_t totalDelayMS,
- const int32_t clockDrift,
- const uint32_t currentMicLevel,
- const bool keyPressed,
- uint32_t& newMicLevel) {
- newMicLevel = static_cast<uint32_t>(ProcessRecordedDataWithAPM(
- nullptr, 0, audioSamples, samplesPerSec, nChannels, nSamples,
- totalDelayMS, clockDrift, currentMicLevel, keyPressed));
+int32_t VoEBaseImpl::RecordedDataIsAvailable(
+ const void* audio_data,
+ const size_t number_of_frames,
+ const size_t bytes_per_sample,
+ const size_t number_of_channels,
+ const uint32_t sample_rate,
+ const uint32_t audio_delay_milliseconds,
+ const int32_t clock_drift,
+ const uint32_t volume,
+ const bool key_pressed,
+ uint32_t& new_mic_volume) {
+ RTC_DCHECK_EQ(2 * number_of_channels, bytes_per_sample);
+ RTC_DCHECK(shared_->transmit_mixer() != nullptr);
+ RTC_DCHECK(shared_->audio_device() != nullptr);
+
+ uint32_t max_volume = 0;
+ uint16_t voe_mic_level = 0;
+ // Check for zero to skip this calculation; the consumer may use this to
+ // indicate no volume is available.
+ if (volume != 0) {
+ // Scale from ADM to VoE level range
+ if (shared_->audio_device()->MaxMicrophoneVolume(&max_volume) == 0) {
+ if (max_volume) {
+ voe_mic_level = static_cast<uint16_t>(
+ (volume * kMaxVolumeLevel + static_cast<int>(max_volume / 2)) /
+ max_volume);
+ }
+ }
+ // We learned that on certain systems (e.g Linux) the voe_mic_level
+ // can be greater than the maxVolumeLevel therefore
+ // we are going to cap the voe_mic_level to the maxVolumeLevel
+ // and change the maxVolume to volume if it turns out that
+ // the voe_mic_level is indeed greater than the maxVolumeLevel.
+ if (voe_mic_level > kMaxVolumeLevel) {
+ voe_mic_level = kMaxVolumeLevel;
+ max_volume = volume;
+ }
+ }
+
+ // Perform channel-independent operations
+ // (APM, mix with file, record to file, mute, etc.)
+ shared_->transmit_mixer()->PrepareDemux(
+ audio_data, number_of_frames, number_of_channels, sample_rate,
+ static_cast<uint16_t>(audio_delay_milliseconds), clock_drift,
+ voe_mic_level, key_pressed);
+
+ // Copy the audio frame to each sending channel and perform
+ // channel-dependent operations (file mixing, mute, etc.), encode and
+ // packetize+transmit the RTP packet.
+ shared_->transmit_mixer()->ProcessAndEncodeAudio();
+
+ // Scale from VoE to ADM level range.
+ uint32_t new_voe_mic_level = shared_->transmit_mixer()->CaptureLevel();
+ if (new_voe_mic_level != voe_mic_level) {
+ // Return the new volume if AGC has changed the volume.
+ return static_cast<int>((new_voe_mic_level * max_volume +
+ static_cast<int>(kMaxVolumeLevel / 2)) /
+ kMaxVolumeLevel);
+ }
+
return 0;
}
@@ -112,14 +159,15 @@
size_t number_of_channels,
size_t number_of_frames) {
voe::ChannelOwner ch = shared_->channel_manager().GetChannel(voe_channel);
- voe::Channel* channel_ptr = ch.channel();
- if (!channel_ptr) return;
-
- if (channel_ptr->Sending()) {
- channel_ptr->Demultiplex(static_cast<const int16_t*>(audio_data),
- sample_rate, number_of_frames, number_of_channels);
- channel_ptr->PrepareEncodeAndSend(sample_rate);
- channel_ptr->EncodeAndSend();
+ voe::Channel* channel = ch.channel();
+ if (!channel)
+ return;
+ if (channel->Sending()) {
+ // Send the audio to each channel directly without using the APM in the
+ // transmit mixer.
+ channel->ProcessAndEncodeAudio(static_cast<const int16_t*>(audio_data),
+ sample_rate, number_of_frames,
+ number_of_channels);
}
}
@@ -377,7 +425,8 @@
if (channel_owner->channel()->SetEngineInformation(
shared_->statistics(), *shared_->output_mixer(),
*shared_->process_thread(), *shared_->audio_device(),
- voiceEngineObserverPtr_, &callbackCritSect_) != 0) {
+ voiceEngineObserverPtr_, &callbackCritSect_,
+ shared_->encoder_queue()) != 0) {
shared_->SetLastError(
VE_CHANNEL_NOT_CREATED, kTraceError,
"CreateChannel() failed to associate engine and channel."
@@ -521,10 +570,7 @@
"StopSend() failed to locate channel");
return -1;
}
- if (channelPtr->StopSend() != 0) {
- LOG_F(LS_WARNING) << "StopSend() failed to stop sending for channel "
- << channel;
- }
+ channelPtr->StopSend();
return StopSend();
}
@@ -648,73 +694,6 @@
return shared_->statistics().SetUnInitialized();
}
-int VoEBaseImpl::ProcessRecordedDataWithAPM(
- const int voe_channels[], size_t number_of_voe_channels,
- const void* audio_data, uint32_t sample_rate, size_t number_of_channels,
- size_t number_of_frames, uint32_t audio_delay_milliseconds,
- int32_t clock_drift, uint32_t volume, bool key_pressed) {
- assert(shared_->transmit_mixer() != nullptr);
- assert(shared_->audio_device() != nullptr);
-
- uint32_t max_volume = 0;
- uint16_t voe_mic_level = 0;
- // Check for zero to skip this calculation; the consumer may use this to
- // indicate no volume is available.
- if (volume != 0) {
- // Scale from ADM to VoE level range
- if (shared_->audio_device()->MaxMicrophoneVolume(&max_volume) == 0) {
- if (max_volume) {
- voe_mic_level = static_cast<uint16_t>(
- (volume * kMaxVolumeLevel + static_cast<int>(max_volume / 2)) /
- max_volume);
- }
- }
- // We learned that on certain systems (e.g Linux) the voe_mic_level
- // can be greater than the maxVolumeLevel therefore
- // we are going to cap the voe_mic_level to the maxVolumeLevel
- // and change the maxVolume to volume if it turns out that
- // the voe_mic_level is indeed greater than the maxVolumeLevel.
- if (voe_mic_level > kMaxVolumeLevel) {
- voe_mic_level = kMaxVolumeLevel;
- max_volume = volume;
- }
- }
-
- // Perform channel-independent operations
- // (APM, mix with file, record to file, mute, etc.)
- shared_->transmit_mixer()->PrepareDemux(
- audio_data, number_of_frames, number_of_channels, sample_rate,
- static_cast<uint16_t>(audio_delay_milliseconds), clock_drift,
- voe_mic_level, key_pressed);
-
- // Copy the audio frame to each sending channel and perform
- // channel-dependent operations (file mixing, mute, etc.), encode and
- // packetize+transmit the RTP packet. When |number_of_voe_channels| == 0,
- // do the operations on all the existing VoE channels; otherwise the
- // operations will be done on specific channels.
- if (number_of_voe_channels == 0) {
- shared_->transmit_mixer()->DemuxAndMix();
- shared_->transmit_mixer()->EncodeAndSend();
- } else {
- shared_->transmit_mixer()->DemuxAndMix(voe_channels,
- number_of_voe_channels);
- shared_->transmit_mixer()->EncodeAndSend(voe_channels,
- number_of_voe_channels);
- }
-
- // Scale from VoE to ADM level range.
- uint32_t new_voe_mic_level = shared_->transmit_mixer()->CaptureLevel();
- if (new_voe_mic_level != voe_mic_level) {
- // Return the new volume if AGC has changed the volume.
- return static_cast<int>((new_voe_mic_level * max_volume +
- static_cast<int>(kMaxVolumeLevel / 2)) /
- kMaxVolumeLevel);
- }
-
- // Return 0 to indicate no change on the volume.
- return 0;
-}
-
void VoEBaseImpl::GetPlayoutData(int sample_rate, size_t number_of_channels,
size_t number_of_frames, bool feed_data_to_apm,
void* audio_data, int64_t* elapsed_time_ms,
diff --git a/webrtc/voice_engine/voe_base_impl.h b/webrtc/voice_engine/voe_base_impl.h
index b544c09..b9f3282 100644
--- a/webrtc/voice_engine/voe_base_impl.h
+++ b/webrtc/voice_engine/voe_base_impl.h
@@ -62,16 +62,16 @@
int AssociateSendChannel(int channel, int accociate_send_channel) override;
// AudioTransport
- int32_t RecordedDataIsAvailable(const void* audioSamples,
- const size_t nSamples,
- const size_t nBytesPerSample,
- const size_t nChannels,
- const uint32_t samplesPerSec,
- const uint32_t totalDelayMS,
- const int32_t clockDrift,
- const uint32_t currentMicLevel,
- const bool keyPressed,
- uint32_t& newMicLevel) override;
+ int32_t RecordedDataIsAvailable(const void* audio_data,
+ const size_t number_of_frames,
+ const size_t bytes_per_sample,
+ const size_t number_of_channels,
+ const uint32_t sample_rate,
+ const uint32_t audio_delay_milliseconds,
+ const int32_t clock_drift,
+ const uint32_t volume,
+ const bool key_pressed,
+ uint32_t& new_mic_volume) override;
int32_t NeedMorePlayData(const size_t nSamples,
const size_t nBytesPerSample,
const size_t nChannels,
@@ -109,18 +109,6 @@
int32_t StopSend();
int32_t TerminateInternal();
- // Helper function to process the recorded data with AudioProcessing Module,
- // demultiplex the data to specific voe channels, encode and send to the
- // network. When |number_of_VoE_channels| is 0, it will demultiplex the
- // data to all the existing VoE channels.
- // It returns new AGC microphone volume or 0 if no volume changes
- // should be done.
- int ProcessRecordedDataWithAPM(
- const int voe_channels[], size_t number_of_voe_channels,
- const void* audio_data, uint32_t sample_rate, size_t number_of_channels,
- size_t number_of_frames, uint32_t audio_delay_milliseconds,
- int32_t clock_drift, uint32_t volume, bool key_pressed);
-
void GetPlayoutData(int sample_rate, size_t number_of_channels,
size_t number_of_frames, bool feed_data_to_apm,
void* audio_data, int64_t* elapsed_time_ms,