blob: 18325f4159097fc20ef701bdf6ede825b6cad7c0 [file] [log] [blame]
turaj@webrtc.org7959e162013-09-12 18:30:26 +00001/*
2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020011#include "modules/audio_coding/include/audio_coding_module.h"
turaj@webrtc.org7959e162013-09-12 18:30:26 +000012
Jonathan Yu36344a02017-07-30 01:55:34 -070013#include <algorithm>
14
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020015#include "api/audio_codecs/builtin_audio_decoder_factory.h"
16#include "modules/audio_coding/acm2/acm_receiver.h"
17#include "modules/audio_coding/acm2/acm_resampler.h"
18#include "modules/audio_coding/acm2/codec_manager.h"
19#include "modules/audio_coding/acm2/rent_a_codec.h"
20#include "rtc_base/checks.h"
21#include "rtc_base/logging.h"
Karl Wiberge40468b2017-11-22 10:42:26 +010022#include "rtc_base/numerics/safe_conversions.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020023#include "system_wrappers/include/metrics.h"
turaj@webrtc.org7959e162013-09-12 18:30:26 +000024
25namespace webrtc {
26
kwibergc13ded52016-06-17 06:00:45 -070027namespace {
28
29struct EncoderFactory {
30 AudioEncoder* external_speech_encoder = nullptr;
31 acm2::CodecManager codec_manager;
32 acm2::RentACodec rent_a_codec;
33};
34
35class AudioCodingModuleImpl final : public AudioCodingModule {
36 public:
37 explicit AudioCodingModuleImpl(const AudioCodingModule::Config& config);
38 ~AudioCodingModuleImpl() override;
39
40 /////////////////////////////////////////
41 // Sender
42 //
43
44 // Can be called multiple times for Codec, CNG, RED.
45 int RegisterSendCodec(const CodecInst& send_codec) override;
46
47 void RegisterExternalSendCodec(
48 AudioEncoder* external_speech_encoder) override;
49
kwiberg24c7c122016-09-28 11:57:10 -070050 void ModifyEncoder(rtc::FunctionView<void(std::unique_ptr<AudioEncoder>*)>
51 modifier) override;
kwibergc13ded52016-06-17 06:00:45 -070052
kwiberg24c7c122016-09-28 11:57:10 -070053 void QueryEncoder(
54 rtc::FunctionView<void(const AudioEncoder*)> query) override;
ivoc85228d62016-07-27 04:53:47 -070055
kwibergc13ded52016-06-17 06:00:45 -070056 // Get current send codec.
57 rtc::Optional<CodecInst> SendCodec() const override;
58
59 // Get current send frequency.
60 int SendFrequency() const override;
61
62 // Sets the bitrate to the specified value in bits/sec. In case the codec does
63 // not support the requested value it will choose an appropriate value
64 // instead.
65 void SetBitRate(int bitrate_bps) override;
66
67 // Register a transport callback which will be
68 // called to deliver the encoded buffers.
69 int RegisterTransportCallback(AudioPacketizationCallback* transport) override;
70
71 // Add 10 ms of raw (PCM) audio data to the encoder.
72 int Add10MsData(const AudioFrame& audio_frame) override;
73
74 /////////////////////////////////////////
75 // (RED) Redundant Coding
76 //
77
78 // Configure RED status i.e. on/off.
79 int SetREDStatus(bool enable_red) override;
80
81 // Get RED status.
82 bool REDStatus() const override;
83
84 /////////////////////////////////////////
85 // (FEC) Forward Error Correction (codec internal)
86 //
87
88 // Configure FEC status i.e. on/off.
89 int SetCodecFEC(bool enabled_codec_fec) override;
90
91 // Get FEC status.
92 bool CodecFEC() const override;
93
94 // Set target packet loss rate
95 int SetPacketLossRate(int loss_rate) override;
96
97 /////////////////////////////////////////
98 // (VAD) Voice Activity Detection
99 // and
100 // (CNG) Comfort Noise Generation
101 //
102
103 int SetVAD(bool enable_dtx = true,
104 bool enable_vad = false,
105 ACMVADMode mode = VADNormal) override;
106
107 int VAD(bool* dtx_enabled,
108 bool* vad_enabled,
109 ACMVADMode* mode) const override;
110
111 int RegisterVADCallback(ACMVADCallback* vad_callback) override;
112
113 /////////////////////////////////////////
114 // Receiver
115 //
116
117 // Initialize receiver, resets codec database etc.
118 int InitializeReceiver() override;
119
120 // Get current receive frequency.
121 int ReceiveFrequency() const override;
122
123 // Get current playout frequency.
124 int PlayoutFrequency() const override;
125
kwiberg1c07c702017-03-27 07:15:49 -0700126 void SetReceiveCodecs(const std::map<int, SdpAudioFormat>& codecs) override;
127
kwiberg5adaf732016-10-04 09:33:27 -0700128 bool RegisterReceiveCodec(int rtp_payload_type,
129 const SdpAudioFormat& audio_format) override;
130
kwibergc13ded52016-06-17 06:00:45 -0700131 int RegisterReceiveCodec(const CodecInst& receive_codec) override;
132 int RegisterReceiveCodec(
133 const CodecInst& receive_codec,
kwiberg24c7c122016-09-28 11:57:10 -0700134 rtc::FunctionView<std::unique_ptr<AudioDecoder>()> isac_factory) override;
kwibergc13ded52016-06-17 06:00:45 -0700135
136 int RegisterExternalReceiveCodec(int rtp_payload_type,
137 AudioDecoder* external_decoder,
138 int sample_rate_hz,
139 int num_channels,
140 const std::string& name) override;
141
142 // Get current received codec.
143 int ReceiveCodec(CodecInst* current_codec) const override;
144
ossue280cde2016-10-12 11:04:10 -0700145 rtc::Optional<SdpAudioFormat> ReceiveFormat() const override;
146
kwibergc13ded52016-06-17 06:00:45 -0700147 // Incoming packet from network parsed and ready for decode.
148 int IncomingPacket(const uint8_t* incoming_payload,
149 const size_t payload_length,
150 const WebRtcRTPHeader& rtp_info) override;
151
kwibergc13ded52016-06-17 06:00:45 -0700152 // Minimum playout delay.
153 int SetMinimumPlayoutDelay(int time_ms) override;
154
155 // Maximum playout delay.
156 int SetMaximumPlayoutDelay(int time_ms) override;
157
158 // Smallest latency NetEq will maintain.
159 int LeastRequiredDelayMs() const override;
160
161 RTC_DEPRECATED int32_t PlayoutTimestamp(uint32_t* timestamp) override;
162
163 rtc::Optional<uint32_t> PlayoutTimestamp() override;
164
henrik.lundinb3f1c5d2016-08-22 15:39:53 -0700165 int FilteredCurrentDelayMs() const override;
166
kwibergc13ded52016-06-17 06:00:45 -0700167 // Get 10 milliseconds of raw audio data to play out, and
168 // automatic resample to the requested frequency if > 0.
169 int PlayoutData10Ms(int desired_freq_hz,
170 AudioFrame* audio_frame,
171 bool* muted) override;
172 int PlayoutData10Ms(int desired_freq_hz, AudioFrame* audio_frame) override;
173
174 /////////////////////////////////////////
175 // Statistics
176 //
177
178 int GetNetworkStatistics(NetworkStatistics* statistics) override;
179
180 int SetOpusApplication(OpusApplicationMode application) override;
181
182 // If current send codec is Opus, informs it about the maximum playback rate
183 // the receiver will render.
184 int SetOpusMaxPlaybackRate(int frequency_hz) override;
185
186 int EnableOpusDtx() override;
187
188 int DisableOpusDtx() override;
189
190 int UnregisterReceiveCodec(uint8_t payload_type) override;
191
192 int EnableNack(size_t max_nack_list_size) override;
193
194 void DisableNack() override;
195
196 std::vector<uint16_t> GetNackList(int64_t round_trip_time_ms) const override;
197
198 void GetDecodingCallStatistics(AudioDecodingCallStats* stats) const override;
199
ivoce1198e02017-09-08 08:13:19 -0700200 ANAStats GetANAStats() const override;
201
kwibergc13ded52016-06-17 06:00:45 -0700202 private:
203 struct InputData {
204 uint32_t input_timestamp;
205 const int16_t* audio;
206 size_t length_per_channel;
207 size_t audio_channel;
208 // If a re-mix is required (up or down), this buffer will store a re-mixed
209 // version of the input.
210 int16_t buffer[WEBRTC_10MS_PCM_AUDIO];
211 };
212
213 // This member class writes values to the named UMA histogram, but only if
214 // the value has changed since the last time (and always for the first call).
215 class ChangeLogger {
216 public:
217 explicit ChangeLogger(const std::string& histogram_name)
218 : histogram_name_(histogram_name) {}
219 // Logs the new value if it is different from the last logged value, or if
220 // this is the first call.
221 void MaybeLog(int value);
222
223 private:
224 int last_value_ = 0;
225 int first_time_ = true;
226 const std::string histogram_name_;
227 };
228
229 int RegisterReceiveCodecUnlocked(
230 const CodecInst& codec,
kwiberg24c7c122016-09-28 11:57:10 -0700231 rtc::FunctionView<std::unique_ptr<AudioDecoder>()> isac_factory)
danilchap56359be2017-09-07 07:53:45 -0700232 RTC_EXCLUSIVE_LOCKS_REQUIRED(acm_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700233
234 int Add10MsDataInternal(const AudioFrame& audio_frame, InputData* input_data)
danilchap56359be2017-09-07 07:53:45 -0700235 RTC_EXCLUSIVE_LOCKS_REQUIRED(acm_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700236 int Encode(const InputData& input_data)
danilchap56359be2017-09-07 07:53:45 -0700237 RTC_EXCLUSIVE_LOCKS_REQUIRED(acm_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700238
danilchap56359be2017-09-07 07:53:45 -0700239 int InitializeReceiverSafe() RTC_EXCLUSIVE_LOCKS_REQUIRED(acm_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700240
241 bool HaveValidEncoder(const char* caller_name) const
danilchap56359be2017-09-07 07:53:45 -0700242 RTC_EXCLUSIVE_LOCKS_REQUIRED(acm_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700243
244 // Preprocessing of input audio, including resampling and down-mixing if
245 // required, before pushing audio into encoder's buffer.
246 //
247 // in_frame: input audio-frame
248 // ptr_out: pointer to output audio_frame. If no preprocessing is required
249 // |ptr_out| will be pointing to |in_frame|, otherwise pointing to
250 // |preprocess_frame_|.
251 //
252 // Return value:
253 // -1: if encountering an error.
254 // 0: otherwise.
255 int PreprocessToAddData(const AudioFrame& in_frame,
256 const AudioFrame** ptr_out)
danilchap56359be2017-09-07 07:53:45 -0700257 RTC_EXCLUSIVE_LOCKS_REQUIRED(acm_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700258
259 // Change required states after starting to receive the codec corresponding
260 // to |index|.
261 int UpdateUponReceivingCodec(int index);
262
263 rtc::CriticalSection acm_crit_sect_;
danilchap56359be2017-09-07 07:53:45 -0700264 rtc::Buffer encode_buffer_ RTC_GUARDED_BY(acm_crit_sect_);
danilchap56359be2017-09-07 07:53:45 -0700265 uint32_t expected_codec_ts_ RTC_GUARDED_BY(acm_crit_sect_);
266 uint32_t expected_in_ts_ RTC_GUARDED_BY(acm_crit_sect_);
267 acm2::ACMResampler resampler_ RTC_GUARDED_BY(acm_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700268 acm2::AcmReceiver receiver_; // AcmReceiver has it's own internal lock.
danilchap56359be2017-09-07 07:53:45 -0700269 ChangeLogger bitrate_logger_ RTC_GUARDED_BY(acm_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700270
danilchap56359be2017-09-07 07:53:45 -0700271 std::unique_ptr<EncoderFactory> encoder_factory_
272 RTC_GUARDED_BY(acm_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700273
274 // Current encoder stack, either obtained from
275 // encoder_factory_->rent_a_codec.RentEncoderStack or provided by a call to
276 // RegisterEncoder.
danilchap56359be2017-09-07 07:53:45 -0700277 std::unique_ptr<AudioEncoder> encoder_stack_ RTC_GUARDED_BY(acm_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700278
danilchap56359be2017-09-07 07:53:45 -0700279 std::unique_ptr<AudioDecoder> isac_decoder_16k_
280 RTC_GUARDED_BY(acm_crit_sect_);
281 std::unique_ptr<AudioDecoder> isac_decoder_32k_
282 RTC_GUARDED_BY(acm_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700283
284 // This is to keep track of CN instances where we can send DTMFs.
danilchap56359be2017-09-07 07:53:45 -0700285 uint8_t previous_pltype_ RTC_GUARDED_BY(acm_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700286
danilchap56359be2017-09-07 07:53:45 -0700287 bool receiver_initialized_ RTC_GUARDED_BY(acm_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700288
danilchap56359be2017-09-07 07:53:45 -0700289 AudioFrame preprocess_frame_ RTC_GUARDED_BY(acm_crit_sect_);
290 bool first_10ms_data_ RTC_GUARDED_BY(acm_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700291
danilchap56359be2017-09-07 07:53:45 -0700292 bool first_frame_ RTC_GUARDED_BY(acm_crit_sect_);
293 uint32_t last_timestamp_ RTC_GUARDED_BY(acm_crit_sect_);
294 uint32_t last_rtp_timestamp_ RTC_GUARDED_BY(acm_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700295
296 rtc::CriticalSection callback_crit_sect_;
297 AudioPacketizationCallback* packetization_callback_
danilchap56359be2017-09-07 07:53:45 -0700298 RTC_GUARDED_BY(callback_crit_sect_);
299 ACMVADCallback* vad_callback_ RTC_GUARDED_BY(callback_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700300
301 int codec_histogram_bins_log_[static_cast<size_t>(
302 AudioEncoder::CodecType::kMaxLoggedAudioCodecTypes)];
303 int number_of_consecutive_empty_packets_;
304};
305
306// Adds a codec usage sample to the histogram.
307void UpdateCodecTypeHistogram(size_t codec_type) {
308 RTC_HISTOGRAM_ENUMERATION(
309 "WebRTC.Audio.Encoder.CodecType", static_cast<int>(codec_type),
310 static_cast<int>(
311 webrtc::AudioEncoder::CodecType::kMaxLoggedAudioCodecTypes));
312}
313
kwibergc13ded52016-06-17 06:00:45 -0700314// Stereo-to-mono can be used as in-place.
315int DownMix(const AudioFrame& frame,
316 size_t length_out_buff,
317 int16_t* out_buff) {
yujo36b1a5f2017-06-12 12:45:32 -0700318 RTC_DCHECK_EQ(frame.num_channels_, 2);
319 RTC_DCHECK_GE(length_out_buff, frame.samples_per_channel_);
320
321 if (!frame.muted()) {
322 const int16_t* frame_data = frame.data();
323 for (size_t n = 0; n < frame.samples_per_channel_; ++n) {
324 out_buff[n] = static_cast<int16_t>(
325 (static_cast<int32_t>(frame_data[2 * n]) +
326 static_cast<int32_t>(frame_data[2 * n + 1])) >> 1);
327 }
328 } else {
Jonathan Yu36344a02017-07-30 01:55:34 -0700329 std::fill(out_buff, out_buff + frame.samples_per_channel_, 0);
kwibergc13ded52016-06-17 06:00:45 -0700330 }
kwibergc13ded52016-06-17 06:00:45 -0700331 return 0;
332}
333
334// Mono-to-stereo can be used as in-place.
335int UpMix(const AudioFrame& frame, size_t length_out_buff, int16_t* out_buff) {
yujo36b1a5f2017-06-12 12:45:32 -0700336 RTC_DCHECK_EQ(frame.num_channels_, 1);
337 RTC_DCHECK_GE(length_out_buff, 2 * frame.samples_per_channel_);
338
339 if (!frame.muted()) {
340 const int16_t* frame_data = frame.data();
341 for (size_t n = frame.samples_per_channel_; n != 0; --n) {
342 size_t i = n - 1;
343 int16_t sample = frame_data[i];
344 out_buff[2 * i + 1] = sample;
345 out_buff[2 * i] = sample;
346 }
347 } else {
Jonathan Yu36344a02017-07-30 01:55:34 -0700348 std::fill(out_buff, out_buff + frame.samples_per_channel_ * 2, 0);
kwibergc13ded52016-06-17 06:00:45 -0700349 }
350 return 0;
351}
352
353void ConvertEncodedInfoToFragmentationHeader(
354 const AudioEncoder::EncodedInfo& info,
355 RTPFragmentationHeader* frag) {
356 if (info.redundant.empty()) {
357 frag->fragmentationVectorSize = 0;
358 return;
359 }
360
361 frag->VerifyAndAllocateFragmentationHeader(
362 static_cast<uint16_t>(info.redundant.size()));
363 frag->fragmentationVectorSize = static_cast<uint16_t>(info.redundant.size());
364 size_t offset = 0;
365 for (size_t i = 0; i < info.redundant.size(); ++i) {
366 frag->fragmentationOffset[i] = offset;
367 offset += info.redundant[i].encoded_bytes;
368 frag->fragmentationLength[i] = info.redundant[i].encoded_bytes;
kwibergd3edd772017-03-01 18:52:48 -0800369 frag->fragmentationTimeDiff[i] = rtc::dchecked_cast<uint16_t>(
kwibergc13ded52016-06-17 06:00:45 -0700370 info.encoded_timestamp - info.redundant[i].encoded_timestamp);
371 frag->fragmentationPlType[i] = info.redundant[i].payload_type;
372 }
373}
374
375// Wraps a raw AudioEncoder pointer. The idea is that you can put one of these
376// in a unique_ptr, to protect the contained raw pointer from being deleted
377// when the unique_ptr expires. (This is of course a bad idea in general, but
378// backwards compatibility.)
379class RawAudioEncoderWrapper final : public AudioEncoder {
380 public:
381 RawAudioEncoderWrapper(AudioEncoder* enc) : enc_(enc) {}
382 int SampleRateHz() const override { return enc_->SampleRateHz(); }
383 size_t NumChannels() const override { return enc_->NumChannels(); }
384 int RtpTimestampRateHz() const override { return enc_->RtpTimestampRateHz(); }
385 size_t Num10MsFramesInNextPacket() const override {
386 return enc_->Num10MsFramesInNextPacket();
387 }
388 size_t Max10MsFramesInAPacket() const override {
389 return enc_->Max10MsFramesInAPacket();
390 }
391 int GetTargetBitrate() const override { return enc_->GetTargetBitrate(); }
392 EncodedInfo EncodeImpl(uint32_t rtp_timestamp,
393 rtc::ArrayView<const int16_t> audio,
394 rtc::Buffer* encoded) override {
395 return enc_->Encode(rtp_timestamp, audio, encoded);
396 }
397 void Reset() override { return enc_->Reset(); }
398 bool SetFec(bool enable) override { return enc_->SetFec(enable); }
399 bool SetDtx(bool enable) override { return enc_->SetDtx(enable); }
400 bool SetApplication(Application application) override {
401 return enc_->SetApplication(application);
402 }
403 void SetMaxPlaybackRate(int frequency_hz) override {
404 return enc_->SetMaxPlaybackRate(frequency_hz);
405 }
kwibergc13ded52016-06-17 06:00:45 -0700406
407 private:
408 AudioEncoder* enc_;
409};
410
411// Return false on error.
412bool CreateSpeechEncoderIfNecessary(EncoderFactory* ef) {
413 auto* sp = ef->codec_manager.GetStackParams();
414 if (sp->speech_encoder) {
415 // Do nothing; we already have a speech encoder.
416 } else if (ef->codec_manager.GetCodecInst()) {
417 RTC_DCHECK(!ef->external_speech_encoder);
418 // We have no speech encoder, but we have a specification for making one.
419 std::unique_ptr<AudioEncoder> enc =
420 ef->rent_a_codec.RentEncoder(*ef->codec_manager.GetCodecInst());
421 if (!enc)
422 return false; // Encoder spec was bad.
423 sp->speech_encoder = std::move(enc);
424 } else if (ef->external_speech_encoder) {
425 RTC_DCHECK(!ef->codec_manager.GetCodecInst());
426 // We have an external speech encoder.
427 sp->speech_encoder = std::unique_ptr<AudioEncoder>(
428 new RawAudioEncoderWrapper(ef->external_speech_encoder));
429 }
430 return true;
431}
432
433void AudioCodingModuleImpl::ChangeLogger::MaybeLog(int value) {
434 if (value != last_value_ || first_time_) {
435 first_time_ = false;
436 last_value_ = value;
437 RTC_HISTOGRAM_COUNTS_SPARSE_100(histogram_name_, value);
438 }
439}
440
441AudioCodingModuleImpl::AudioCodingModuleImpl(
442 const AudioCodingModule::Config& config)
solenbergc7b4a452017-09-28 07:37:11 -0700443 : expected_codec_ts_(0xD87F3F9F),
kwibergc13ded52016-06-17 06:00:45 -0700444 expected_in_ts_(0xD87F3F9F),
445 receiver_(config),
446 bitrate_logger_("WebRTC.Audio.TargetBitrateInKbps"),
447 encoder_factory_(new EncoderFactory),
448 encoder_stack_(nullptr),
449 previous_pltype_(255),
450 receiver_initialized_(false),
451 first_10ms_data_(false),
452 first_frame_(true),
453 packetization_callback_(NULL),
454 vad_callback_(NULL),
455 codec_histogram_bins_log_(),
456 number_of_consecutive_empty_packets_(0) {
457 if (InitializeReceiverSafe() < 0) {
Mirko Bonadei675513b2017-11-09 11:09:25 +0100458 RTC_LOG(LS_ERROR) << "Cannot initialize receiver";
kwibergc13ded52016-06-17 06:00:45 -0700459 }
Mirko Bonadei675513b2017-11-09 11:09:25 +0100460 RTC_LOG(LS_INFO) << "Created";
kwibergc13ded52016-06-17 06:00:45 -0700461}
462
463AudioCodingModuleImpl::~AudioCodingModuleImpl() = default;
464
465int32_t AudioCodingModuleImpl::Encode(const InputData& input_data) {
466 AudioEncoder::EncodedInfo encoded_info;
467 uint8_t previous_pltype;
468
469 // Check if there is an encoder before.
470 if (!HaveValidEncoder("Process"))
471 return -1;
472
ossu63fb95a2016-07-06 09:34:22 -0700473 if(!first_frame_) {
deadbeeffcada902016-08-24 12:45:13 -0700474 RTC_DCHECK(IsNewerTimestamp(input_data.input_timestamp, last_timestamp_))
ossu63fb95a2016-07-06 09:34:22 -0700475 << "Time should not move backwards";
476 }
477
kwibergc13ded52016-06-17 06:00:45 -0700478 // Scale the timestamp to the codec's RTP timestamp rate.
479 uint32_t rtp_timestamp =
480 first_frame_ ? input_data.input_timestamp
481 : last_rtp_timestamp_ +
482 rtc::CheckedDivExact(
483 input_data.input_timestamp - last_timestamp_,
484 static_cast<uint32_t>(rtc::CheckedDivExact(
485 encoder_stack_->SampleRateHz(),
486 encoder_stack_->RtpTimestampRateHz())));
487 last_timestamp_ = input_data.input_timestamp;
488 last_rtp_timestamp_ = rtp_timestamp;
489 first_frame_ = false;
490
491 // Clear the buffer before reuse - encoded data will get appended.
492 encode_buffer_.Clear();
493 encoded_info = encoder_stack_->Encode(
494 rtp_timestamp, rtc::ArrayView<const int16_t>(
495 input_data.audio, input_data.audio_channel *
496 input_data.length_per_channel),
497 &encode_buffer_);
498
499 bitrate_logger_.MaybeLog(encoder_stack_->GetTargetBitrate() / 1000);
500 if (encode_buffer_.size() == 0 && !encoded_info.send_even_if_empty) {
501 // Not enough data.
502 return 0;
503 }
504 previous_pltype = previous_pltype_; // Read it while we have the critsect.
505
506 // Log codec type to histogram once every 500 packets.
507 if (encoded_info.encoded_bytes == 0) {
508 ++number_of_consecutive_empty_packets_;
509 } else {
510 size_t codec_type = static_cast<size_t>(encoded_info.encoder_type);
511 codec_histogram_bins_log_[codec_type] +=
512 number_of_consecutive_empty_packets_ + 1;
513 number_of_consecutive_empty_packets_ = 0;
514 if (codec_histogram_bins_log_[codec_type] >= 500) {
515 codec_histogram_bins_log_[codec_type] -= 500;
516 UpdateCodecTypeHistogram(codec_type);
517 }
518 }
519
520 RTPFragmentationHeader my_fragmentation;
521 ConvertEncodedInfoToFragmentationHeader(encoded_info, &my_fragmentation);
522 FrameType frame_type;
523 if (encode_buffer_.size() == 0 && encoded_info.send_even_if_empty) {
524 frame_type = kEmptyFrame;
525 encoded_info.payload_type = previous_pltype;
526 } else {
kwibergaf476c72016-11-28 15:21:39 -0800527 RTC_DCHECK_GT(encode_buffer_.size(), 0);
kwibergc13ded52016-06-17 06:00:45 -0700528 frame_type = encoded_info.speech ? kAudioFrameSpeech : kAudioFrameCN;
529 }
530
531 {
532 rtc::CritScope lock(&callback_crit_sect_);
533 if (packetization_callback_) {
534 packetization_callback_->SendData(
535 frame_type, encoded_info.payload_type, encoded_info.encoded_timestamp,
536 encode_buffer_.data(), encode_buffer_.size(),
537 my_fragmentation.fragmentationVectorSize > 0 ? &my_fragmentation
538 : nullptr);
539 }
540
541 if (vad_callback_) {
542 // Callback with VAD decision.
543 vad_callback_->InFrameType(frame_type);
544 }
545 }
546 previous_pltype_ = encoded_info.payload_type;
547 return static_cast<int32_t>(encode_buffer_.size());
548}
549
550/////////////////////////////////////////
551// Sender
552//
553
554// Can be called multiple times for Codec, CNG, RED.
555int AudioCodingModuleImpl::RegisterSendCodec(const CodecInst& send_codec) {
556 rtc::CritScope lock(&acm_crit_sect_);
557 if (!encoder_factory_->codec_manager.RegisterEncoder(send_codec)) {
558 return -1;
559 }
560 if (encoder_factory_->codec_manager.GetCodecInst()) {
561 encoder_factory_->external_speech_encoder = nullptr;
562 }
563 if (!CreateSpeechEncoderIfNecessary(encoder_factory_.get())) {
564 return -1;
565 }
566 auto* sp = encoder_factory_->codec_manager.GetStackParams();
567 if (sp->speech_encoder)
568 encoder_stack_ = encoder_factory_->rent_a_codec.RentEncoderStack(sp);
569 return 0;
570}
571
572void AudioCodingModuleImpl::RegisterExternalSendCodec(
573 AudioEncoder* external_speech_encoder) {
574 rtc::CritScope lock(&acm_crit_sect_);
575 encoder_factory_->codec_manager.UnsetCodecInst();
576 encoder_factory_->external_speech_encoder = external_speech_encoder;
577 RTC_CHECK(CreateSpeechEncoderIfNecessary(encoder_factory_.get()));
578 auto* sp = encoder_factory_->codec_manager.GetStackParams();
579 RTC_CHECK(sp->speech_encoder);
580 encoder_stack_ = encoder_factory_->rent_a_codec.RentEncoderStack(sp);
581}
582
583void AudioCodingModuleImpl::ModifyEncoder(
kwiberg24c7c122016-09-28 11:57:10 -0700584 rtc::FunctionView<void(std::unique_ptr<AudioEncoder>*)> modifier) {
kwibergc13ded52016-06-17 06:00:45 -0700585 rtc::CritScope lock(&acm_crit_sect_);
586
587 // Wipe the encoder factory, so that everything that relies on it will fail.
588 // We don't want the complexity of supporting swapping back and forth.
589 if (encoder_factory_) {
590 encoder_factory_.reset();
591 RTC_CHECK(!encoder_stack_); // Ensure we hadn't started using the factory.
592 }
593
594 modifier(&encoder_stack_);
595}
596
ivoc85228d62016-07-27 04:53:47 -0700597void AudioCodingModuleImpl::QueryEncoder(
kwiberg24c7c122016-09-28 11:57:10 -0700598 rtc::FunctionView<void(const AudioEncoder*)> query) {
ivoc85228d62016-07-27 04:53:47 -0700599 rtc::CritScope lock(&acm_crit_sect_);
600 query(encoder_stack_.get());
601}
602
kwibergc13ded52016-06-17 06:00:45 -0700603// Get current send codec.
604rtc::Optional<CodecInst> AudioCodingModuleImpl::SendCodec() const {
605 rtc::CritScope lock(&acm_crit_sect_);
606 if (encoder_factory_) {
607 auto* ci = encoder_factory_->codec_manager.GetCodecInst();
608 if (ci) {
Oskar Sundbom12ab00b2017-11-16 15:31:38 +0100609 return *ci;
kwibergc13ded52016-06-17 06:00:45 -0700610 }
611 CreateSpeechEncoderIfNecessary(encoder_factory_.get());
612 const std::unique_ptr<AudioEncoder>& enc =
613 encoder_factory_->codec_manager.GetStackParams()->speech_encoder;
614 if (enc) {
Oskar Sundbom12ab00b2017-11-16 15:31:38 +0100615 return acm2::CodecManager::ForgeCodecInst(enc.get());
kwibergc13ded52016-06-17 06:00:45 -0700616 }
Oskar Sundbom12ab00b2017-11-16 15:31:38 +0100617 return rtc::nullopt;
kwibergc13ded52016-06-17 06:00:45 -0700618 } else {
619 return encoder_stack_
620 ? rtc::Optional<CodecInst>(
621 acm2::CodecManager::ForgeCodecInst(encoder_stack_.get()))
Oskar Sundbom12ab00b2017-11-16 15:31:38 +0100622 : rtc::nullopt;
kwibergc13ded52016-06-17 06:00:45 -0700623 }
624}
625
626// Get current send frequency.
627int AudioCodingModuleImpl::SendFrequency() const {
kwibergc13ded52016-06-17 06:00:45 -0700628 rtc::CritScope lock(&acm_crit_sect_);
629
630 if (!encoder_stack_) {
Mirko Bonadei675513b2017-11-09 11:09:25 +0100631 RTC_LOG(LS_ERROR) << "SendFrequency Failed, no codec is registered";
kwibergc13ded52016-06-17 06:00:45 -0700632 return -1;
633 }
634
635 return encoder_stack_->SampleRateHz();
636}
637
638void AudioCodingModuleImpl::SetBitRate(int bitrate_bps) {
639 rtc::CritScope lock(&acm_crit_sect_);
640 if (encoder_stack_) {
Oskar Sundbom12ab00b2017-11-16 15:31:38 +0100641 encoder_stack_->OnReceivedUplinkBandwidth(bitrate_bps, rtc::nullopt);
kwibergc13ded52016-06-17 06:00:45 -0700642 }
643}
644
645// Register a transport callback which will be called to deliver
646// the encoded buffers.
647int AudioCodingModuleImpl::RegisterTransportCallback(
648 AudioPacketizationCallback* transport) {
649 rtc::CritScope lock(&callback_crit_sect_);
650 packetization_callback_ = transport;
651 return 0;
652}
653
654// Add 10MS of raw (PCM) audio data to the encoder.
655int AudioCodingModuleImpl::Add10MsData(const AudioFrame& audio_frame) {
656 InputData input_data;
657 rtc::CritScope lock(&acm_crit_sect_);
658 int r = Add10MsDataInternal(audio_frame, &input_data);
659 return r < 0 ? r : Encode(input_data);
660}
661
662int AudioCodingModuleImpl::Add10MsDataInternal(const AudioFrame& audio_frame,
663 InputData* input_data) {
664 if (audio_frame.samples_per_channel_ == 0) {
665 assert(false);
Mirko Bonadei675513b2017-11-09 11:09:25 +0100666 RTC_LOG(LS_ERROR) << "Cannot Add 10 ms audio, payload length is zero";
kwibergc13ded52016-06-17 06:00:45 -0700667 return -1;
668 }
669
670 if (audio_frame.sample_rate_hz_ > 48000) {
671 assert(false);
Mirko Bonadei675513b2017-11-09 11:09:25 +0100672 RTC_LOG(LS_ERROR) << "Cannot Add 10 ms audio, input frequency not valid";
kwibergc13ded52016-06-17 06:00:45 -0700673 return -1;
674 }
675
676 // If the length and frequency matches. We currently just support raw PCM.
677 if (static_cast<size_t>(audio_frame.sample_rate_hz_ / 100) !=
678 audio_frame.samples_per_channel_) {
Mirko Bonadei675513b2017-11-09 11:09:25 +0100679 RTC_LOG(LS_ERROR)
Alex Loiko300ec8c2017-05-30 17:23:28 +0200680 << "Cannot Add 10 ms audio, input frequency and length doesn't match";
kwibergc13ded52016-06-17 06:00:45 -0700681 return -1;
682 }
683
684 if (audio_frame.num_channels_ != 1 && audio_frame.num_channels_ != 2) {
Mirko Bonadei675513b2017-11-09 11:09:25 +0100685 RTC_LOG(LS_ERROR) << "Cannot Add 10 ms audio, invalid number of channels.";
kwibergc13ded52016-06-17 06:00:45 -0700686 return -1;
687 }
688
689 // Do we have a codec registered?
690 if (!HaveValidEncoder("Add10MsData")) {
691 return -1;
692 }
693
694 const AudioFrame* ptr_frame;
695 // Perform a resampling, also down-mix if it is required and can be
696 // performed before resampling (a down mix prior to resampling will take
697 // place if both primary and secondary encoders are mono and input is in
698 // stereo).
699 if (PreprocessToAddData(audio_frame, &ptr_frame) < 0) {
700 return -1;
701 }
702
703 // Check whether we need an up-mix or down-mix?
704 const size_t current_num_channels = encoder_stack_->NumChannels();
705 const bool same_num_channels =
706 ptr_frame->num_channels_ == current_num_channels;
707
708 if (!same_num_channels) {
709 if (ptr_frame->num_channels_ == 1) {
710 if (UpMix(*ptr_frame, WEBRTC_10MS_PCM_AUDIO, input_data->buffer) < 0)
711 return -1;
712 } else {
713 if (DownMix(*ptr_frame, WEBRTC_10MS_PCM_AUDIO, input_data->buffer) < 0)
714 return -1;
715 }
716 }
717
718 // When adding data to encoders this pointer is pointing to an audio buffer
719 // with correct number of channels.
yujo36b1a5f2017-06-12 12:45:32 -0700720 const int16_t* ptr_audio = ptr_frame->data();
kwibergc13ded52016-06-17 06:00:45 -0700721
722 // For pushing data to primary, point the |ptr_audio| to correct buffer.
723 if (!same_num_channels)
724 ptr_audio = input_data->buffer;
725
yujo36b1a5f2017-06-12 12:45:32 -0700726 // TODO(yujo): Skip encode of muted frames.
kwibergc13ded52016-06-17 06:00:45 -0700727 input_data->input_timestamp = ptr_frame->timestamp_;
728 input_data->audio = ptr_audio;
729 input_data->length_per_channel = ptr_frame->samples_per_channel_;
730 input_data->audio_channel = current_num_channels;
731
732 return 0;
733}
734
735// Perform a resampling and down-mix if required. We down-mix only if
736// encoder is mono and input is stereo. In case of dual-streaming, both
737// encoders has to be mono for down-mix to take place.
738// |*ptr_out| will point to the pre-processed audio-frame. If no pre-processing
739// is required, |*ptr_out| points to |in_frame|.
yujo36b1a5f2017-06-12 12:45:32 -0700740// TODO(yujo): Make this more efficient for muted frames.
kwibergc13ded52016-06-17 06:00:45 -0700741int AudioCodingModuleImpl::PreprocessToAddData(const AudioFrame& in_frame,
742 const AudioFrame** ptr_out) {
743 const bool resample =
744 in_frame.sample_rate_hz_ != encoder_stack_->SampleRateHz();
745
746 // This variable is true if primary codec and secondary codec (if exists)
747 // are both mono and input is stereo.
748 // TODO(henrik.lundin): This condition should probably be
749 // in_frame.num_channels_ > encoder_stack_->NumChannels()
750 const bool down_mix =
751 in_frame.num_channels_ == 2 && encoder_stack_->NumChannels() == 1;
752
753 if (!first_10ms_data_) {
754 expected_in_ts_ = in_frame.timestamp_;
755 expected_codec_ts_ = in_frame.timestamp_;
756 first_10ms_data_ = true;
757 } else if (in_frame.timestamp_ != expected_in_ts_) {
Mirko Bonadei675513b2017-11-09 11:09:25 +0100758 RTC_LOG(LS_WARNING) << "Unexpected input timestamp: " << in_frame.timestamp_
759 << ", expected: " << expected_in_ts_;
kwibergc13ded52016-06-17 06:00:45 -0700760 expected_codec_ts_ +=
761 (in_frame.timestamp_ - expected_in_ts_) *
762 static_cast<uint32_t>(
763 static_cast<double>(encoder_stack_->SampleRateHz()) /
764 static_cast<double>(in_frame.sample_rate_hz_));
765 expected_in_ts_ = in_frame.timestamp_;
766 }
767
768
769 if (!down_mix && !resample) {
770 // No pre-processing is required.
ossu63fb95a2016-07-06 09:34:22 -0700771 if (expected_in_ts_ == expected_codec_ts_) {
772 // If we've never resampled, we can use the input frame as-is
773 *ptr_out = &in_frame;
774 } else {
775 // Otherwise we'll need to alter the timestamp. Since in_frame is const,
776 // we'll have to make a copy of it.
777 preprocess_frame_.CopyFrom(in_frame);
778 preprocess_frame_.timestamp_ = expected_codec_ts_;
779 *ptr_out = &preprocess_frame_;
780 }
781
kwibergc13ded52016-06-17 06:00:45 -0700782 expected_in_ts_ += static_cast<uint32_t>(in_frame.samples_per_channel_);
783 expected_codec_ts_ += static_cast<uint32_t>(in_frame.samples_per_channel_);
kwibergc13ded52016-06-17 06:00:45 -0700784 return 0;
785 }
786
787 *ptr_out = &preprocess_frame_;
788 preprocess_frame_.num_channels_ = in_frame.num_channels_;
789 int16_t audio[WEBRTC_10MS_PCM_AUDIO];
yujo36b1a5f2017-06-12 12:45:32 -0700790 const int16_t* src_ptr_audio = in_frame.data();
kwibergc13ded52016-06-17 06:00:45 -0700791 if (down_mix) {
792 // If a resampling is required the output of a down-mix is written into a
793 // local buffer, otherwise, it will be written to the output frame.
yujo36b1a5f2017-06-12 12:45:32 -0700794 int16_t* dest_ptr_audio = resample ?
795 audio : preprocess_frame_.mutable_data();
kwibergc13ded52016-06-17 06:00:45 -0700796 if (DownMix(in_frame, WEBRTC_10MS_PCM_AUDIO, dest_ptr_audio) < 0)
797 return -1;
798 preprocess_frame_.num_channels_ = 1;
799 // Set the input of the resampler is the down-mixed signal.
800 src_ptr_audio = audio;
801 }
802
803 preprocess_frame_.timestamp_ = expected_codec_ts_;
804 preprocess_frame_.samples_per_channel_ = in_frame.samples_per_channel_;
805 preprocess_frame_.sample_rate_hz_ = in_frame.sample_rate_hz_;
806 // If it is required, we have to do a resampling.
807 if (resample) {
808 // The result of the resampler is written to output frame.
yujo36b1a5f2017-06-12 12:45:32 -0700809 int16_t* dest_ptr_audio = preprocess_frame_.mutable_data();
kwibergc13ded52016-06-17 06:00:45 -0700810
811 int samples_per_channel = resampler_.Resample10Msec(
812 src_ptr_audio, in_frame.sample_rate_hz_, encoder_stack_->SampleRateHz(),
813 preprocess_frame_.num_channels_, AudioFrame::kMaxDataSizeSamples,
814 dest_ptr_audio);
815
816 if (samples_per_channel < 0) {
Mirko Bonadei675513b2017-11-09 11:09:25 +0100817 RTC_LOG(LS_ERROR) << "Cannot add 10 ms audio, resampling failed";
kwibergc13ded52016-06-17 06:00:45 -0700818 return -1;
819 }
820 preprocess_frame_.samples_per_channel_ =
821 static_cast<size_t>(samples_per_channel);
822 preprocess_frame_.sample_rate_hz_ = encoder_stack_->SampleRateHz();
823 }
824
825 expected_codec_ts_ +=
826 static_cast<uint32_t>(preprocess_frame_.samples_per_channel_);
827 expected_in_ts_ += static_cast<uint32_t>(in_frame.samples_per_channel_);
828
829 return 0;
830}
831
832/////////////////////////////////////////
833// (RED) Redundant Coding
834//
835
836bool AudioCodingModuleImpl::REDStatus() const {
837 rtc::CritScope lock(&acm_crit_sect_);
838 return encoder_factory_->codec_manager.GetStackParams()->use_red;
839}
840
841// Configure RED status i.e on/off.
842int AudioCodingModuleImpl::SetREDStatus(bool enable_red) {
843#ifdef WEBRTC_CODEC_RED
844 rtc::CritScope lock(&acm_crit_sect_);
845 CreateSpeechEncoderIfNecessary(encoder_factory_.get());
846 if (!encoder_factory_->codec_manager.SetCopyRed(enable_red)) {
847 return -1;
848 }
849 auto* sp = encoder_factory_->codec_manager.GetStackParams();
850 if (sp->speech_encoder)
851 encoder_stack_ = encoder_factory_->rent_a_codec.RentEncoderStack(sp);
852 return 0;
853#else
Mirko Bonadei675513b2017-11-09 11:09:25 +0100854 RTC_LOG(LS_WARNING) << " WEBRTC_CODEC_RED is undefined";
kwibergc13ded52016-06-17 06:00:45 -0700855 return -1;
856#endif
857}
858
859/////////////////////////////////////////
860// (FEC) Forward Error Correction (codec internal)
861//
862
863bool AudioCodingModuleImpl::CodecFEC() const {
864 rtc::CritScope lock(&acm_crit_sect_);
865 return encoder_factory_->codec_manager.GetStackParams()->use_codec_fec;
866}
867
868int AudioCodingModuleImpl::SetCodecFEC(bool enable_codec_fec) {
869 rtc::CritScope lock(&acm_crit_sect_);
870 CreateSpeechEncoderIfNecessary(encoder_factory_.get());
871 if (!encoder_factory_->codec_manager.SetCodecFEC(enable_codec_fec)) {
872 return -1;
873 }
874 auto* sp = encoder_factory_->codec_manager.GetStackParams();
875 if (sp->speech_encoder)
876 encoder_stack_ = encoder_factory_->rent_a_codec.RentEncoderStack(sp);
877 if (enable_codec_fec) {
878 return sp->use_codec_fec ? 0 : -1;
879 } else {
880 RTC_DCHECK(!sp->use_codec_fec);
881 return 0;
882 }
883}
884
885int AudioCodingModuleImpl::SetPacketLossRate(int loss_rate) {
886 rtc::CritScope lock(&acm_crit_sect_);
887 if (HaveValidEncoder("SetPacketLossRate")) {
minyue4b9a2cb2016-11-30 06:49:59 -0800888 encoder_stack_->OnReceivedUplinkPacketLossFraction(loss_rate / 100.0);
kwibergc13ded52016-06-17 06:00:45 -0700889 }
890 return 0;
891}
892
893/////////////////////////////////////////
894// (VAD) Voice Activity Detection
895//
896int AudioCodingModuleImpl::SetVAD(bool enable_dtx,
897 bool enable_vad,
898 ACMVADMode mode) {
899 // Note: |enable_vad| is not used; VAD is enabled based on the DTX setting.
900 RTC_DCHECK_EQ(enable_dtx, enable_vad);
901 rtc::CritScope lock(&acm_crit_sect_);
902 CreateSpeechEncoderIfNecessary(encoder_factory_.get());
903 if (!encoder_factory_->codec_manager.SetVAD(enable_dtx, mode)) {
904 return -1;
905 }
906 auto* sp = encoder_factory_->codec_manager.GetStackParams();
907 if (sp->speech_encoder)
908 encoder_stack_ = encoder_factory_->rent_a_codec.RentEncoderStack(sp);
909 return 0;
910}
911
912// Get VAD/DTX settings.
913int AudioCodingModuleImpl::VAD(bool* dtx_enabled, bool* vad_enabled,
914 ACMVADMode* mode) const {
915 rtc::CritScope lock(&acm_crit_sect_);
916 const auto* sp = encoder_factory_->codec_manager.GetStackParams();
917 *dtx_enabled = *vad_enabled = sp->use_cng;
918 *mode = sp->vad_mode;
919 return 0;
920}
921
922/////////////////////////////////////////
923// Receiver
924//
925
926int AudioCodingModuleImpl::InitializeReceiver() {
927 rtc::CritScope lock(&acm_crit_sect_);
928 return InitializeReceiverSafe();
929}
930
931// Initialize receiver, resets codec database etc.
932int AudioCodingModuleImpl::InitializeReceiverSafe() {
933 // If the receiver is already initialized then we want to destroy any
934 // existing decoders. After a call to this function, we should have a clean
935 // start-up.
kwiberg6b19b562016-09-20 04:02:25 -0700936 if (receiver_initialized_)
937 receiver_.RemoveAllCodecs();
kwibergc13ded52016-06-17 06:00:45 -0700938 receiver_.ResetInitialDelay();
939 receiver_.SetMinimumDelay(0);
940 receiver_.SetMaximumDelay(0);
941 receiver_.FlushBuffers();
942
kwibergc13ded52016-06-17 06:00:45 -0700943 receiver_initialized_ = true;
944 return 0;
945}
946
947// Get current receive frequency.
948int AudioCodingModuleImpl::ReceiveFrequency() const {
949 const auto last_packet_sample_rate = receiver_.last_packet_sample_rate_hz();
950 return last_packet_sample_rate ? *last_packet_sample_rate
951 : receiver_.last_output_sample_rate_hz();
952}
953
954// Get current playout frequency.
955int AudioCodingModuleImpl::PlayoutFrequency() const {
kwibergc13ded52016-06-17 06:00:45 -0700956 return receiver_.last_output_sample_rate_hz();
957}
958
kwiberg1c07c702017-03-27 07:15:49 -0700959void AudioCodingModuleImpl::SetReceiveCodecs(
960 const std::map<int, SdpAudioFormat>& codecs) {
961 rtc::CritScope lock(&acm_crit_sect_);
962 receiver_.SetCodecs(codecs);
963}
964
kwiberg5adaf732016-10-04 09:33:27 -0700965bool AudioCodingModuleImpl::RegisterReceiveCodec(
966 int rtp_payload_type,
967 const SdpAudioFormat& audio_format) {
968 rtc::CritScope lock(&acm_crit_sect_);
969 RTC_DCHECK(receiver_initialized_);
970
971 if (!acm2::RentACodec::IsPayloadTypeValid(rtp_payload_type)) {
Mirko Bonadei675513b2017-11-09 11:09:25 +0100972 RTC_LOG_F(LS_ERROR) << "Invalid payload-type " << rtp_payload_type
973 << " for decoder.";
kwiberg5adaf732016-10-04 09:33:27 -0700974 return false;
975 }
976
977 return receiver_.AddCodec(rtp_payload_type, audio_format);
978}
979
kwibergc13ded52016-06-17 06:00:45 -0700980int AudioCodingModuleImpl::RegisterReceiveCodec(const CodecInst& codec) {
981 rtc::CritScope lock(&acm_crit_sect_);
982 auto* ef = encoder_factory_.get();
983 return RegisterReceiveCodecUnlocked(
984 codec, [&] { return ef->rent_a_codec.RentIsacDecoder(codec.plfreq); });
985}
986
987int AudioCodingModuleImpl::RegisterReceiveCodec(
988 const CodecInst& codec,
kwiberg24c7c122016-09-28 11:57:10 -0700989 rtc::FunctionView<std::unique_ptr<AudioDecoder>()> isac_factory) {
kwibergc13ded52016-06-17 06:00:45 -0700990 rtc::CritScope lock(&acm_crit_sect_);
991 return RegisterReceiveCodecUnlocked(codec, isac_factory);
992}
993
994int AudioCodingModuleImpl::RegisterReceiveCodecUnlocked(
995 const CodecInst& codec,
kwiberg24c7c122016-09-28 11:57:10 -0700996 rtc::FunctionView<std::unique_ptr<AudioDecoder>()> isac_factory) {
kwibergc13ded52016-06-17 06:00:45 -0700997 RTC_DCHECK(receiver_initialized_);
998 if (codec.channels > 2) {
Mirko Bonadei675513b2017-11-09 11:09:25 +0100999 RTC_LOG_F(LS_ERROR) << "Unsupported number of channels: " << codec.channels;
kwibergc13ded52016-06-17 06:00:45 -07001000 return -1;
1001 }
1002
1003 auto codec_id = acm2::RentACodec::CodecIdByParams(codec.plname, codec.plfreq,
1004 codec.channels);
1005 if (!codec_id) {
Mirko Bonadei675513b2017-11-09 11:09:25 +01001006 RTC_LOG_F(LS_ERROR)
1007 << "Wrong codec params to be registered as receive codec";
kwibergc13ded52016-06-17 06:00:45 -07001008 return -1;
1009 }
1010 auto codec_index = acm2::RentACodec::CodecIndexFromId(*codec_id);
1011 RTC_CHECK(codec_index) << "Invalid codec ID: " << static_cast<int>(*codec_id);
1012
1013 // Check if the payload-type is valid.
1014 if (!acm2::RentACodec::IsPayloadTypeValid(codec.pltype)) {
Mirko Bonadei675513b2017-11-09 11:09:25 +01001015 RTC_LOG_F(LS_ERROR) << "Invalid payload type " << codec.pltype << " for "
1016 << codec.plname;
kwibergc13ded52016-06-17 06:00:45 -07001017 return -1;
1018 }
1019
1020 AudioDecoder* isac_decoder = nullptr;
1021 if (STR_CASE_CMP(codec.plname, "isac") == 0) {
1022 std::unique_ptr<AudioDecoder>& saved_isac_decoder =
1023 codec.plfreq == 16000 ? isac_decoder_16k_ : isac_decoder_32k_;
1024 if (!saved_isac_decoder) {
1025 saved_isac_decoder = isac_factory();
1026 }
1027 isac_decoder = saved_isac_decoder.get();
1028 }
1029 return receiver_.AddCodec(*codec_index, codec.pltype, codec.channels,
1030 codec.plfreq, isac_decoder, codec.plname);
1031}
1032
1033int AudioCodingModuleImpl::RegisterExternalReceiveCodec(
1034 int rtp_payload_type,
1035 AudioDecoder* external_decoder,
1036 int sample_rate_hz,
1037 int num_channels,
1038 const std::string& name) {
1039 rtc::CritScope lock(&acm_crit_sect_);
1040 RTC_DCHECK(receiver_initialized_);
1041 if (num_channels > 2 || num_channels < 0) {
Mirko Bonadei675513b2017-11-09 11:09:25 +01001042 RTC_LOG_F(LS_ERROR) << "Unsupported number of channels: " << num_channels;
kwibergc13ded52016-06-17 06:00:45 -07001043 return -1;
1044 }
1045
1046 // Check if the payload-type is valid.
1047 if (!acm2::RentACodec::IsPayloadTypeValid(rtp_payload_type)) {
Mirko Bonadei675513b2017-11-09 11:09:25 +01001048 RTC_LOG_F(LS_ERROR) << "Invalid payload-type " << rtp_payload_type
1049 << " for external decoder.";
kwibergc13ded52016-06-17 06:00:45 -07001050 return -1;
1051 }
1052
1053 return receiver_.AddCodec(-1 /* external */, rtp_payload_type, num_channels,
1054 sample_rate_hz, external_decoder, name);
1055}
1056
1057// Get current received codec.
1058int AudioCodingModuleImpl::ReceiveCodec(CodecInst* current_codec) const {
1059 rtc::CritScope lock(&acm_crit_sect_);
1060 return receiver_.LastAudioCodec(current_codec);
1061}
1062
ossue280cde2016-10-12 11:04:10 -07001063rtc::Optional<SdpAudioFormat> AudioCodingModuleImpl::ReceiveFormat() const {
1064 rtc::CritScope lock(&acm_crit_sect_);
1065 return receiver_.LastAudioFormat();
1066}
1067
kwibergc13ded52016-06-17 06:00:45 -07001068// Incoming packet from network parsed and ready for decode.
1069int AudioCodingModuleImpl::IncomingPacket(const uint8_t* incoming_payload,
1070 const size_t payload_length,
1071 const WebRtcRTPHeader& rtp_header) {
henrik.lundinb8c55b12017-05-10 07:38:01 -07001072 RTC_DCHECK_EQ(payload_length == 0, incoming_payload == nullptr);
kwibergc13ded52016-06-17 06:00:45 -07001073 return receiver_.InsertPacket(
1074 rtp_header,
1075 rtc::ArrayView<const uint8_t>(incoming_payload, payload_length));
1076}
1077
1078// Minimum playout delay (Used for lip-sync).
1079int AudioCodingModuleImpl::SetMinimumPlayoutDelay(int time_ms) {
1080 if ((time_ms < 0) || (time_ms > 10000)) {
Mirko Bonadei675513b2017-11-09 11:09:25 +01001081 RTC_LOG(LS_ERROR) << "Delay must be in the range of 0-10000 milliseconds.";
kwibergc13ded52016-06-17 06:00:45 -07001082 return -1;
1083 }
1084 return receiver_.SetMinimumDelay(time_ms);
1085}
1086
1087int AudioCodingModuleImpl::SetMaximumPlayoutDelay(int time_ms) {
1088 if ((time_ms < 0) || (time_ms > 10000)) {
Mirko Bonadei675513b2017-11-09 11:09:25 +01001089 RTC_LOG(LS_ERROR) << "Delay must be in the range of 0-10000 milliseconds.";
kwibergc13ded52016-06-17 06:00:45 -07001090 return -1;
1091 }
1092 return receiver_.SetMaximumDelay(time_ms);
1093}
1094
1095// Get 10 milliseconds of raw audio data to play out.
1096// Automatic resample to the requested frequency.
1097int AudioCodingModuleImpl::PlayoutData10Ms(int desired_freq_hz,
1098 AudioFrame* audio_frame,
1099 bool* muted) {
1100 // GetAudio always returns 10 ms, at the requested sample rate.
1101 if (receiver_.GetAudio(desired_freq_hz, audio_frame, muted) != 0) {
Mirko Bonadei675513b2017-11-09 11:09:25 +01001102 RTC_LOG(LS_ERROR) << "PlayoutData failed, RecOut Failed";
kwibergc13ded52016-06-17 06:00:45 -07001103 return -1;
1104 }
kwibergc13ded52016-06-17 06:00:45 -07001105 return 0;
1106}
1107
1108int AudioCodingModuleImpl::PlayoutData10Ms(int desired_freq_hz,
1109 AudioFrame* audio_frame) {
1110 bool muted;
1111 int ret = PlayoutData10Ms(desired_freq_hz, audio_frame, &muted);
1112 RTC_DCHECK(!muted);
1113 return ret;
1114}
1115
1116/////////////////////////////////////////
1117// Statistics
1118//
1119
1120// TODO(turajs) change the return value to void. Also change the corresponding
1121// NetEq function.
1122int AudioCodingModuleImpl::GetNetworkStatistics(NetworkStatistics* statistics) {
1123 receiver_.GetNetworkStatistics(statistics);
1124 return 0;
1125}
1126
1127int AudioCodingModuleImpl::RegisterVADCallback(ACMVADCallback* vad_callback) {
Mirko Bonadei675513b2017-11-09 11:09:25 +01001128 RTC_LOG(LS_VERBOSE) << "RegisterVADCallback()";
kwibergc13ded52016-06-17 06:00:45 -07001129 rtc::CritScope lock(&callback_crit_sect_);
1130 vad_callback_ = vad_callback;
1131 return 0;
1132}
1133
kwibergc13ded52016-06-17 06:00:45 -07001134int AudioCodingModuleImpl::SetOpusApplication(OpusApplicationMode application) {
1135 rtc::CritScope lock(&acm_crit_sect_);
1136 if (!HaveValidEncoder("SetOpusApplication")) {
1137 return -1;
1138 }
1139 AudioEncoder::Application app;
1140 switch (application) {
1141 case kVoip:
1142 app = AudioEncoder::Application::kSpeech;
1143 break;
1144 case kAudio:
1145 app = AudioEncoder::Application::kAudio;
1146 break;
1147 default:
1148 FATAL();
1149 return 0;
1150 }
1151 return encoder_stack_->SetApplication(app) ? 0 : -1;
1152}
1153
1154// Informs Opus encoder of the maximum playback rate the receiver will render.
1155int AudioCodingModuleImpl::SetOpusMaxPlaybackRate(int frequency_hz) {
1156 rtc::CritScope lock(&acm_crit_sect_);
1157 if (!HaveValidEncoder("SetOpusMaxPlaybackRate")) {
1158 return -1;
1159 }
1160 encoder_stack_->SetMaxPlaybackRate(frequency_hz);
1161 return 0;
1162}
1163
1164int AudioCodingModuleImpl::EnableOpusDtx() {
1165 rtc::CritScope lock(&acm_crit_sect_);
1166 if (!HaveValidEncoder("EnableOpusDtx")) {
1167 return -1;
1168 }
1169 return encoder_stack_->SetDtx(true) ? 0 : -1;
1170}
1171
1172int AudioCodingModuleImpl::DisableOpusDtx() {
1173 rtc::CritScope lock(&acm_crit_sect_);
1174 if (!HaveValidEncoder("DisableOpusDtx")) {
1175 return -1;
1176 }
1177 return encoder_stack_->SetDtx(false) ? 0 : -1;
1178}
1179
1180int32_t AudioCodingModuleImpl::PlayoutTimestamp(uint32_t* timestamp) {
1181 rtc::Optional<uint32_t> ts = PlayoutTimestamp();
1182 if (!ts)
1183 return -1;
1184 *timestamp = *ts;
1185 return 0;
1186}
1187
1188rtc::Optional<uint32_t> AudioCodingModuleImpl::PlayoutTimestamp() {
1189 return receiver_.GetPlayoutTimestamp();
1190}
1191
henrik.lundinb3f1c5d2016-08-22 15:39:53 -07001192int AudioCodingModuleImpl::FilteredCurrentDelayMs() const {
1193 return receiver_.FilteredCurrentDelayMs();
1194}
1195
kwibergc13ded52016-06-17 06:00:45 -07001196bool AudioCodingModuleImpl::HaveValidEncoder(const char* caller_name) const {
1197 if (!encoder_stack_) {
Mirko Bonadei675513b2017-11-09 11:09:25 +01001198 RTC_LOG(LS_ERROR) << caller_name << " failed: No send codec is registered.";
kwibergc13ded52016-06-17 06:00:45 -07001199 return false;
1200 }
1201 return true;
1202}
1203
1204int AudioCodingModuleImpl::UnregisterReceiveCodec(uint8_t payload_type) {
1205 return receiver_.RemoveCodec(payload_type);
1206}
1207
1208int AudioCodingModuleImpl::EnableNack(size_t max_nack_list_size) {
1209 return receiver_.EnableNack(max_nack_list_size);
1210}
1211
1212void AudioCodingModuleImpl::DisableNack() {
1213 receiver_.DisableNack();
1214}
1215
1216std::vector<uint16_t> AudioCodingModuleImpl::GetNackList(
1217 int64_t round_trip_time_ms) const {
1218 return receiver_.GetNackList(round_trip_time_ms);
1219}
1220
1221int AudioCodingModuleImpl::LeastRequiredDelayMs() const {
1222 return receiver_.LeastRequiredDelayMs();
1223}
1224
1225void AudioCodingModuleImpl::GetDecodingCallStatistics(
1226 AudioDecodingCallStats* call_stats) const {
1227 receiver_.GetDecodingCallStatistics(call_stats);
1228}
1229
ivoce1198e02017-09-08 08:13:19 -07001230ANAStats AudioCodingModuleImpl::GetANAStats() const {
1231 rtc::CritScope lock(&acm_crit_sect_);
1232 if (encoder_stack_)
1233 return encoder_stack_->GetANAStats();
1234 // If no encoder is set, return default stats.
1235 return ANAStats();
1236}
1237
kwibergc13ded52016-06-17 06:00:45 -07001238} // namespace
1239
kwiberg36a43882016-08-29 05:33:32 -07001240AudioCodingModule::Config::Config()
solenbergc7b4a452017-09-28 07:37:11 -07001241 : neteq_config(), clock(Clock::GetRealTimeClock()) {
kwiberg36a43882016-08-29 05:33:32 -07001242 // Post-decode VAD is disabled by default in NetEq, however, Audio
1243 // Conference Mixer relies on VAD decisions and fails without them.
1244 neteq_config.enable_post_decode_vad = true;
1245}
1246
1247AudioCodingModule::Config::Config(const Config&) = default;
1248AudioCodingModule::Config::~Config() = default;
1249
solenberge423a9de2017-09-27 11:28:14 -07001250AudioCodingModule* AudioCodingModule::Create(int id) {
solenbergc7b4a452017-09-28 07:37:11 -07001251 RTC_UNUSED(id);
1252 return Create();
1253}
1254
1255// Create module
1256AudioCodingModule* AudioCodingModule::Create() {
Henrik Lundin64dad832015-05-11 12:44:23 +02001257 Config config;
Henrik Lundin64dad832015-05-11 12:44:23 +02001258 config.clock = Clock::GetRealTimeClock();
ossue3525782016-05-25 07:37:43 -07001259 config.decoder_factory = CreateBuiltinAudioDecoderFactory();
Henrik Lundin64dad832015-05-11 12:44:23 +02001260 return Create(config);
turaj@webrtc.org532f3dc2013-09-19 00:12:23 +00001261}
1262
solenbergc7b4a452017-09-28 07:37:11 -07001263AudioCodingModule* AudioCodingModule::Create(Clock* clock) {
Henrik Lundin64dad832015-05-11 12:44:23 +02001264 Config config;
henrik.lundin@webrtc.org0bc9b5a2014-04-29 08:09:31 +00001265 config.clock = clock;
ossue3525782016-05-25 07:37:43 -07001266 config.decoder_factory = CreateBuiltinAudioDecoderFactory();
Henrik Lundin64dad832015-05-11 12:44:23 +02001267 return Create(config);
1268}
1269
1270AudioCodingModule* AudioCodingModule::Create(const Config& config) {
ossue3525782016-05-25 07:37:43 -07001271 if (!config.decoder_factory) {
1272 // TODO(ossu): Backwards compatibility. Will be removed after a deprecation
1273 // cycle.
1274 Config config_copy = config;
1275 config_copy.decoder_factory = CreateBuiltinAudioDecoderFactory();
kwibergc13ded52016-06-17 06:00:45 -07001276 return new AudioCodingModuleImpl(config_copy);
ossue3525782016-05-25 07:37:43 -07001277 }
kwibergc13ded52016-06-17 06:00:45 -07001278 return new AudioCodingModuleImpl(config);
turaj@webrtc.org7959e162013-09-12 18:30:26 +00001279}
1280
turaj@webrtc.org7959e162013-09-12 18:30:26 +00001281int AudioCodingModule::NumberOfCodecs() {
kwibergfce4a942015-10-27 11:40:24 -07001282 return static_cast<int>(acm2::RentACodec::NumberOfCodecs());
turaj@webrtc.org7959e162013-09-12 18:30:26 +00001283}
1284
turaj@webrtc.org7959e162013-09-12 18:30:26 +00001285int AudioCodingModule::Codec(int list_id, CodecInst* codec) {
kwibergfce4a942015-10-27 11:40:24 -07001286 auto codec_id = acm2::RentACodec::CodecIdFromIndex(list_id);
1287 if (!codec_id)
1288 return -1;
1289 auto ci = acm2::RentACodec::CodecInstById(*codec_id);
1290 if (!ci)
1291 return -1;
1292 *codec = *ci;
1293 return 0;
turaj@webrtc.org7959e162013-09-12 18:30:26 +00001294}
1295
turaj@webrtc.org7959e162013-09-12 18:30:26 +00001296int AudioCodingModule::Codec(const char* payload_name,
1297 CodecInst* codec,
1298 int sampling_freq_hz,
Peter Kasting69558702016-01-12 16:26:35 -08001299 size_t channels) {
Karl Wibergbe579832015-11-10 22:34:18 +01001300 rtc::Optional<CodecInst> ci = acm2::RentACodec::CodecInstByParams(
turaj@webrtc.org6d5d2482013-10-06 04:47:28 +00001301 payload_name, sampling_freq_hz, channels);
kwibergfce4a942015-10-27 11:40:24 -07001302 if (ci) {
1303 *codec = *ci;
1304 return 0;
1305 } else {
1306 // We couldn't find a matching codec, so set the parameters to unacceptable
turaj@webrtc.org7959e162013-09-12 18:30:26 +00001307 // values and return.
1308 codec->plname[0] = '\0';
1309 codec->pltype = -1;
1310 codec->pacsize = 0;
1311 codec->rate = 0;
1312 codec->plfreq = 0;
1313 return -1;
1314 }
turaj@webrtc.org7959e162013-09-12 18:30:26 +00001315}
1316
turaj@webrtc.org7959e162013-09-12 18:30:26 +00001317int AudioCodingModule::Codec(const char* payload_name,
1318 int sampling_freq_hz,
Peter Kasting69558702016-01-12 16:26:35 -08001319 size_t channels) {
Karl Wibergbe579832015-11-10 22:34:18 +01001320 rtc::Optional<acm2::RentACodec::CodecId> ci =
1321 acm2::RentACodec::CodecIdByParams(payload_name, sampling_freq_hz,
1322 channels);
kwibergfce4a942015-10-27 11:40:24 -07001323 if (!ci)
1324 return -1;
Karl Wibergbe579832015-11-10 22:34:18 +01001325 rtc::Optional<int> i = acm2::RentACodec::CodecIndexFromId(*ci);
kwibergfce4a942015-10-27 11:40:24 -07001326 return i ? *i : -1;
turaj@webrtc.org7959e162013-09-12 18:30:26 +00001327}
1328
1329// Checks the validity of the parameters of the given codec
1330bool AudioCodingModule::IsCodecValid(const CodecInst& codec) {
kwibergfce4a942015-10-27 11:40:24 -07001331 bool valid = acm2::RentACodec::IsCodecValid(codec);
1332 if (!valid)
Mirko Bonadei675513b2017-11-09 11:09:25 +01001333 RTC_LOG(LS_ERROR) << "Invalid codec setting";
kwibergfce4a942015-10-27 11:40:24 -07001334 return valid;
turaj@webrtc.org7959e162013-09-12 18:30:26 +00001335}
1336
turaj@webrtc.org7959e162013-09-12 18:30:26 +00001337} // namespace webrtc