blob: 0c296636f574cc4f92bf3fa073c31d540e437511 [file] [log] [blame]
turaj@webrtc.org7959e162013-09-12 18:30:26 +00001/*
2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020011#include "modules/audio_coding/include/audio_coding_module.h"
turaj@webrtc.org7959e162013-09-12 18:30:26 +000012
Jonathan Yu36344a02017-07-30 01:55:34 -070013#include <algorithm>
14
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020015#include "modules/audio_coding/acm2/acm_receiver.h"
16#include "modules/audio_coding/acm2/acm_resampler.h"
17#include "modules/audio_coding/acm2/codec_manager.h"
18#include "modules/audio_coding/acm2/rent_a_codec.h"
19#include "rtc_base/checks.h"
20#include "rtc_base/logging.h"
Karl Wiberge40468b2017-11-22 10:42:26 +010021#include "rtc_base/numerics/safe_conversions.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020022#include "system_wrappers/include/metrics.h"
turaj@webrtc.org7959e162013-09-12 18:30:26 +000023
24namespace webrtc {
25
kwibergc13ded52016-06-17 06:00:45 -070026namespace {
27
28struct EncoderFactory {
29 AudioEncoder* external_speech_encoder = nullptr;
30 acm2::CodecManager codec_manager;
31 acm2::RentACodec rent_a_codec;
32};
33
34class AudioCodingModuleImpl final : public AudioCodingModule {
35 public:
36 explicit AudioCodingModuleImpl(const AudioCodingModule::Config& config);
37 ~AudioCodingModuleImpl() override;
38
39 /////////////////////////////////////////
40 // Sender
41 //
42
43 // Can be called multiple times for Codec, CNG, RED.
44 int RegisterSendCodec(const CodecInst& send_codec) override;
45
46 void RegisterExternalSendCodec(
47 AudioEncoder* external_speech_encoder) override;
48
kwiberg24c7c122016-09-28 11:57:10 -070049 void ModifyEncoder(rtc::FunctionView<void(std::unique_ptr<AudioEncoder>*)>
50 modifier) override;
kwibergc13ded52016-06-17 06:00:45 -070051
kwiberg24c7c122016-09-28 11:57:10 -070052 void QueryEncoder(
53 rtc::FunctionView<void(const AudioEncoder*)> query) override;
ivoc85228d62016-07-27 04:53:47 -070054
kwibergc13ded52016-06-17 06:00:45 -070055 // Get current send codec.
56 rtc::Optional<CodecInst> SendCodec() const override;
57
58 // Get current send frequency.
59 int SendFrequency() const override;
60
61 // Sets the bitrate to the specified value in bits/sec. In case the codec does
62 // not support the requested value it will choose an appropriate value
63 // instead.
64 void SetBitRate(int bitrate_bps) override;
65
66 // Register a transport callback which will be
67 // called to deliver the encoded buffers.
68 int RegisterTransportCallback(AudioPacketizationCallback* transport) override;
69
70 // Add 10 ms of raw (PCM) audio data to the encoder.
71 int Add10MsData(const AudioFrame& audio_frame) override;
72
73 /////////////////////////////////////////
74 // (RED) Redundant Coding
75 //
76
77 // Configure RED status i.e. on/off.
78 int SetREDStatus(bool enable_red) override;
79
80 // Get RED status.
81 bool REDStatus() const override;
82
83 /////////////////////////////////////////
84 // (FEC) Forward Error Correction (codec internal)
85 //
86
87 // Configure FEC status i.e. on/off.
88 int SetCodecFEC(bool enabled_codec_fec) override;
89
90 // Get FEC status.
91 bool CodecFEC() const override;
92
93 // Set target packet loss rate
94 int SetPacketLossRate(int loss_rate) override;
95
96 /////////////////////////////////////////
97 // (VAD) Voice Activity Detection
98 // and
99 // (CNG) Comfort Noise Generation
100 //
101
102 int SetVAD(bool enable_dtx = true,
103 bool enable_vad = false,
104 ACMVADMode mode = VADNormal) override;
105
106 int VAD(bool* dtx_enabled,
107 bool* vad_enabled,
108 ACMVADMode* mode) const override;
109
110 int RegisterVADCallback(ACMVADCallback* vad_callback) override;
111
112 /////////////////////////////////////////
113 // Receiver
114 //
115
116 // Initialize receiver, resets codec database etc.
117 int InitializeReceiver() override;
118
119 // Get current receive frequency.
120 int ReceiveFrequency() const override;
121
122 // Get current playout frequency.
123 int PlayoutFrequency() const override;
124
kwiberg1c07c702017-03-27 07:15:49 -0700125 void SetReceiveCodecs(const std::map<int, SdpAudioFormat>& codecs) override;
126
kwiberg5adaf732016-10-04 09:33:27 -0700127 bool RegisterReceiveCodec(int rtp_payload_type,
128 const SdpAudioFormat& audio_format) override;
129
kwibergc13ded52016-06-17 06:00:45 -0700130 int RegisterReceiveCodec(const CodecInst& receive_codec) override;
131 int RegisterReceiveCodec(
132 const CodecInst& receive_codec,
kwiberg24c7c122016-09-28 11:57:10 -0700133 rtc::FunctionView<std::unique_ptr<AudioDecoder>()> isac_factory) override;
kwibergc13ded52016-06-17 06:00:45 -0700134
135 int RegisterExternalReceiveCodec(int rtp_payload_type,
136 AudioDecoder* external_decoder,
137 int sample_rate_hz,
138 int num_channels,
139 const std::string& name) override;
140
141 // Get current received codec.
142 int ReceiveCodec(CodecInst* current_codec) const override;
143
ossue280cde2016-10-12 11:04:10 -0700144 rtc::Optional<SdpAudioFormat> ReceiveFormat() const override;
145
kwibergc13ded52016-06-17 06:00:45 -0700146 // Incoming packet from network parsed and ready for decode.
147 int IncomingPacket(const uint8_t* incoming_payload,
148 const size_t payload_length,
149 const WebRtcRTPHeader& rtp_info) override;
150
kwibergc13ded52016-06-17 06:00:45 -0700151 // Minimum playout delay.
152 int SetMinimumPlayoutDelay(int time_ms) override;
153
154 // Maximum playout delay.
155 int SetMaximumPlayoutDelay(int time_ms) override;
156
157 // Smallest latency NetEq will maintain.
158 int LeastRequiredDelayMs() const override;
159
160 RTC_DEPRECATED int32_t PlayoutTimestamp(uint32_t* timestamp) override;
161
162 rtc::Optional<uint32_t> PlayoutTimestamp() override;
163
henrik.lundinb3f1c5d2016-08-22 15:39:53 -0700164 int FilteredCurrentDelayMs() const override;
165
Henrik Lundinabbff892017-11-29 09:14:04 +0100166 int TargetDelayMs() const override;
167
kwibergc13ded52016-06-17 06:00:45 -0700168 // Get 10 milliseconds of raw audio data to play out, and
169 // automatic resample to the requested frequency if > 0.
170 int PlayoutData10Ms(int desired_freq_hz,
171 AudioFrame* audio_frame,
172 bool* muted) override;
173 int PlayoutData10Ms(int desired_freq_hz, AudioFrame* audio_frame) override;
174
175 /////////////////////////////////////////
176 // Statistics
177 //
178
179 int GetNetworkStatistics(NetworkStatistics* statistics) override;
180
181 int SetOpusApplication(OpusApplicationMode application) override;
182
183 // If current send codec is Opus, informs it about the maximum playback rate
184 // the receiver will render.
185 int SetOpusMaxPlaybackRate(int frequency_hz) override;
186
187 int EnableOpusDtx() override;
188
189 int DisableOpusDtx() override;
190
191 int UnregisterReceiveCodec(uint8_t payload_type) override;
192
193 int EnableNack(size_t max_nack_list_size) override;
194
195 void DisableNack() override;
196
197 std::vector<uint16_t> GetNackList(int64_t round_trip_time_ms) const override;
198
199 void GetDecodingCallStatistics(AudioDecodingCallStats* stats) const override;
200
ivoce1198e02017-09-08 08:13:19 -0700201 ANAStats GetANAStats() const override;
202
kwibergc13ded52016-06-17 06:00:45 -0700203 private:
204 struct InputData {
205 uint32_t input_timestamp;
206 const int16_t* audio;
207 size_t length_per_channel;
208 size_t audio_channel;
209 // If a re-mix is required (up or down), this buffer will store a re-mixed
210 // version of the input.
211 int16_t buffer[WEBRTC_10MS_PCM_AUDIO];
212 };
213
214 // This member class writes values to the named UMA histogram, but only if
215 // the value has changed since the last time (and always for the first call).
216 class ChangeLogger {
217 public:
218 explicit ChangeLogger(const std::string& histogram_name)
219 : histogram_name_(histogram_name) {}
220 // Logs the new value if it is different from the last logged value, or if
221 // this is the first call.
222 void MaybeLog(int value);
223
224 private:
225 int last_value_ = 0;
226 int first_time_ = true;
227 const std::string histogram_name_;
228 };
229
230 int RegisterReceiveCodecUnlocked(
231 const CodecInst& codec,
kwiberg24c7c122016-09-28 11:57:10 -0700232 rtc::FunctionView<std::unique_ptr<AudioDecoder>()> isac_factory)
danilchap56359be2017-09-07 07:53:45 -0700233 RTC_EXCLUSIVE_LOCKS_REQUIRED(acm_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700234
235 int Add10MsDataInternal(const AudioFrame& audio_frame, InputData* input_data)
danilchap56359be2017-09-07 07:53:45 -0700236 RTC_EXCLUSIVE_LOCKS_REQUIRED(acm_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700237 int Encode(const InputData& input_data)
danilchap56359be2017-09-07 07:53:45 -0700238 RTC_EXCLUSIVE_LOCKS_REQUIRED(acm_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700239
danilchap56359be2017-09-07 07:53:45 -0700240 int InitializeReceiverSafe() RTC_EXCLUSIVE_LOCKS_REQUIRED(acm_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700241
242 bool HaveValidEncoder(const char* caller_name) const
danilchap56359be2017-09-07 07:53:45 -0700243 RTC_EXCLUSIVE_LOCKS_REQUIRED(acm_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700244
245 // Preprocessing of input audio, including resampling and down-mixing if
246 // required, before pushing audio into encoder's buffer.
247 //
248 // in_frame: input audio-frame
249 // ptr_out: pointer to output audio_frame. If no preprocessing is required
250 // |ptr_out| will be pointing to |in_frame|, otherwise pointing to
251 // |preprocess_frame_|.
252 //
253 // Return value:
254 // -1: if encountering an error.
255 // 0: otherwise.
256 int PreprocessToAddData(const AudioFrame& in_frame,
257 const AudioFrame** ptr_out)
danilchap56359be2017-09-07 07:53:45 -0700258 RTC_EXCLUSIVE_LOCKS_REQUIRED(acm_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700259
260 // Change required states after starting to receive the codec corresponding
261 // to |index|.
262 int UpdateUponReceivingCodec(int index);
263
264 rtc::CriticalSection acm_crit_sect_;
danilchap56359be2017-09-07 07:53:45 -0700265 rtc::Buffer encode_buffer_ RTC_GUARDED_BY(acm_crit_sect_);
danilchap56359be2017-09-07 07:53:45 -0700266 uint32_t expected_codec_ts_ RTC_GUARDED_BY(acm_crit_sect_);
267 uint32_t expected_in_ts_ RTC_GUARDED_BY(acm_crit_sect_);
268 acm2::ACMResampler resampler_ RTC_GUARDED_BY(acm_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700269 acm2::AcmReceiver receiver_; // AcmReceiver has it's own internal lock.
danilchap56359be2017-09-07 07:53:45 -0700270 ChangeLogger bitrate_logger_ RTC_GUARDED_BY(acm_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700271
danilchap56359be2017-09-07 07:53:45 -0700272 std::unique_ptr<EncoderFactory> encoder_factory_
273 RTC_GUARDED_BY(acm_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700274
275 // Current encoder stack, either obtained from
276 // encoder_factory_->rent_a_codec.RentEncoderStack or provided by a call to
277 // RegisterEncoder.
danilchap56359be2017-09-07 07:53:45 -0700278 std::unique_ptr<AudioEncoder> encoder_stack_ RTC_GUARDED_BY(acm_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700279
danilchap56359be2017-09-07 07:53:45 -0700280 std::unique_ptr<AudioDecoder> isac_decoder_16k_
281 RTC_GUARDED_BY(acm_crit_sect_);
282 std::unique_ptr<AudioDecoder> isac_decoder_32k_
283 RTC_GUARDED_BY(acm_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700284
285 // This is to keep track of CN instances where we can send DTMFs.
danilchap56359be2017-09-07 07:53:45 -0700286 uint8_t previous_pltype_ RTC_GUARDED_BY(acm_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700287
danilchap56359be2017-09-07 07:53:45 -0700288 bool receiver_initialized_ RTC_GUARDED_BY(acm_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700289
danilchap56359be2017-09-07 07:53:45 -0700290 AudioFrame preprocess_frame_ RTC_GUARDED_BY(acm_crit_sect_);
291 bool first_10ms_data_ RTC_GUARDED_BY(acm_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700292
danilchap56359be2017-09-07 07:53:45 -0700293 bool first_frame_ RTC_GUARDED_BY(acm_crit_sect_);
294 uint32_t last_timestamp_ RTC_GUARDED_BY(acm_crit_sect_);
295 uint32_t last_rtp_timestamp_ RTC_GUARDED_BY(acm_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700296
297 rtc::CriticalSection callback_crit_sect_;
298 AudioPacketizationCallback* packetization_callback_
danilchap56359be2017-09-07 07:53:45 -0700299 RTC_GUARDED_BY(callback_crit_sect_);
300 ACMVADCallback* vad_callback_ RTC_GUARDED_BY(callback_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700301
302 int codec_histogram_bins_log_[static_cast<size_t>(
303 AudioEncoder::CodecType::kMaxLoggedAudioCodecTypes)];
304 int number_of_consecutive_empty_packets_;
305};
306
307// Adds a codec usage sample to the histogram.
308void UpdateCodecTypeHistogram(size_t codec_type) {
309 RTC_HISTOGRAM_ENUMERATION(
310 "WebRTC.Audio.Encoder.CodecType", static_cast<int>(codec_type),
311 static_cast<int>(
312 webrtc::AudioEncoder::CodecType::kMaxLoggedAudioCodecTypes));
313}
314
kwibergc13ded52016-06-17 06:00:45 -0700315// Stereo-to-mono can be used as in-place.
316int DownMix(const AudioFrame& frame,
317 size_t length_out_buff,
318 int16_t* out_buff) {
yujo36b1a5f2017-06-12 12:45:32 -0700319 RTC_DCHECK_EQ(frame.num_channels_, 2);
320 RTC_DCHECK_GE(length_out_buff, frame.samples_per_channel_);
321
322 if (!frame.muted()) {
323 const int16_t* frame_data = frame.data();
324 for (size_t n = 0; n < frame.samples_per_channel_; ++n) {
325 out_buff[n] = static_cast<int16_t>(
326 (static_cast<int32_t>(frame_data[2 * n]) +
327 static_cast<int32_t>(frame_data[2 * n + 1])) >> 1);
328 }
329 } else {
Jonathan Yu36344a02017-07-30 01:55:34 -0700330 std::fill(out_buff, out_buff + frame.samples_per_channel_, 0);
kwibergc13ded52016-06-17 06:00:45 -0700331 }
kwibergc13ded52016-06-17 06:00:45 -0700332 return 0;
333}
334
335// Mono-to-stereo can be used as in-place.
336int UpMix(const AudioFrame& frame, size_t length_out_buff, int16_t* out_buff) {
yujo36b1a5f2017-06-12 12:45:32 -0700337 RTC_DCHECK_EQ(frame.num_channels_, 1);
338 RTC_DCHECK_GE(length_out_buff, 2 * frame.samples_per_channel_);
339
340 if (!frame.muted()) {
341 const int16_t* frame_data = frame.data();
342 for (size_t n = frame.samples_per_channel_; n != 0; --n) {
343 size_t i = n - 1;
344 int16_t sample = frame_data[i];
345 out_buff[2 * i + 1] = sample;
346 out_buff[2 * i] = sample;
347 }
348 } else {
Jonathan Yu36344a02017-07-30 01:55:34 -0700349 std::fill(out_buff, out_buff + frame.samples_per_channel_ * 2, 0);
kwibergc13ded52016-06-17 06:00:45 -0700350 }
351 return 0;
352}
353
354void ConvertEncodedInfoToFragmentationHeader(
355 const AudioEncoder::EncodedInfo& info,
356 RTPFragmentationHeader* frag) {
357 if (info.redundant.empty()) {
358 frag->fragmentationVectorSize = 0;
359 return;
360 }
361
362 frag->VerifyAndAllocateFragmentationHeader(
363 static_cast<uint16_t>(info.redundant.size()));
364 frag->fragmentationVectorSize = static_cast<uint16_t>(info.redundant.size());
365 size_t offset = 0;
366 for (size_t i = 0; i < info.redundant.size(); ++i) {
367 frag->fragmentationOffset[i] = offset;
368 offset += info.redundant[i].encoded_bytes;
369 frag->fragmentationLength[i] = info.redundant[i].encoded_bytes;
kwibergd3edd772017-03-01 18:52:48 -0800370 frag->fragmentationTimeDiff[i] = rtc::dchecked_cast<uint16_t>(
kwibergc13ded52016-06-17 06:00:45 -0700371 info.encoded_timestamp - info.redundant[i].encoded_timestamp);
372 frag->fragmentationPlType[i] = info.redundant[i].payload_type;
373 }
374}
375
376// Wraps a raw AudioEncoder pointer. The idea is that you can put one of these
377// in a unique_ptr, to protect the contained raw pointer from being deleted
378// when the unique_ptr expires. (This is of course a bad idea in general, but
379// backwards compatibility.)
380class RawAudioEncoderWrapper final : public AudioEncoder {
381 public:
382 RawAudioEncoderWrapper(AudioEncoder* enc) : enc_(enc) {}
383 int SampleRateHz() const override { return enc_->SampleRateHz(); }
384 size_t NumChannels() const override { return enc_->NumChannels(); }
385 int RtpTimestampRateHz() const override { return enc_->RtpTimestampRateHz(); }
386 size_t Num10MsFramesInNextPacket() const override {
387 return enc_->Num10MsFramesInNextPacket();
388 }
389 size_t Max10MsFramesInAPacket() const override {
390 return enc_->Max10MsFramesInAPacket();
391 }
392 int GetTargetBitrate() const override { return enc_->GetTargetBitrate(); }
393 EncodedInfo EncodeImpl(uint32_t rtp_timestamp,
394 rtc::ArrayView<const int16_t> audio,
395 rtc::Buffer* encoded) override {
396 return enc_->Encode(rtp_timestamp, audio, encoded);
397 }
398 void Reset() override { return enc_->Reset(); }
399 bool SetFec(bool enable) override { return enc_->SetFec(enable); }
400 bool SetDtx(bool enable) override { return enc_->SetDtx(enable); }
401 bool SetApplication(Application application) override {
402 return enc_->SetApplication(application);
403 }
404 void SetMaxPlaybackRate(int frequency_hz) override {
405 return enc_->SetMaxPlaybackRate(frequency_hz);
406 }
kwibergc13ded52016-06-17 06:00:45 -0700407
408 private:
409 AudioEncoder* enc_;
410};
411
412// Return false on error.
413bool CreateSpeechEncoderIfNecessary(EncoderFactory* ef) {
414 auto* sp = ef->codec_manager.GetStackParams();
415 if (sp->speech_encoder) {
416 // Do nothing; we already have a speech encoder.
417 } else if (ef->codec_manager.GetCodecInst()) {
418 RTC_DCHECK(!ef->external_speech_encoder);
419 // We have no speech encoder, but we have a specification for making one.
420 std::unique_ptr<AudioEncoder> enc =
421 ef->rent_a_codec.RentEncoder(*ef->codec_manager.GetCodecInst());
422 if (!enc)
423 return false; // Encoder spec was bad.
424 sp->speech_encoder = std::move(enc);
425 } else if (ef->external_speech_encoder) {
426 RTC_DCHECK(!ef->codec_manager.GetCodecInst());
427 // We have an external speech encoder.
428 sp->speech_encoder = std::unique_ptr<AudioEncoder>(
429 new RawAudioEncoderWrapper(ef->external_speech_encoder));
430 }
431 return true;
432}
433
434void AudioCodingModuleImpl::ChangeLogger::MaybeLog(int value) {
435 if (value != last_value_ || first_time_) {
436 first_time_ = false;
437 last_value_ = value;
438 RTC_HISTOGRAM_COUNTS_SPARSE_100(histogram_name_, value);
439 }
440}
441
442AudioCodingModuleImpl::AudioCodingModuleImpl(
443 const AudioCodingModule::Config& config)
solenbergc7b4a452017-09-28 07:37:11 -0700444 : expected_codec_ts_(0xD87F3F9F),
kwibergc13ded52016-06-17 06:00:45 -0700445 expected_in_ts_(0xD87F3F9F),
446 receiver_(config),
447 bitrate_logger_("WebRTC.Audio.TargetBitrateInKbps"),
448 encoder_factory_(new EncoderFactory),
449 encoder_stack_(nullptr),
450 previous_pltype_(255),
451 receiver_initialized_(false),
452 first_10ms_data_(false),
453 first_frame_(true),
454 packetization_callback_(NULL),
455 vad_callback_(NULL),
456 codec_histogram_bins_log_(),
457 number_of_consecutive_empty_packets_(0) {
458 if (InitializeReceiverSafe() < 0) {
Mirko Bonadei675513b2017-11-09 11:09:25 +0100459 RTC_LOG(LS_ERROR) << "Cannot initialize receiver";
kwibergc13ded52016-06-17 06:00:45 -0700460 }
Mirko Bonadei675513b2017-11-09 11:09:25 +0100461 RTC_LOG(LS_INFO) << "Created";
kwibergc13ded52016-06-17 06:00:45 -0700462}
463
464AudioCodingModuleImpl::~AudioCodingModuleImpl() = default;
465
466int32_t AudioCodingModuleImpl::Encode(const InputData& input_data) {
467 AudioEncoder::EncodedInfo encoded_info;
468 uint8_t previous_pltype;
469
470 // Check if there is an encoder before.
471 if (!HaveValidEncoder("Process"))
472 return -1;
473
ossu63fb95a2016-07-06 09:34:22 -0700474 if(!first_frame_) {
deadbeeffcada902016-08-24 12:45:13 -0700475 RTC_DCHECK(IsNewerTimestamp(input_data.input_timestamp, last_timestamp_))
ossu63fb95a2016-07-06 09:34:22 -0700476 << "Time should not move backwards";
477 }
478
kwibergc13ded52016-06-17 06:00:45 -0700479 // Scale the timestamp to the codec's RTP timestamp rate.
480 uint32_t rtp_timestamp =
481 first_frame_ ? input_data.input_timestamp
482 : last_rtp_timestamp_ +
483 rtc::CheckedDivExact(
484 input_data.input_timestamp - last_timestamp_,
485 static_cast<uint32_t>(rtc::CheckedDivExact(
486 encoder_stack_->SampleRateHz(),
487 encoder_stack_->RtpTimestampRateHz())));
488 last_timestamp_ = input_data.input_timestamp;
489 last_rtp_timestamp_ = rtp_timestamp;
490 first_frame_ = false;
491
492 // Clear the buffer before reuse - encoded data will get appended.
493 encode_buffer_.Clear();
494 encoded_info = encoder_stack_->Encode(
495 rtp_timestamp, rtc::ArrayView<const int16_t>(
496 input_data.audio, input_data.audio_channel *
497 input_data.length_per_channel),
498 &encode_buffer_);
499
500 bitrate_logger_.MaybeLog(encoder_stack_->GetTargetBitrate() / 1000);
501 if (encode_buffer_.size() == 0 && !encoded_info.send_even_if_empty) {
502 // Not enough data.
503 return 0;
504 }
505 previous_pltype = previous_pltype_; // Read it while we have the critsect.
506
507 // Log codec type to histogram once every 500 packets.
508 if (encoded_info.encoded_bytes == 0) {
509 ++number_of_consecutive_empty_packets_;
510 } else {
511 size_t codec_type = static_cast<size_t>(encoded_info.encoder_type);
512 codec_histogram_bins_log_[codec_type] +=
513 number_of_consecutive_empty_packets_ + 1;
514 number_of_consecutive_empty_packets_ = 0;
515 if (codec_histogram_bins_log_[codec_type] >= 500) {
516 codec_histogram_bins_log_[codec_type] -= 500;
517 UpdateCodecTypeHistogram(codec_type);
518 }
519 }
520
521 RTPFragmentationHeader my_fragmentation;
522 ConvertEncodedInfoToFragmentationHeader(encoded_info, &my_fragmentation);
523 FrameType frame_type;
524 if (encode_buffer_.size() == 0 && encoded_info.send_even_if_empty) {
525 frame_type = kEmptyFrame;
526 encoded_info.payload_type = previous_pltype;
527 } else {
kwibergaf476c72016-11-28 15:21:39 -0800528 RTC_DCHECK_GT(encode_buffer_.size(), 0);
kwibergc13ded52016-06-17 06:00:45 -0700529 frame_type = encoded_info.speech ? kAudioFrameSpeech : kAudioFrameCN;
530 }
531
532 {
533 rtc::CritScope lock(&callback_crit_sect_);
534 if (packetization_callback_) {
535 packetization_callback_->SendData(
536 frame_type, encoded_info.payload_type, encoded_info.encoded_timestamp,
537 encode_buffer_.data(), encode_buffer_.size(),
538 my_fragmentation.fragmentationVectorSize > 0 ? &my_fragmentation
539 : nullptr);
540 }
541
542 if (vad_callback_) {
543 // Callback with VAD decision.
544 vad_callback_->InFrameType(frame_type);
545 }
546 }
547 previous_pltype_ = encoded_info.payload_type;
548 return static_cast<int32_t>(encode_buffer_.size());
549}
550
551/////////////////////////////////////////
552// Sender
553//
554
555// Can be called multiple times for Codec, CNG, RED.
556int AudioCodingModuleImpl::RegisterSendCodec(const CodecInst& send_codec) {
557 rtc::CritScope lock(&acm_crit_sect_);
558 if (!encoder_factory_->codec_manager.RegisterEncoder(send_codec)) {
559 return -1;
560 }
561 if (encoder_factory_->codec_manager.GetCodecInst()) {
562 encoder_factory_->external_speech_encoder = nullptr;
563 }
564 if (!CreateSpeechEncoderIfNecessary(encoder_factory_.get())) {
565 return -1;
566 }
567 auto* sp = encoder_factory_->codec_manager.GetStackParams();
568 if (sp->speech_encoder)
569 encoder_stack_ = encoder_factory_->rent_a_codec.RentEncoderStack(sp);
570 return 0;
571}
572
573void AudioCodingModuleImpl::RegisterExternalSendCodec(
574 AudioEncoder* external_speech_encoder) {
575 rtc::CritScope lock(&acm_crit_sect_);
576 encoder_factory_->codec_manager.UnsetCodecInst();
577 encoder_factory_->external_speech_encoder = external_speech_encoder;
578 RTC_CHECK(CreateSpeechEncoderIfNecessary(encoder_factory_.get()));
579 auto* sp = encoder_factory_->codec_manager.GetStackParams();
580 RTC_CHECK(sp->speech_encoder);
581 encoder_stack_ = encoder_factory_->rent_a_codec.RentEncoderStack(sp);
582}
583
584void AudioCodingModuleImpl::ModifyEncoder(
kwiberg24c7c122016-09-28 11:57:10 -0700585 rtc::FunctionView<void(std::unique_ptr<AudioEncoder>*)> modifier) {
kwibergc13ded52016-06-17 06:00:45 -0700586 rtc::CritScope lock(&acm_crit_sect_);
587
588 // Wipe the encoder factory, so that everything that relies on it will fail.
589 // We don't want the complexity of supporting swapping back and forth.
590 if (encoder_factory_) {
591 encoder_factory_.reset();
592 RTC_CHECK(!encoder_stack_); // Ensure we hadn't started using the factory.
593 }
594
595 modifier(&encoder_stack_);
596}
597
ivoc85228d62016-07-27 04:53:47 -0700598void AudioCodingModuleImpl::QueryEncoder(
kwiberg24c7c122016-09-28 11:57:10 -0700599 rtc::FunctionView<void(const AudioEncoder*)> query) {
ivoc85228d62016-07-27 04:53:47 -0700600 rtc::CritScope lock(&acm_crit_sect_);
601 query(encoder_stack_.get());
602}
603
kwibergc13ded52016-06-17 06:00:45 -0700604// Get current send codec.
605rtc::Optional<CodecInst> AudioCodingModuleImpl::SendCodec() const {
606 rtc::CritScope lock(&acm_crit_sect_);
607 if (encoder_factory_) {
608 auto* ci = encoder_factory_->codec_manager.GetCodecInst();
609 if (ci) {
Oskar Sundbom12ab00b2017-11-16 15:31:38 +0100610 return *ci;
kwibergc13ded52016-06-17 06:00:45 -0700611 }
612 CreateSpeechEncoderIfNecessary(encoder_factory_.get());
613 const std::unique_ptr<AudioEncoder>& enc =
614 encoder_factory_->codec_manager.GetStackParams()->speech_encoder;
615 if (enc) {
Oskar Sundbom12ab00b2017-11-16 15:31:38 +0100616 return acm2::CodecManager::ForgeCodecInst(enc.get());
kwibergc13ded52016-06-17 06:00:45 -0700617 }
Oskar Sundbom12ab00b2017-11-16 15:31:38 +0100618 return rtc::nullopt;
kwibergc13ded52016-06-17 06:00:45 -0700619 } else {
620 return encoder_stack_
621 ? rtc::Optional<CodecInst>(
622 acm2::CodecManager::ForgeCodecInst(encoder_stack_.get()))
Oskar Sundbom12ab00b2017-11-16 15:31:38 +0100623 : rtc::nullopt;
kwibergc13ded52016-06-17 06:00:45 -0700624 }
625}
626
627// Get current send frequency.
628int AudioCodingModuleImpl::SendFrequency() const {
kwibergc13ded52016-06-17 06:00:45 -0700629 rtc::CritScope lock(&acm_crit_sect_);
630
631 if (!encoder_stack_) {
Mirko Bonadei675513b2017-11-09 11:09:25 +0100632 RTC_LOG(LS_ERROR) << "SendFrequency Failed, no codec is registered";
kwibergc13ded52016-06-17 06:00:45 -0700633 return -1;
634 }
635
636 return encoder_stack_->SampleRateHz();
637}
638
639void AudioCodingModuleImpl::SetBitRate(int bitrate_bps) {
640 rtc::CritScope lock(&acm_crit_sect_);
641 if (encoder_stack_) {
Oskar Sundbom12ab00b2017-11-16 15:31:38 +0100642 encoder_stack_->OnReceivedUplinkBandwidth(bitrate_bps, rtc::nullopt);
kwibergc13ded52016-06-17 06:00:45 -0700643 }
644}
645
646// Register a transport callback which will be called to deliver
647// the encoded buffers.
648int AudioCodingModuleImpl::RegisterTransportCallback(
649 AudioPacketizationCallback* transport) {
650 rtc::CritScope lock(&callback_crit_sect_);
651 packetization_callback_ = transport;
652 return 0;
653}
654
655// Add 10MS of raw (PCM) audio data to the encoder.
656int AudioCodingModuleImpl::Add10MsData(const AudioFrame& audio_frame) {
657 InputData input_data;
658 rtc::CritScope lock(&acm_crit_sect_);
659 int r = Add10MsDataInternal(audio_frame, &input_data);
660 return r < 0 ? r : Encode(input_data);
661}
662
663int AudioCodingModuleImpl::Add10MsDataInternal(const AudioFrame& audio_frame,
664 InputData* input_data) {
665 if (audio_frame.samples_per_channel_ == 0) {
666 assert(false);
Mirko Bonadei675513b2017-11-09 11:09:25 +0100667 RTC_LOG(LS_ERROR) << "Cannot Add 10 ms audio, payload length is zero";
kwibergc13ded52016-06-17 06:00:45 -0700668 return -1;
669 }
670
671 if (audio_frame.sample_rate_hz_ > 48000) {
672 assert(false);
Mirko Bonadei675513b2017-11-09 11:09:25 +0100673 RTC_LOG(LS_ERROR) << "Cannot Add 10 ms audio, input frequency not valid";
kwibergc13ded52016-06-17 06:00:45 -0700674 return -1;
675 }
676
677 // If the length and frequency matches. We currently just support raw PCM.
678 if (static_cast<size_t>(audio_frame.sample_rate_hz_ / 100) !=
679 audio_frame.samples_per_channel_) {
Mirko Bonadei675513b2017-11-09 11:09:25 +0100680 RTC_LOG(LS_ERROR)
Alex Loiko300ec8c2017-05-30 17:23:28 +0200681 << "Cannot Add 10 ms audio, input frequency and length doesn't match";
kwibergc13ded52016-06-17 06:00:45 -0700682 return -1;
683 }
684
685 if (audio_frame.num_channels_ != 1 && audio_frame.num_channels_ != 2) {
Mirko Bonadei675513b2017-11-09 11:09:25 +0100686 RTC_LOG(LS_ERROR) << "Cannot Add 10 ms audio, invalid number of channels.";
kwibergc13ded52016-06-17 06:00:45 -0700687 return -1;
688 }
689
690 // Do we have a codec registered?
691 if (!HaveValidEncoder("Add10MsData")) {
692 return -1;
693 }
694
695 const AudioFrame* ptr_frame;
696 // Perform a resampling, also down-mix if it is required and can be
697 // performed before resampling (a down mix prior to resampling will take
698 // place if both primary and secondary encoders are mono and input is in
699 // stereo).
700 if (PreprocessToAddData(audio_frame, &ptr_frame) < 0) {
701 return -1;
702 }
703
704 // Check whether we need an up-mix or down-mix?
705 const size_t current_num_channels = encoder_stack_->NumChannels();
706 const bool same_num_channels =
707 ptr_frame->num_channels_ == current_num_channels;
708
709 if (!same_num_channels) {
710 if (ptr_frame->num_channels_ == 1) {
711 if (UpMix(*ptr_frame, WEBRTC_10MS_PCM_AUDIO, input_data->buffer) < 0)
712 return -1;
713 } else {
714 if (DownMix(*ptr_frame, WEBRTC_10MS_PCM_AUDIO, input_data->buffer) < 0)
715 return -1;
716 }
717 }
718
719 // When adding data to encoders this pointer is pointing to an audio buffer
720 // with correct number of channels.
yujo36b1a5f2017-06-12 12:45:32 -0700721 const int16_t* ptr_audio = ptr_frame->data();
kwibergc13ded52016-06-17 06:00:45 -0700722
723 // For pushing data to primary, point the |ptr_audio| to correct buffer.
724 if (!same_num_channels)
725 ptr_audio = input_data->buffer;
726
yujo36b1a5f2017-06-12 12:45:32 -0700727 // TODO(yujo): Skip encode of muted frames.
kwibergc13ded52016-06-17 06:00:45 -0700728 input_data->input_timestamp = ptr_frame->timestamp_;
729 input_data->audio = ptr_audio;
730 input_data->length_per_channel = ptr_frame->samples_per_channel_;
731 input_data->audio_channel = current_num_channels;
732
733 return 0;
734}
735
736// Perform a resampling and down-mix if required. We down-mix only if
737// encoder is mono and input is stereo. In case of dual-streaming, both
738// encoders has to be mono for down-mix to take place.
739// |*ptr_out| will point to the pre-processed audio-frame. If no pre-processing
740// is required, |*ptr_out| points to |in_frame|.
yujo36b1a5f2017-06-12 12:45:32 -0700741// TODO(yujo): Make this more efficient for muted frames.
kwibergc13ded52016-06-17 06:00:45 -0700742int AudioCodingModuleImpl::PreprocessToAddData(const AudioFrame& in_frame,
743 const AudioFrame** ptr_out) {
744 const bool resample =
745 in_frame.sample_rate_hz_ != encoder_stack_->SampleRateHz();
746
747 // This variable is true if primary codec and secondary codec (if exists)
748 // are both mono and input is stereo.
749 // TODO(henrik.lundin): This condition should probably be
750 // in_frame.num_channels_ > encoder_stack_->NumChannels()
751 const bool down_mix =
752 in_frame.num_channels_ == 2 && encoder_stack_->NumChannels() == 1;
753
754 if (!first_10ms_data_) {
755 expected_in_ts_ = in_frame.timestamp_;
756 expected_codec_ts_ = in_frame.timestamp_;
757 first_10ms_data_ = true;
758 } else if (in_frame.timestamp_ != expected_in_ts_) {
Mirko Bonadei675513b2017-11-09 11:09:25 +0100759 RTC_LOG(LS_WARNING) << "Unexpected input timestamp: " << in_frame.timestamp_
760 << ", expected: " << expected_in_ts_;
kwibergc13ded52016-06-17 06:00:45 -0700761 expected_codec_ts_ +=
762 (in_frame.timestamp_ - expected_in_ts_) *
763 static_cast<uint32_t>(
764 static_cast<double>(encoder_stack_->SampleRateHz()) /
765 static_cast<double>(in_frame.sample_rate_hz_));
766 expected_in_ts_ = in_frame.timestamp_;
767 }
768
769
770 if (!down_mix && !resample) {
771 // No pre-processing is required.
ossu63fb95a2016-07-06 09:34:22 -0700772 if (expected_in_ts_ == expected_codec_ts_) {
773 // If we've never resampled, we can use the input frame as-is
774 *ptr_out = &in_frame;
775 } else {
776 // Otherwise we'll need to alter the timestamp. Since in_frame is const,
777 // we'll have to make a copy of it.
778 preprocess_frame_.CopyFrom(in_frame);
779 preprocess_frame_.timestamp_ = expected_codec_ts_;
780 *ptr_out = &preprocess_frame_;
781 }
782
kwibergc13ded52016-06-17 06:00:45 -0700783 expected_in_ts_ += static_cast<uint32_t>(in_frame.samples_per_channel_);
784 expected_codec_ts_ += static_cast<uint32_t>(in_frame.samples_per_channel_);
kwibergc13ded52016-06-17 06:00:45 -0700785 return 0;
786 }
787
788 *ptr_out = &preprocess_frame_;
789 preprocess_frame_.num_channels_ = in_frame.num_channels_;
790 int16_t audio[WEBRTC_10MS_PCM_AUDIO];
yujo36b1a5f2017-06-12 12:45:32 -0700791 const int16_t* src_ptr_audio = in_frame.data();
kwibergc13ded52016-06-17 06:00:45 -0700792 if (down_mix) {
793 // If a resampling is required the output of a down-mix is written into a
794 // local buffer, otherwise, it will be written to the output frame.
yujo36b1a5f2017-06-12 12:45:32 -0700795 int16_t* dest_ptr_audio = resample ?
796 audio : preprocess_frame_.mutable_data();
kwibergc13ded52016-06-17 06:00:45 -0700797 if (DownMix(in_frame, WEBRTC_10MS_PCM_AUDIO, dest_ptr_audio) < 0)
798 return -1;
799 preprocess_frame_.num_channels_ = 1;
800 // Set the input of the resampler is the down-mixed signal.
801 src_ptr_audio = audio;
802 }
803
804 preprocess_frame_.timestamp_ = expected_codec_ts_;
805 preprocess_frame_.samples_per_channel_ = in_frame.samples_per_channel_;
806 preprocess_frame_.sample_rate_hz_ = in_frame.sample_rate_hz_;
807 // If it is required, we have to do a resampling.
808 if (resample) {
809 // The result of the resampler is written to output frame.
yujo36b1a5f2017-06-12 12:45:32 -0700810 int16_t* dest_ptr_audio = preprocess_frame_.mutable_data();
kwibergc13ded52016-06-17 06:00:45 -0700811
812 int samples_per_channel = resampler_.Resample10Msec(
813 src_ptr_audio, in_frame.sample_rate_hz_, encoder_stack_->SampleRateHz(),
814 preprocess_frame_.num_channels_, AudioFrame::kMaxDataSizeSamples,
815 dest_ptr_audio);
816
817 if (samples_per_channel < 0) {
Mirko Bonadei675513b2017-11-09 11:09:25 +0100818 RTC_LOG(LS_ERROR) << "Cannot add 10 ms audio, resampling failed";
kwibergc13ded52016-06-17 06:00:45 -0700819 return -1;
820 }
821 preprocess_frame_.samples_per_channel_ =
822 static_cast<size_t>(samples_per_channel);
823 preprocess_frame_.sample_rate_hz_ = encoder_stack_->SampleRateHz();
824 }
825
826 expected_codec_ts_ +=
827 static_cast<uint32_t>(preprocess_frame_.samples_per_channel_);
828 expected_in_ts_ += static_cast<uint32_t>(in_frame.samples_per_channel_);
829
830 return 0;
831}
832
833/////////////////////////////////////////
834// (RED) Redundant Coding
835//
836
837bool AudioCodingModuleImpl::REDStatus() const {
838 rtc::CritScope lock(&acm_crit_sect_);
839 return encoder_factory_->codec_manager.GetStackParams()->use_red;
840}
841
842// Configure RED status i.e on/off.
843int AudioCodingModuleImpl::SetREDStatus(bool enable_red) {
844#ifdef WEBRTC_CODEC_RED
845 rtc::CritScope lock(&acm_crit_sect_);
846 CreateSpeechEncoderIfNecessary(encoder_factory_.get());
847 if (!encoder_factory_->codec_manager.SetCopyRed(enable_red)) {
848 return -1;
849 }
850 auto* sp = encoder_factory_->codec_manager.GetStackParams();
851 if (sp->speech_encoder)
852 encoder_stack_ = encoder_factory_->rent_a_codec.RentEncoderStack(sp);
853 return 0;
854#else
Mirko Bonadei675513b2017-11-09 11:09:25 +0100855 RTC_LOG(LS_WARNING) << " WEBRTC_CODEC_RED is undefined";
kwibergc13ded52016-06-17 06:00:45 -0700856 return -1;
857#endif
858}
859
860/////////////////////////////////////////
861// (FEC) Forward Error Correction (codec internal)
862//
863
864bool AudioCodingModuleImpl::CodecFEC() const {
865 rtc::CritScope lock(&acm_crit_sect_);
866 return encoder_factory_->codec_manager.GetStackParams()->use_codec_fec;
867}
868
869int AudioCodingModuleImpl::SetCodecFEC(bool enable_codec_fec) {
870 rtc::CritScope lock(&acm_crit_sect_);
871 CreateSpeechEncoderIfNecessary(encoder_factory_.get());
872 if (!encoder_factory_->codec_manager.SetCodecFEC(enable_codec_fec)) {
873 return -1;
874 }
875 auto* sp = encoder_factory_->codec_manager.GetStackParams();
876 if (sp->speech_encoder)
877 encoder_stack_ = encoder_factory_->rent_a_codec.RentEncoderStack(sp);
878 if (enable_codec_fec) {
879 return sp->use_codec_fec ? 0 : -1;
880 } else {
881 RTC_DCHECK(!sp->use_codec_fec);
882 return 0;
883 }
884}
885
886int AudioCodingModuleImpl::SetPacketLossRate(int loss_rate) {
887 rtc::CritScope lock(&acm_crit_sect_);
888 if (HaveValidEncoder("SetPacketLossRate")) {
minyue4b9a2cb2016-11-30 06:49:59 -0800889 encoder_stack_->OnReceivedUplinkPacketLossFraction(loss_rate / 100.0);
kwibergc13ded52016-06-17 06:00:45 -0700890 }
891 return 0;
892}
893
894/////////////////////////////////////////
895// (VAD) Voice Activity Detection
896//
897int AudioCodingModuleImpl::SetVAD(bool enable_dtx,
898 bool enable_vad,
899 ACMVADMode mode) {
900 // Note: |enable_vad| is not used; VAD is enabled based on the DTX setting.
901 RTC_DCHECK_EQ(enable_dtx, enable_vad);
902 rtc::CritScope lock(&acm_crit_sect_);
903 CreateSpeechEncoderIfNecessary(encoder_factory_.get());
904 if (!encoder_factory_->codec_manager.SetVAD(enable_dtx, mode)) {
905 return -1;
906 }
907 auto* sp = encoder_factory_->codec_manager.GetStackParams();
908 if (sp->speech_encoder)
909 encoder_stack_ = encoder_factory_->rent_a_codec.RentEncoderStack(sp);
910 return 0;
911}
912
913// Get VAD/DTX settings.
914int AudioCodingModuleImpl::VAD(bool* dtx_enabled, bool* vad_enabled,
915 ACMVADMode* mode) const {
916 rtc::CritScope lock(&acm_crit_sect_);
917 const auto* sp = encoder_factory_->codec_manager.GetStackParams();
918 *dtx_enabled = *vad_enabled = sp->use_cng;
919 *mode = sp->vad_mode;
920 return 0;
921}
922
923/////////////////////////////////////////
924// Receiver
925//
926
927int AudioCodingModuleImpl::InitializeReceiver() {
928 rtc::CritScope lock(&acm_crit_sect_);
929 return InitializeReceiverSafe();
930}
931
932// Initialize receiver, resets codec database etc.
933int AudioCodingModuleImpl::InitializeReceiverSafe() {
934 // If the receiver is already initialized then we want to destroy any
935 // existing decoders. After a call to this function, we should have a clean
936 // start-up.
kwiberg6b19b562016-09-20 04:02:25 -0700937 if (receiver_initialized_)
938 receiver_.RemoveAllCodecs();
kwibergc13ded52016-06-17 06:00:45 -0700939 receiver_.ResetInitialDelay();
940 receiver_.SetMinimumDelay(0);
941 receiver_.SetMaximumDelay(0);
942 receiver_.FlushBuffers();
943
kwibergc13ded52016-06-17 06:00:45 -0700944 receiver_initialized_ = true;
945 return 0;
946}
947
948// Get current receive frequency.
949int AudioCodingModuleImpl::ReceiveFrequency() const {
950 const auto last_packet_sample_rate = receiver_.last_packet_sample_rate_hz();
951 return last_packet_sample_rate ? *last_packet_sample_rate
952 : receiver_.last_output_sample_rate_hz();
953}
954
955// Get current playout frequency.
956int AudioCodingModuleImpl::PlayoutFrequency() const {
kwibergc13ded52016-06-17 06:00:45 -0700957 return receiver_.last_output_sample_rate_hz();
958}
959
kwiberg1c07c702017-03-27 07:15:49 -0700960void AudioCodingModuleImpl::SetReceiveCodecs(
961 const std::map<int, SdpAudioFormat>& codecs) {
962 rtc::CritScope lock(&acm_crit_sect_);
963 receiver_.SetCodecs(codecs);
964}
965
kwiberg5adaf732016-10-04 09:33:27 -0700966bool AudioCodingModuleImpl::RegisterReceiveCodec(
967 int rtp_payload_type,
968 const SdpAudioFormat& audio_format) {
969 rtc::CritScope lock(&acm_crit_sect_);
970 RTC_DCHECK(receiver_initialized_);
971
972 if (!acm2::RentACodec::IsPayloadTypeValid(rtp_payload_type)) {
Mirko Bonadei675513b2017-11-09 11:09:25 +0100973 RTC_LOG_F(LS_ERROR) << "Invalid payload-type " << rtp_payload_type
974 << " for decoder.";
kwiberg5adaf732016-10-04 09:33:27 -0700975 return false;
976 }
977
978 return receiver_.AddCodec(rtp_payload_type, audio_format);
979}
980
kwibergc13ded52016-06-17 06:00:45 -0700981int AudioCodingModuleImpl::RegisterReceiveCodec(const CodecInst& codec) {
982 rtc::CritScope lock(&acm_crit_sect_);
983 auto* ef = encoder_factory_.get();
984 return RegisterReceiveCodecUnlocked(
985 codec, [&] { return ef->rent_a_codec.RentIsacDecoder(codec.plfreq); });
986}
987
988int AudioCodingModuleImpl::RegisterReceiveCodec(
989 const CodecInst& codec,
kwiberg24c7c122016-09-28 11:57:10 -0700990 rtc::FunctionView<std::unique_ptr<AudioDecoder>()> isac_factory) {
kwibergc13ded52016-06-17 06:00:45 -0700991 rtc::CritScope lock(&acm_crit_sect_);
992 return RegisterReceiveCodecUnlocked(codec, isac_factory);
993}
994
995int AudioCodingModuleImpl::RegisterReceiveCodecUnlocked(
996 const CodecInst& codec,
kwiberg24c7c122016-09-28 11:57:10 -0700997 rtc::FunctionView<std::unique_ptr<AudioDecoder>()> isac_factory) {
kwibergc13ded52016-06-17 06:00:45 -0700998 RTC_DCHECK(receiver_initialized_);
999 if (codec.channels > 2) {
Mirko Bonadei675513b2017-11-09 11:09:25 +01001000 RTC_LOG_F(LS_ERROR) << "Unsupported number of channels: " << codec.channels;
kwibergc13ded52016-06-17 06:00:45 -07001001 return -1;
1002 }
1003
1004 auto codec_id = acm2::RentACodec::CodecIdByParams(codec.plname, codec.plfreq,
1005 codec.channels);
1006 if (!codec_id) {
Mirko Bonadei675513b2017-11-09 11:09:25 +01001007 RTC_LOG_F(LS_ERROR)
1008 << "Wrong codec params to be registered as receive codec";
kwibergc13ded52016-06-17 06:00:45 -07001009 return -1;
1010 }
1011 auto codec_index = acm2::RentACodec::CodecIndexFromId(*codec_id);
1012 RTC_CHECK(codec_index) << "Invalid codec ID: " << static_cast<int>(*codec_id);
1013
1014 // Check if the payload-type is valid.
1015 if (!acm2::RentACodec::IsPayloadTypeValid(codec.pltype)) {
Mirko Bonadei675513b2017-11-09 11:09:25 +01001016 RTC_LOG_F(LS_ERROR) << "Invalid payload type " << codec.pltype << " for "
1017 << codec.plname;
kwibergc13ded52016-06-17 06:00:45 -07001018 return -1;
1019 }
1020
1021 AudioDecoder* isac_decoder = nullptr;
1022 if (STR_CASE_CMP(codec.plname, "isac") == 0) {
1023 std::unique_ptr<AudioDecoder>& saved_isac_decoder =
1024 codec.plfreq == 16000 ? isac_decoder_16k_ : isac_decoder_32k_;
1025 if (!saved_isac_decoder) {
1026 saved_isac_decoder = isac_factory();
1027 }
1028 isac_decoder = saved_isac_decoder.get();
1029 }
1030 return receiver_.AddCodec(*codec_index, codec.pltype, codec.channels,
1031 codec.plfreq, isac_decoder, codec.plname);
1032}
1033
1034int AudioCodingModuleImpl::RegisterExternalReceiveCodec(
1035 int rtp_payload_type,
1036 AudioDecoder* external_decoder,
1037 int sample_rate_hz,
1038 int num_channels,
1039 const std::string& name) {
1040 rtc::CritScope lock(&acm_crit_sect_);
1041 RTC_DCHECK(receiver_initialized_);
1042 if (num_channels > 2 || num_channels < 0) {
Mirko Bonadei675513b2017-11-09 11:09:25 +01001043 RTC_LOG_F(LS_ERROR) << "Unsupported number of channels: " << num_channels;
kwibergc13ded52016-06-17 06:00:45 -07001044 return -1;
1045 }
1046
1047 // Check if the payload-type is valid.
1048 if (!acm2::RentACodec::IsPayloadTypeValid(rtp_payload_type)) {
Mirko Bonadei675513b2017-11-09 11:09:25 +01001049 RTC_LOG_F(LS_ERROR) << "Invalid payload-type " << rtp_payload_type
1050 << " for external decoder.";
kwibergc13ded52016-06-17 06:00:45 -07001051 return -1;
1052 }
1053
1054 return receiver_.AddCodec(-1 /* external */, rtp_payload_type, num_channels,
1055 sample_rate_hz, external_decoder, name);
1056}
1057
1058// Get current received codec.
1059int AudioCodingModuleImpl::ReceiveCodec(CodecInst* current_codec) const {
1060 rtc::CritScope lock(&acm_crit_sect_);
1061 return receiver_.LastAudioCodec(current_codec);
1062}
1063
ossue280cde2016-10-12 11:04:10 -07001064rtc::Optional<SdpAudioFormat> AudioCodingModuleImpl::ReceiveFormat() const {
1065 rtc::CritScope lock(&acm_crit_sect_);
1066 return receiver_.LastAudioFormat();
1067}
1068
kwibergc13ded52016-06-17 06:00:45 -07001069// Incoming packet from network parsed and ready for decode.
1070int AudioCodingModuleImpl::IncomingPacket(const uint8_t* incoming_payload,
1071 const size_t payload_length,
1072 const WebRtcRTPHeader& rtp_header) {
henrik.lundinb8c55b12017-05-10 07:38:01 -07001073 RTC_DCHECK_EQ(payload_length == 0, incoming_payload == nullptr);
kwibergc13ded52016-06-17 06:00:45 -07001074 return receiver_.InsertPacket(
1075 rtp_header,
1076 rtc::ArrayView<const uint8_t>(incoming_payload, payload_length));
1077}
1078
1079// Minimum playout delay (Used for lip-sync).
1080int AudioCodingModuleImpl::SetMinimumPlayoutDelay(int time_ms) {
1081 if ((time_ms < 0) || (time_ms > 10000)) {
Mirko Bonadei675513b2017-11-09 11:09:25 +01001082 RTC_LOG(LS_ERROR) << "Delay must be in the range of 0-10000 milliseconds.";
kwibergc13ded52016-06-17 06:00:45 -07001083 return -1;
1084 }
1085 return receiver_.SetMinimumDelay(time_ms);
1086}
1087
1088int AudioCodingModuleImpl::SetMaximumPlayoutDelay(int time_ms) {
1089 if ((time_ms < 0) || (time_ms > 10000)) {
Mirko Bonadei675513b2017-11-09 11:09:25 +01001090 RTC_LOG(LS_ERROR) << "Delay must be in the range of 0-10000 milliseconds.";
kwibergc13ded52016-06-17 06:00:45 -07001091 return -1;
1092 }
1093 return receiver_.SetMaximumDelay(time_ms);
1094}
1095
1096// Get 10 milliseconds of raw audio data to play out.
1097// Automatic resample to the requested frequency.
1098int AudioCodingModuleImpl::PlayoutData10Ms(int desired_freq_hz,
1099 AudioFrame* audio_frame,
1100 bool* muted) {
1101 // GetAudio always returns 10 ms, at the requested sample rate.
1102 if (receiver_.GetAudio(desired_freq_hz, audio_frame, muted) != 0) {
Mirko Bonadei675513b2017-11-09 11:09:25 +01001103 RTC_LOG(LS_ERROR) << "PlayoutData failed, RecOut Failed";
kwibergc13ded52016-06-17 06:00:45 -07001104 return -1;
1105 }
kwibergc13ded52016-06-17 06:00:45 -07001106 return 0;
1107}
1108
1109int AudioCodingModuleImpl::PlayoutData10Ms(int desired_freq_hz,
1110 AudioFrame* audio_frame) {
1111 bool muted;
1112 int ret = PlayoutData10Ms(desired_freq_hz, audio_frame, &muted);
1113 RTC_DCHECK(!muted);
1114 return ret;
1115}
1116
1117/////////////////////////////////////////
1118// Statistics
1119//
1120
1121// TODO(turajs) change the return value to void. Also change the corresponding
1122// NetEq function.
1123int AudioCodingModuleImpl::GetNetworkStatistics(NetworkStatistics* statistics) {
1124 receiver_.GetNetworkStatistics(statistics);
1125 return 0;
1126}
1127
1128int AudioCodingModuleImpl::RegisterVADCallback(ACMVADCallback* vad_callback) {
Mirko Bonadei675513b2017-11-09 11:09:25 +01001129 RTC_LOG(LS_VERBOSE) << "RegisterVADCallback()";
kwibergc13ded52016-06-17 06:00:45 -07001130 rtc::CritScope lock(&callback_crit_sect_);
1131 vad_callback_ = vad_callback;
1132 return 0;
1133}
1134
kwibergc13ded52016-06-17 06:00:45 -07001135int AudioCodingModuleImpl::SetOpusApplication(OpusApplicationMode application) {
1136 rtc::CritScope lock(&acm_crit_sect_);
1137 if (!HaveValidEncoder("SetOpusApplication")) {
1138 return -1;
1139 }
1140 AudioEncoder::Application app;
1141 switch (application) {
1142 case kVoip:
1143 app = AudioEncoder::Application::kSpeech;
1144 break;
1145 case kAudio:
1146 app = AudioEncoder::Application::kAudio;
1147 break;
1148 default:
1149 FATAL();
1150 return 0;
1151 }
1152 return encoder_stack_->SetApplication(app) ? 0 : -1;
1153}
1154
1155// Informs Opus encoder of the maximum playback rate the receiver will render.
1156int AudioCodingModuleImpl::SetOpusMaxPlaybackRate(int frequency_hz) {
1157 rtc::CritScope lock(&acm_crit_sect_);
1158 if (!HaveValidEncoder("SetOpusMaxPlaybackRate")) {
1159 return -1;
1160 }
1161 encoder_stack_->SetMaxPlaybackRate(frequency_hz);
1162 return 0;
1163}
1164
1165int AudioCodingModuleImpl::EnableOpusDtx() {
1166 rtc::CritScope lock(&acm_crit_sect_);
1167 if (!HaveValidEncoder("EnableOpusDtx")) {
1168 return -1;
1169 }
1170 return encoder_stack_->SetDtx(true) ? 0 : -1;
1171}
1172
1173int AudioCodingModuleImpl::DisableOpusDtx() {
1174 rtc::CritScope lock(&acm_crit_sect_);
1175 if (!HaveValidEncoder("DisableOpusDtx")) {
1176 return -1;
1177 }
1178 return encoder_stack_->SetDtx(false) ? 0 : -1;
1179}
1180
1181int32_t AudioCodingModuleImpl::PlayoutTimestamp(uint32_t* timestamp) {
1182 rtc::Optional<uint32_t> ts = PlayoutTimestamp();
1183 if (!ts)
1184 return -1;
1185 *timestamp = *ts;
1186 return 0;
1187}
1188
1189rtc::Optional<uint32_t> AudioCodingModuleImpl::PlayoutTimestamp() {
1190 return receiver_.GetPlayoutTimestamp();
1191}
1192
henrik.lundinb3f1c5d2016-08-22 15:39:53 -07001193int AudioCodingModuleImpl::FilteredCurrentDelayMs() const {
1194 return receiver_.FilteredCurrentDelayMs();
1195}
1196
Henrik Lundinabbff892017-11-29 09:14:04 +01001197int AudioCodingModuleImpl::TargetDelayMs() const {
1198 return receiver_.TargetDelayMs();
1199}
1200
kwibergc13ded52016-06-17 06:00:45 -07001201bool AudioCodingModuleImpl::HaveValidEncoder(const char* caller_name) const {
1202 if (!encoder_stack_) {
Mirko Bonadei675513b2017-11-09 11:09:25 +01001203 RTC_LOG(LS_ERROR) << caller_name << " failed: No send codec is registered.";
kwibergc13ded52016-06-17 06:00:45 -07001204 return false;
1205 }
1206 return true;
1207}
1208
1209int AudioCodingModuleImpl::UnregisterReceiveCodec(uint8_t payload_type) {
1210 return receiver_.RemoveCodec(payload_type);
1211}
1212
1213int AudioCodingModuleImpl::EnableNack(size_t max_nack_list_size) {
1214 return receiver_.EnableNack(max_nack_list_size);
1215}
1216
1217void AudioCodingModuleImpl::DisableNack() {
1218 receiver_.DisableNack();
1219}
1220
1221std::vector<uint16_t> AudioCodingModuleImpl::GetNackList(
1222 int64_t round_trip_time_ms) const {
1223 return receiver_.GetNackList(round_trip_time_ms);
1224}
1225
1226int AudioCodingModuleImpl::LeastRequiredDelayMs() const {
1227 return receiver_.LeastRequiredDelayMs();
1228}
1229
1230void AudioCodingModuleImpl::GetDecodingCallStatistics(
1231 AudioDecodingCallStats* call_stats) const {
1232 receiver_.GetDecodingCallStatistics(call_stats);
1233}
1234
ivoce1198e02017-09-08 08:13:19 -07001235ANAStats AudioCodingModuleImpl::GetANAStats() const {
1236 rtc::CritScope lock(&acm_crit_sect_);
1237 if (encoder_stack_)
1238 return encoder_stack_->GetANAStats();
1239 // If no encoder is set, return default stats.
1240 return ANAStats();
1241}
1242
kwibergc13ded52016-06-17 06:00:45 -07001243} // namespace
1244
Karl Wiberg5817d3d2018-04-06 10:06:42 +02001245AudioCodingModule::Config::Config(
1246 rtc::scoped_refptr<AudioDecoderFactory> decoder_factory)
1247 : neteq_config(),
1248 clock(Clock::GetRealTimeClock()),
1249 decoder_factory(decoder_factory) {
kwiberg36a43882016-08-29 05:33:32 -07001250 // Post-decode VAD is disabled by default in NetEq, however, Audio
1251 // Conference Mixer relies on VAD decisions and fails without them.
1252 neteq_config.enable_post_decode_vad = true;
1253}
1254
1255AudioCodingModule::Config::Config(const Config&) = default;
1256AudioCodingModule::Config::~Config() = default;
1257
Henrik Lundin64dad832015-05-11 12:44:23 +02001258AudioCodingModule* AudioCodingModule::Create(const Config& config) {
kwibergc13ded52016-06-17 06:00:45 -07001259 return new AudioCodingModuleImpl(config);
turaj@webrtc.org7959e162013-09-12 18:30:26 +00001260}
1261
turaj@webrtc.org7959e162013-09-12 18:30:26 +00001262int AudioCodingModule::NumberOfCodecs() {
kwibergfce4a942015-10-27 11:40:24 -07001263 return static_cast<int>(acm2::RentACodec::NumberOfCodecs());
turaj@webrtc.org7959e162013-09-12 18:30:26 +00001264}
1265
turaj@webrtc.org7959e162013-09-12 18:30:26 +00001266int AudioCodingModule::Codec(int list_id, CodecInst* codec) {
kwibergfce4a942015-10-27 11:40:24 -07001267 auto codec_id = acm2::RentACodec::CodecIdFromIndex(list_id);
1268 if (!codec_id)
1269 return -1;
1270 auto ci = acm2::RentACodec::CodecInstById(*codec_id);
1271 if (!ci)
1272 return -1;
1273 *codec = *ci;
1274 return 0;
turaj@webrtc.org7959e162013-09-12 18:30:26 +00001275}
1276
turaj@webrtc.org7959e162013-09-12 18:30:26 +00001277int AudioCodingModule::Codec(const char* payload_name,
1278 CodecInst* codec,
1279 int sampling_freq_hz,
Peter Kasting69558702016-01-12 16:26:35 -08001280 size_t channels) {
Karl Wibergbe579832015-11-10 22:34:18 +01001281 rtc::Optional<CodecInst> ci = acm2::RentACodec::CodecInstByParams(
turaj@webrtc.org6d5d2482013-10-06 04:47:28 +00001282 payload_name, sampling_freq_hz, channels);
kwibergfce4a942015-10-27 11:40:24 -07001283 if (ci) {
1284 *codec = *ci;
1285 return 0;
1286 } else {
1287 // We couldn't find a matching codec, so set the parameters to unacceptable
turaj@webrtc.org7959e162013-09-12 18:30:26 +00001288 // values and return.
1289 codec->plname[0] = '\0';
1290 codec->pltype = -1;
1291 codec->pacsize = 0;
1292 codec->rate = 0;
1293 codec->plfreq = 0;
1294 return -1;
1295 }
turaj@webrtc.org7959e162013-09-12 18:30:26 +00001296}
1297
turaj@webrtc.org7959e162013-09-12 18:30:26 +00001298int AudioCodingModule::Codec(const char* payload_name,
1299 int sampling_freq_hz,
Peter Kasting69558702016-01-12 16:26:35 -08001300 size_t channels) {
Karl Wibergbe579832015-11-10 22:34:18 +01001301 rtc::Optional<acm2::RentACodec::CodecId> ci =
1302 acm2::RentACodec::CodecIdByParams(payload_name, sampling_freq_hz,
1303 channels);
kwibergfce4a942015-10-27 11:40:24 -07001304 if (!ci)
1305 return -1;
Karl Wibergbe579832015-11-10 22:34:18 +01001306 rtc::Optional<int> i = acm2::RentACodec::CodecIndexFromId(*ci);
kwibergfce4a942015-10-27 11:40:24 -07001307 return i ? *i : -1;
turaj@webrtc.org7959e162013-09-12 18:30:26 +00001308}
1309
1310// Checks the validity of the parameters of the given codec
1311bool AudioCodingModule::IsCodecValid(const CodecInst& codec) {
kwibergfce4a942015-10-27 11:40:24 -07001312 bool valid = acm2::RentACodec::IsCodecValid(codec);
1313 if (!valid)
Mirko Bonadei675513b2017-11-09 11:09:25 +01001314 RTC_LOG(LS_ERROR) << "Invalid codec setting";
kwibergfce4a942015-10-27 11:40:24 -07001315 return valid;
turaj@webrtc.org7959e162013-09-12 18:30:26 +00001316}
1317
turaj@webrtc.org7959e162013-09-12 18:30:26 +00001318} // namespace webrtc