blob: 7f652a215d6d0a8f4ef7b4001403647d64d1c9c4 [file] [log] [blame]
turaj@webrtc.org7959e162013-09-12 18:30:26 +00001/*
2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020011#include "modules/audio_coding/include/audio_coding_module.h"
turaj@webrtc.org7959e162013-09-12 18:30:26 +000012
Jonathan Yu36344a02017-07-30 01:55:34 -070013#include <algorithm>
14
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020015#include "modules/audio_coding/acm2/acm_receiver.h"
16#include "modules/audio_coding/acm2/acm_resampler.h"
17#include "modules/audio_coding/acm2/codec_manager.h"
18#include "modules/audio_coding/acm2/rent_a_codec.h"
Fredrik Solenbergbbf21a32018-04-12 22:44:09 +020019#include "modules/include/module_common_types.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020020#include "rtc_base/checks.h"
21#include "rtc_base/logging.h"
Karl Wiberge40468b2017-11-22 10:42:26 +010022#include "rtc_base/numerics/safe_conversions.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020023#include "system_wrappers/include/metrics.h"
turaj@webrtc.org7959e162013-09-12 18:30:26 +000024
25namespace webrtc {
26
kwibergc13ded52016-06-17 06:00:45 -070027namespace {
28
29struct EncoderFactory {
30 AudioEncoder* external_speech_encoder = nullptr;
31 acm2::CodecManager codec_manager;
32 acm2::RentACodec rent_a_codec;
33};
34
35class AudioCodingModuleImpl final : public AudioCodingModule {
36 public:
37 explicit AudioCodingModuleImpl(const AudioCodingModule::Config& config);
38 ~AudioCodingModuleImpl() override;
39
40 /////////////////////////////////////////
41 // Sender
42 //
43
44 // Can be called multiple times for Codec, CNG, RED.
45 int RegisterSendCodec(const CodecInst& send_codec) override;
46
47 void RegisterExternalSendCodec(
48 AudioEncoder* external_speech_encoder) override;
49
kwiberg24c7c122016-09-28 11:57:10 -070050 void ModifyEncoder(rtc::FunctionView<void(std::unique_ptr<AudioEncoder>*)>
51 modifier) override;
kwibergc13ded52016-06-17 06:00:45 -070052
kwiberg24c7c122016-09-28 11:57:10 -070053 void QueryEncoder(
54 rtc::FunctionView<void(const AudioEncoder*)> query) override;
ivoc85228d62016-07-27 04:53:47 -070055
kwibergc13ded52016-06-17 06:00:45 -070056 // Get current send codec.
Danil Chapovalovb6021232018-06-19 13:26:36 +020057 absl::optional<CodecInst> SendCodec() const override;
kwibergc13ded52016-06-17 06:00:45 -070058
59 // Get current send frequency.
60 int SendFrequency() const override;
61
62 // Sets the bitrate to the specified value in bits/sec. In case the codec does
63 // not support the requested value it will choose an appropriate value
64 // instead.
65 void SetBitRate(int bitrate_bps) override;
66
67 // Register a transport callback which will be
68 // called to deliver the encoded buffers.
69 int RegisterTransportCallback(AudioPacketizationCallback* transport) override;
70
71 // Add 10 ms of raw (PCM) audio data to the encoder.
72 int Add10MsData(const AudioFrame& audio_frame) override;
73
74 /////////////////////////////////////////
75 // (RED) Redundant Coding
76 //
77
78 // Configure RED status i.e. on/off.
79 int SetREDStatus(bool enable_red) override;
80
81 // Get RED status.
82 bool REDStatus() const override;
83
84 /////////////////////////////////////////
85 // (FEC) Forward Error Correction (codec internal)
86 //
87
88 // Configure FEC status i.e. on/off.
89 int SetCodecFEC(bool enabled_codec_fec) override;
90
91 // Get FEC status.
92 bool CodecFEC() const override;
93
94 // Set target packet loss rate
95 int SetPacketLossRate(int loss_rate) override;
96
97 /////////////////////////////////////////
98 // (VAD) Voice Activity Detection
99 // and
100 // (CNG) Comfort Noise Generation
101 //
102
103 int SetVAD(bool enable_dtx = true,
104 bool enable_vad = false,
105 ACMVADMode mode = VADNormal) override;
106
107 int VAD(bool* dtx_enabled,
108 bool* vad_enabled,
109 ACMVADMode* mode) const override;
110
111 int RegisterVADCallback(ACMVADCallback* vad_callback) override;
112
113 /////////////////////////////////////////
114 // Receiver
115 //
116
117 // Initialize receiver, resets codec database etc.
118 int InitializeReceiver() override;
119
120 // Get current receive frequency.
121 int ReceiveFrequency() const override;
122
123 // Get current playout frequency.
124 int PlayoutFrequency() const override;
125
kwiberg1c07c702017-03-27 07:15:49 -0700126 void SetReceiveCodecs(const std::map<int, SdpAudioFormat>& codecs) override;
127
kwiberg5adaf732016-10-04 09:33:27 -0700128 bool RegisterReceiveCodec(int rtp_payload_type,
129 const SdpAudioFormat& audio_format) override;
130
kwibergc13ded52016-06-17 06:00:45 -0700131 int RegisterReceiveCodec(const CodecInst& receive_codec) override;
132 int RegisterReceiveCodec(
133 const CodecInst& receive_codec,
kwiberg24c7c122016-09-28 11:57:10 -0700134 rtc::FunctionView<std::unique_ptr<AudioDecoder>()> isac_factory) override;
kwibergc13ded52016-06-17 06:00:45 -0700135
136 int RegisterExternalReceiveCodec(int rtp_payload_type,
137 AudioDecoder* external_decoder,
138 int sample_rate_hz,
139 int num_channels,
140 const std::string& name) override;
141
142 // Get current received codec.
143 int ReceiveCodec(CodecInst* current_codec) const override;
144
Danil Chapovalovb6021232018-06-19 13:26:36 +0200145 absl::optional<SdpAudioFormat> ReceiveFormat() const override;
ossue280cde2016-10-12 11:04:10 -0700146
kwibergc13ded52016-06-17 06:00:45 -0700147 // Incoming packet from network parsed and ready for decode.
148 int IncomingPacket(const uint8_t* incoming_payload,
149 const size_t payload_length,
150 const WebRtcRTPHeader& rtp_info) override;
151
kwibergc13ded52016-06-17 06:00:45 -0700152 // Minimum playout delay.
153 int SetMinimumPlayoutDelay(int time_ms) override;
154
155 // Maximum playout delay.
156 int SetMaximumPlayoutDelay(int time_ms) override;
157
158 // Smallest latency NetEq will maintain.
159 int LeastRequiredDelayMs() const override;
160
161 RTC_DEPRECATED int32_t PlayoutTimestamp(uint32_t* timestamp) override;
162
Danil Chapovalovb6021232018-06-19 13:26:36 +0200163 absl::optional<uint32_t> PlayoutTimestamp() override;
kwibergc13ded52016-06-17 06:00:45 -0700164
henrik.lundinb3f1c5d2016-08-22 15:39:53 -0700165 int FilteredCurrentDelayMs() const override;
166
Henrik Lundinabbff892017-11-29 09:14:04 +0100167 int TargetDelayMs() const override;
168
kwibergc13ded52016-06-17 06:00:45 -0700169 // Get 10 milliseconds of raw audio data to play out, and
170 // automatic resample to the requested frequency if > 0.
171 int PlayoutData10Ms(int desired_freq_hz,
172 AudioFrame* audio_frame,
173 bool* muted) override;
174 int PlayoutData10Ms(int desired_freq_hz, AudioFrame* audio_frame) override;
175
176 /////////////////////////////////////////
177 // Statistics
178 //
179
180 int GetNetworkStatistics(NetworkStatistics* statistics) override;
181
182 int SetOpusApplication(OpusApplicationMode application) override;
183
184 // If current send codec is Opus, informs it about the maximum playback rate
185 // the receiver will render.
186 int SetOpusMaxPlaybackRate(int frequency_hz) override;
187
188 int EnableOpusDtx() override;
189
190 int DisableOpusDtx() override;
191
192 int UnregisterReceiveCodec(uint8_t payload_type) override;
193
194 int EnableNack(size_t max_nack_list_size) override;
195
196 void DisableNack() override;
197
198 std::vector<uint16_t> GetNackList(int64_t round_trip_time_ms) const override;
199
200 void GetDecodingCallStatistics(AudioDecodingCallStats* stats) const override;
201
ivoce1198e02017-09-08 08:13:19 -0700202 ANAStats GetANAStats() const override;
203
kwibergc13ded52016-06-17 06:00:45 -0700204 private:
205 struct InputData {
206 uint32_t input_timestamp;
207 const int16_t* audio;
208 size_t length_per_channel;
209 size_t audio_channel;
210 // If a re-mix is required (up or down), this buffer will store a re-mixed
211 // version of the input.
212 int16_t buffer[WEBRTC_10MS_PCM_AUDIO];
213 };
214
215 // This member class writes values to the named UMA histogram, but only if
216 // the value has changed since the last time (and always for the first call).
217 class ChangeLogger {
218 public:
219 explicit ChangeLogger(const std::string& histogram_name)
220 : histogram_name_(histogram_name) {}
221 // Logs the new value if it is different from the last logged value, or if
222 // this is the first call.
223 void MaybeLog(int value);
224
225 private:
226 int last_value_ = 0;
227 int first_time_ = true;
228 const std::string histogram_name_;
229 };
230
231 int RegisterReceiveCodecUnlocked(
232 const CodecInst& codec,
kwiberg24c7c122016-09-28 11:57:10 -0700233 rtc::FunctionView<std::unique_ptr<AudioDecoder>()> isac_factory)
danilchap56359be2017-09-07 07:53:45 -0700234 RTC_EXCLUSIVE_LOCKS_REQUIRED(acm_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700235
236 int Add10MsDataInternal(const AudioFrame& audio_frame, InputData* input_data)
danilchap56359be2017-09-07 07:53:45 -0700237 RTC_EXCLUSIVE_LOCKS_REQUIRED(acm_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700238 int Encode(const InputData& input_data)
danilchap56359be2017-09-07 07:53:45 -0700239 RTC_EXCLUSIVE_LOCKS_REQUIRED(acm_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700240
danilchap56359be2017-09-07 07:53:45 -0700241 int InitializeReceiverSafe() RTC_EXCLUSIVE_LOCKS_REQUIRED(acm_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700242
243 bool HaveValidEncoder(const char* caller_name) const
danilchap56359be2017-09-07 07:53:45 -0700244 RTC_EXCLUSIVE_LOCKS_REQUIRED(acm_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700245
246 // Preprocessing of input audio, including resampling and down-mixing if
247 // required, before pushing audio into encoder's buffer.
248 //
249 // in_frame: input audio-frame
250 // ptr_out: pointer to output audio_frame. If no preprocessing is required
251 // |ptr_out| will be pointing to |in_frame|, otherwise pointing to
252 // |preprocess_frame_|.
253 //
254 // Return value:
255 // -1: if encountering an error.
256 // 0: otherwise.
257 int PreprocessToAddData(const AudioFrame& in_frame,
258 const AudioFrame** ptr_out)
danilchap56359be2017-09-07 07:53:45 -0700259 RTC_EXCLUSIVE_LOCKS_REQUIRED(acm_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700260
261 // Change required states after starting to receive the codec corresponding
262 // to |index|.
263 int UpdateUponReceivingCodec(int index);
264
265 rtc::CriticalSection acm_crit_sect_;
danilchap56359be2017-09-07 07:53:45 -0700266 rtc::Buffer encode_buffer_ RTC_GUARDED_BY(acm_crit_sect_);
danilchap56359be2017-09-07 07:53:45 -0700267 uint32_t expected_codec_ts_ RTC_GUARDED_BY(acm_crit_sect_);
268 uint32_t expected_in_ts_ RTC_GUARDED_BY(acm_crit_sect_);
269 acm2::ACMResampler resampler_ RTC_GUARDED_BY(acm_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700270 acm2::AcmReceiver receiver_; // AcmReceiver has it's own internal lock.
danilchap56359be2017-09-07 07:53:45 -0700271 ChangeLogger bitrate_logger_ RTC_GUARDED_BY(acm_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700272
danilchap56359be2017-09-07 07:53:45 -0700273 std::unique_ptr<EncoderFactory> encoder_factory_
274 RTC_GUARDED_BY(acm_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700275
276 // Current encoder stack, either obtained from
277 // encoder_factory_->rent_a_codec.RentEncoderStack or provided by a call to
278 // RegisterEncoder.
danilchap56359be2017-09-07 07:53:45 -0700279 std::unique_ptr<AudioEncoder> encoder_stack_ RTC_GUARDED_BY(acm_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700280
danilchap56359be2017-09-07 07:53:45 -0700281 std::unique_ptr<AudioDecoder> isac_decoder_16k_
282 RTC_GUARDED_BY(acm_crit_sect_);
283 std::unique_ptr<AudioDecoder> isac_decoder_32k_
284 RTC_GUARDED_BY(acm_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700285
286 // This is to keep track of CN instances where we can send DTMFs.
danilchap56359be2017-09-07 07:53:45 -0700287 uint8_t previous_pltype_ RTC_GUARDED_BY(acm_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700288
danilchap56359be2017-09-07 07:53:45 -0700289 bool receiver_initialized_ RTC_GUARDED_BY(acm_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700290
danilchap56359be2017-09-07 07:53:45 -0700291 AudioFrame preprocess_frame_ RTC_GUARDED_BY(acm_crit_sect_);
292 bool first_10ms_data_ RTC_GUARDED_BY(acm_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700293
danilchap56359be2017-09-07 07:53:45 -0700294 bool first_frame_ RTC_GUARDED_BY(acm_crit_sect_);
295 uint32_t last_timestamp_ RTC_GUARDED_BY(acm_crit_sect_);
296 uint32_t last_rtp_timestamp_ RTC_GUARDED_BY(acm_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700297
298 rtc::CriticalSection callback_crit_sect_;
299 AudioPacketizationCallback* packetization_callback_
danilchap56359be2017-09-07 07:53:45 -0700300 RTC_GUARDED_BY(callback_crit_sect_);
301 ACMVADCallback* vad_callback_ RTC_GUARDED_BY(callback_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700302
303 int codec_histogram_bins_log_[static_cast<size_t>(
304 AudioEncoder::CodecType::kMaxLoggedAudioCodecTypes)];
305 int number_of_consecutive_empty_packets_;
306};
307
308// Adds a codec usage sample to the histogram.
309void UpdateCodecTypeHistogram(size_t codec_type) {
310 RTC_HISTOGRAM_ENUMERATION(
311 "WebRTC.Audio.Encoder.CodecType", static_cast<int>(codec_type),
312 static_cast<int>(
313 webrtc::AudioEncoder::CodecType::kMaxLoggedAudioCodecTypes));
314}
315
kwibergc13ded52016-06-17 06:00:45 -0700316// Stereo-to-mono can be used as in-place.
317int DownMix(const AudioFrame& frame,
318 size_t length_out_buff,
319 int16_t* out_buff) {
yujo36b1a5f2017-06-12 12:45:32 -0700320 RTC_DCHECK_EQ(frame.num_channels_, 2);
321 RTC_DCHECK_GE(length_out_buff, frame.samples_per_channel_);
322
323 if (!frame.muted()) {
324 const int16_t* frame_data = frame.data();
325 for (size_t n = 0; n < frame.samples_per_channel_; ++n) {
Yves Gerey665174f2018-06-19 15:03:05 +0200326 out_buff[n] =
327 static_cast<int16_t>((static_cast<int32_t>(frame_data[2 * n]) +
328 static_cast<int32_t>(frame_data[2 * n + 1])) >>
329 1);
yujo36b1a5f2017-06-12 12:45:32 -0700330 }
331 } else {
Jonathan Yu36344a02017-07-30 01:55:34 -0700332 std::fill(out_buff, out_buff + frame.samples_per_channel_, 0);
kwibergc13ded52016-06-17 06:00:45 -0700333 }
kwibergc13ded52016-06-17 06:00:45 -0700334 return 0;
335}
336
337// Mono-to-stereo can be used as in-place.
338int UpMix(const AudioFrame& frame, size_t length_out_buff, int16_t* out_buff) {
yujo36b1a5f2017-06-12 12:45:32 -0700339 RTC_DCHECK_EQ(frame.num_channels_, 1);
340 RTC_DCHECK_GE(length_out_buff, 2 * frame.samples_per_channel_);
341
342 if (!frame.muted()) {
343 const int16_t* frame_data = frame.data();
344 for (size_t n = frame.samples_per_channel_; n != 0; --n) {
345 size_t i = n - 1;
346 int16_t sample = frame_data[i];
347 out_buff[2 * i + 1] = sample;
348 out_buff[2 * i] = sample;
349 }
350 } else {
Jonathan Yu36344a02017-07-30 01:55:34 -0700351 std::fill(out_buff, out_buff + frame.samples_per_channel_ * 2, 0);
kwibergc13ded52016-06-17 06:00:45 -0700352 }
353 return 0;
354}
355
356void ConvertEncodedInfoToFragmentationHeader(
357 const AudioEncoder::EncodedInfo& info,
358 RTPFragmentationHeader* frag) {
359 if (info.redundant.empty()) {
360 frag->fragmentationVectorSize = 0;
361 return;
362 }
363
364 frag->VerifyAndAllocateFragmentationHeader(
365 static_cast<uint16_t>(info.redundant.size()));
366 frag->fragmentationVectorSize = static_cast<uint16_t>(info.redundant.size());
367 size_t offset = 0;
368 for (size_t i = 0; i < info.redundant.size(); ++i) {
369 frag->fragmentationOffset[i] = offset;
370 offset += info.redundant[i].encoded_bytes;
371 frag->fragmentationLength[i] = info.redundant[i].encoded_bytes;
kwibergd3edd772017-03-01 18:52:48 -0800372 frag->fragmentationTimeDiff[i] = rtc::dchecked_cast<uint16_t>(
kwibergc13ded52016-06-17 06:00:45 -0700373 info.encoded_timestamp - info.redundant[i].encoded_timestamp);
374 frag->fragmentationPlType[i] = info.redundant[i].payload_type;
375 }
376}
377
378// Wraps a raw AudioEncoder pointer. The idea is that you can put one of these
379// in a unique_ptr, to protect the contained raw pointer from being deleted
380// when the unique_ptr expires. (This is of course a bad idea in general, but
381// backwards compatibility.)
382class RawAudioEncoderWrapper final : public AudioEncoder {
383 public:
384 RawAudioEncoderWrapper(AudioEncoder* enc) : enc_(enc) {}
385 int SampleRateHz() const override { return enc_->SampleRateHz(); }
386 size_t NumChannels() const override { return enc_->NumChannels(); }
387 int RtpTimestampRateHz() const override { return enc_->RtpTimestampRateHz(); }
388 size_t Num10MsFramesInNextPacket() const override {
389 return enc_->Num10MsFramesInNextPacket();
390 }
391 size_t Max10MsFramesInAPacket() const override {
392 return enc_->Max10MsFramesInAPacket();
393 }
394 int GetTargetBitrate() const override { return enc_->GetTargetBitrate(); }
395 EncodedInfo EncodeImpl(uint32_t rtp_timestamp,
396 rtc::ArrayView<const int16_t> audio,
397 rtc::Buffer* encoded) override {
398 return enc_->Encode(rtp_timestamp, audio, encoded);
399 }
400 void Reset() override { return enc_->Reset(); }
401 bool SetFec(bool enable) override { return enc_->SetFec(enable); }
402 bool SetDtx(bool enable) override { return enc_->SetDtx(enable); }
403 bool SetApplication(Application application) override {
404 return enc_->SetApplication(application);
405 }
406 void SetMaxPlaybackRate(int frequency_hz) override {
407 return enc_->SetMaxPlaybackRate(frequency_hz);
408 }
kwibergc13ded52016-06-17 06:00:45 -0700409
410 private:
411 AudioEncoder* enc_;
412};
413
414// Return false on error.
415bool CreateSpeechEncoderIfNecessary(EncoderFactory* ef) {
416 auto* sp = ef->codec_manager.GetStackParams();
417 if (sp->speech_encoder) {
418 // Do nothing; we already have a speech encoder.
419 } else if (ef->codec_manager.GetCodecInst()) {
420 RTC_DCHECK(!ef->external_speech_encoder);
421 // We have no speech encoder, but we have a specification for making one.
422 std::unique_ptr<AudioEncoder> enc =
423 ef->rent_a_codec.RentEncoder(*ef->codec_manager.GetCodecInst());
424 if (!enc)
425 return false; // Encoder spec was bad.
426 sp->speech_encoder = std::move(enc);
427 } else if (ef->external_speech_encoder) {
428 RTC_DCHECK(!ef->codec_manager.GetCodecInst());
429 // We have an external speech encoder.
430 sp->speech_encoder = std::unique_ptr<AudioEncoder>(
431 new RawAudioEncoderWrapper(ef->external_speech_encoder));
432 }
433 return true;
434}
435
436void AudioCodingModuleImpl::ChangeLogger::MaybeLog(int value) {
437 if (value != last_value_ || first_time_) {
438 first_time_ = false;
439 last_value_ = value;
440 RTC_HISTOGRAM_COUNTS_SPARSE_100(histogram_name_, value);
441 }
442}
443
444AudioCodingModuleImpl::AudioCodingModuleImpl(
445 const AudioCodingModule::Config& config)
solenbergc7b4a452017-09-28 07:37:11 -0700446 : expected_codec_ts_(0xD87F3F9F),
kwibergc13ded52016-06-17 06:00:45 -0700447 expected_in_ts_(0xD87F3F9F),
448 receiver_(config),
449 bitrate_logger_("WebRTC.Audio.TargetBitrateInKbps"),
450 encoder_factory_(new EncoderFactory),
451 encoder_stack_(nullptr),
452 previous_pltype_(255),
453 receiver_initialized_(false),
454 first_10ms_data_(false),
455 first_frame_(true),
456 packetization_callback_(NULL),
457 vad_callback_(NULL),
458 codec_histogram_bins_log_(),
459 number_of_consecutive_empty_packets_(0) {
460 if (InitializeReceiverSafe() < 0) {
Mirko Bonadei675513b2017-11-09 11:09:25 +0100461 RTC_LOG(LS_ERROR) << "Cannot initialize receiver";
kwibergc13ded52016-06-17 06:00:45 -0700462 }
Mirko Bonadei675513b2017-11-09 11:09:25 +0100463 RTC_LOG(LS_INFO) << "Created";
kwibergc13ded52016-06-17 06:00:45 -0700464}
465
466AudioCodingModuleImpl::~AudioCodingModuleImpl() = default;
467
468int32_t AudioCodingModuleImpl::Encode(const InputData& input_data) {
469 AudioEncoder::EncodedInfo encoded_info;
470 uint8_t previous_pltype;
471
472 // Check if there is an encoder before.
473 if (!HaveValidEncoder("Process"))
474 return -1;
475
Yves Gerey665174f2018-06-19 15:03:05 +0200476 if (!first_frame_) {
deadbeeffcada902016-08-24 12:45:13 -0700477 RTC_DCHECK(IsNewerTimestamp(input_data.input_timestamp, last_timestamp_))
ossu63fb95a2016-07-06 09:34:22 -0700478 << "Time should not move backwards";
479 }
480
kwibergc13ded52016-06-17 06:00:45 -0700481 // Scale the timestamp to the codec's RTP timestamp rate.
482 uint32_t rtp_timestamp =
483 first_frame_ ? input_data.input_timestamp
484 : last_rtp_timestamp_ +
485 rtc::CheckedDivExact(
486 input_data.input_timestamp - last_timestamp_,
487 static_cast<uint32_t>(rtc::CheckedDivExact(
488 encoder_stack_->SampleRateHz(),
489 encoder_stack_->RtpTimestampRateHz())));
490 last_timestamp_ = input_data.input_timestamp;
491 last_rtp_timestamp_ = rtp_timestamp;
492 first_frame_ = false;
493
494 // Clear the buffer before reuse - encoded data will get appended.
495 encode_buffer_.Clear();
496 encoded_info = encoder_stack_->Encode(
Yves Gerey665174f2018-06-19 15:03:05 +0200497 rtp_timestamp,
498 rtc::ArrayView<const int16_t>(
499 input_data.audio,
500 input_data.audio_channel * input_data.length_per_channel),
kwibergc13ded52016-06-17 06:00:45 -0700501 &encode_buffer_);
502
503 bitrate_logger_.MaybeLog(encoder_stack_->GetTargetBitrate() / 1000);
504 if (encode_buffer_.size() == 0 && !encoded_info.send_even_if_empty) {
505 // Not enough data.
506 return 0;
507 }
508 previous_pltype = previous_pltype_; // Read it while we have the critsect.
509
510 // Log codec type to histogram once every 500 packets.
511 if (encoded_info.encoded_bytes == 0) {
512 ++number_of_consecutive_empty_packets_;
513 } else {
514 size_t codec_type = static_cast<size_t>(encoded_info.encoder_type);
515 codec_histogram_bins_log_[codec_type] +=
516 number_of_consecutive_empty_packets_ + 1;
517 number_of_consecutive_empty_packets_ = 0;
518 if (codec_histogram_bins_log_[codec_type] >= 500) {
519 codec_histogram_bins_log_[codec_type] -= 500;
520 UpdateCodecTypeHistogram(codec_type);
521 }
522 }
523
524 RTPFragmentationHeader my_fragmentation;
525 ConvertEncodedInfoToFragmentationHeader(encoded_info, &my_fragmentation);
526 FrameType frame_type;
527 if (encode_buffer_.size() == 0 && encoded_info.send_even_if_empty) {
528 frame_type = kEmptyFrame;
529 encoded_info.payload_type = previous_pltype;
530 } else {
kwibergaf476c72016-11-28 15:21:39 -0800531 RTC_DCHECK_GT(encode_buffer_.size(), 0);
kwibergc13ded52016-06-17 06:00:45 -0700532 frame_type = encoded_info.speech ? kAudioFrameSpeech : kAudioFrameCN;
533 }
534
535 {
536 rtc::CritScope lock(&callback_crit_sect_);
537 if (packetization_callback_) {
538 packetization_callback_->SendData(
539 frame_type, encoded_info.payload_type, encoded_info.encoded_timestamp,
540 encode_buffer_.data(), encode_buffer_.size(),
541 my_fragmentation.fragmentationVectorSize > 0 ? &my_fragmentation
542 : nullptr);
543 }
544
545 if (vad_callback_) {
546 // Callback with VAD decision.
547 vad_callback_->InFrameType(frame_type);
548 }
549 }
550 previous_pltype_ = encoded_info.payload_type;
551 return static_cast<int32_t>(encode_buffer_.size());
552}
553
554/////////////////////////////////////////
555// Sender
556//
557
558// Can be called multiple times for Codec, CNG, RED.
559int AudioCodingModuleImpl::RegisterSendCodec(const CodecInst& send_codec) {
560 rtc::CritScope lock(&acm_crit_sect_);
561 if (!encoder_factory_->codec_manager.RegisterEncoder(send_codec)) {
562 return -1;
563 }
564 if (encoder_factory_->codec_manager.GetCodecInst()) {
565 encoder_factory_->external_speech_encoder = nullptr;
566 }
567 if (!CreateSpeechEncoderIfNecessary(encoder_factory_.get())) {
568 return -1;
569 }
570 auto* sp = encoder_factory_->codec_manager.GetStackParams();
571 if (sp->speech_encoder)
572 encoder_stack_ = encoder_factory_->rent_a_codec.RentEncoderStack(sp);
573 return 0;
574}
575
576void AudioCodingModuleImpl::RegisterExternalSendCodec(
577 AudioEncoder* external_speech_encoder) {
578 rtc::CritScope lock(&acm_crit_sect_);
579 encoder_factory_->codec_manager.UnsetCodecInst();
580 encoder_factory_->external_speech_encoder = external_speech_encoder;
581 RTC_CHECK(CreateSpeechEncoderIfNecessary(encoder_factory_.get()));
582 auto* sp = encoder_factory_->codec_manager.GetStackParams();
583 RTC_CHECK(sp->speech_encoder);
584 encoder_stack_ = encoder_factory_->rent_a_codec.RentEncoderStack(sp);
585}
586
587void AudioCodingModuleImpl::ModifyEncoder(
kwiberg24c7c122016-09-28 11:57:10 -0700588 rtc::FunctionView<void(std::unique_ptr<AudioEncoder>*)> modifier) {
kwibergc13ded52016-06-17 06:00:45 -0700589 rtc::CritScope lock(&acm_crit_sect_);
590
591 // Wipe the encoder factory, so that everything that relies on it will fail.
592 // We don't want the complexity of supporting swapping back and forth.
593 if (encoder_factory_) {
594 encoder_factory_.reset();
595 RTC_CHECK(!encoder_stack_); // Ensure we hadn't started using the factory.
596 }
597
598 modifier(&encoder_stack_);
599}
600
ivoc85228d62016-07-27 04:53:47 -0700601void AudioCodingModuleImpl::QueryEncoder(
kwiberg24c7c122016-09-28 11:57:10 -0700602 rtc::FunctionView<void(const AudioEncoder*)> query) {
ivoc85228d62016-07-27 04:53:47 -0700603 rtc::CritScope lock(&acm_crit_sect_);
604 query(encoder_stack_.get());
605}
606
kwibergc13ded52016-06-17 06:00:45 -0700607// Get current send codec.
Danil Chapovalovb6021232018-06-19 13:26:36 +0200608absl::optional<CodecInst> AudioCodingModuleImpl::SendCodec() const {
kwibergc13ded52016-06-17 06:00:45 -0700609 rtc::CritScope lock(&acm_crit_sect_);
610 if (encoder_factory_) {
611 auto* ci = encoder_factory_->codec_manager.GetCodecInst();
612 if (ci) {
Oskar Sundbom12ab00b2017-11-16 15:31:38 +0100613 return *ci;
kwibergc13ded52016-06-17 06:00:45 -0700614 }
615 CreateSpeechEncoderIfNecessary(encoder_factory_.get());
616 const std::unique_ptr<AudioEncoder>& enc =
617 encoder_factory_->codec_manager.GetStackParams()->speech_encoder;
618 if (enc) {
Oskar Sundbom12ab00b2017-11-16 15:31:38 +0100619 return acm2::CodecManager::ForgeCodecInst(enc.get());
kwibergc13ded52016-06-17 06:00:45 -0700620 }
Danil Chapovalovb6021232018-06-19 13:26:36 +0200621 return absl::nullopt;
kwibergc13ded52016-06-17 06:00:45 -0700622 } else {
623 return encoder_stack_
Danil Chapovalovb6021232018-06-19 13:26:36 +0200624 ? absl::optional<CodecInst>(
kwibergc13ded52016-06-17 06:00:45 -0700625 acm2::CodecManager::ForgeCodecInst(encoder_stack_.get()))
Danil Chapovalovb6021232018-06-19 13:26:36 +0200626 : absl::nullopt;
kwibergc13ded52016-06-17 06:00:45 -0700627 }
628}
629
630// Get current send frequency.
631int AudioCodingModuleImpl::SendFrequency() const {
kwibergc13ded52016-06-17 06:00:45 -0700632 rtc::CritScope lock(&acm_crit_sect_);
633
634 if (!encoder_stack_) {
Mirko Bonadei675513b2017-11-09 11:09:25 +0100635 RTC_LOG(LS_ERROR) << "SendFrequency Failed, no codec is registered";
kwibergc13ded52016-06-17 06:00:45 -0700636 return -1;
637 }
638
639 return encoder_stack_->SampleRateHz();
640}
641
642void AudioCodingModuleImpl::SetBitRate(int bitrate_bps) {
643 rtc::CritScope lock(&acm_crit_sect_);
644 if (encoder_stack_) {
Danil Chapovalovb6021232018-06-19 13:26:36 +0200645 encoder_stack_->OnReceivedUplinkBandwidth(bitrate_bps, absl::nullopt);
kwibergc13ded52016-06-17 06:00:45 -0700646 }
647}
648
649// Register a transport callback which will be called to deliver
650// the encoded buffers.
651int AudioCodingModuleImpl::RegisterTransportCallback(
652 AudioPacketizationCallback* transport) {
653 rtc::CritScope lock(&callback_crit_sect_);
654 packetization_callback_ = transport;
655 return 0;
656}
657
658// Add 10MS of raw (PCM) audio data to the encoder.
659int AudioCodingModuleImpl::Add10MsData(const AudioFrame& audio_frame) {
660 InputData input_data;
661 rtc::CritScope lock(&acm_crit_sect_);
662 int r = Add10MsDataInternal(audio_frame, &input_data);
663 return r < 0 ? r : Encode(input_data);
664}
665
666int AudioCodingModuleImpl::Add10MsDataInternal(const AudioFrame& audio_frame,
667 InputData* input_data) {
668 if (audio_frame.samples_per_channel_ == 0) {
669 assert(false);
Mirko Bonadei675513b2017-11-09 11:09:25 +0100670 RTC_LOG(LS_ERROR) << "Cannot Add 10 ms audio, payload length is zero";
kwibergc13ded52016-06-17 06:00:45 -0700671 return -1;
672 }
673
674 if (audio_frame.sample_rate_hz_ > 48000) {
675 assert(false);
Mirko Bonadei675513b2017-11-09 11:09:25 +0100676 RTC_LOG(LS_ERROR) << "Cannot Add 10 ms audio, input frequency not valid";
kwibergc13ded52016-06-17 06:00:45 -0700677 return -1;
678 }
679
680 // If the length and frequency matches. We currently just support raw PCM.
681 if (static_cast<size_t>(audio_frame.sample_rate_hz_ / 100) !=
682 audio_frame.samples_per_channel_) {
Mirko Bonadei675513b2017-11-09 11:09:25 +0100683 RTC_LOG(LS_ERROR)
Alex Loiko300ec8c2017-05-30 17:23:28 +0200684 << "Cannot Add 10 ms audio, input frequency and length doesn't match";
kwibergc13ded52016-06-17 06:00:45 -0700685 return -1;
686 }
687
688 if (audio_frame.num_channels_ != 1 && audio_frame.num_channels_ != 2) {
Mirko Bonadei675513b2017-11-09 11:09:25 +0100689 RTC_LOG(LS_ERROR) << "Cannot Add 10 ms audio, invalid number of channels.";
kwibergc13ded52016-06-17 06:00:45 -0700690 return -1;
691 }
692
693 // Do we have a codec registered?
694 if (!HaveValidEncoder("Add10MsData")) {
695 return -1;
696 }
697
698 const AudioFrame* ptr_frame;
699 // Perform a resampling, also down-mix if it is required and can be
700 // performed before resampling (a down mix prior to resampling will take
701 // place if both primary and secondary encoders are mono and input is in
702 // stereo).
703 if (PreprocessToAddData(audio_frame, &ptr_frame) < 0) {
704 return -1;
705 }
706
707 // Check whether we need an up-mix or down-mix?
708 const size_t current_num_channels = encoder_stack_->NumChannels();
709 const bool same_num_channels =
710 ptr_frame->num_channels_ == current_num_channels;
711
712 if (!same_num_channels) {
713 if (ptr_frame->num_channels_ == 1) {
714 if (UpMix(*ptr_frame, WEBRTC_10MS_PCM_AUDIO, input_data->buffer) < 0)
715 return -1;
716 } else {
717 if (DownMix(*ptr_frame, WEBRTC_10MS_PCM_AUDIO, input_data->buffer) < 0)
718 return -1;
719 }
720 }
721
722 // When adding data to encoders this pointer is pointing to an audio buffer
723 // with correct number of channels.
yujo36b1a5f2017-06-12 12:45:32 -0700724 const int16_t* ptr_audio = ptr_frame->data();
kwibergc13ded52016-06-17 06:00:45 -0700725
726 // For pushing data to primary, point the |ptr_audio| to correct buffer.
727 if (!same_num_channels)
728 ptr_audio = input_data->buffer;
729
yujo36b1a5f2017-06-12 12:45:32 -0700730 // TODO(yujo): Skip encode of muted frames.
kwibergc13ded52016-06-17 06:00:45 -0700731 input_data->input_timestamp = ptr_frame->timestamp_;
732 input_data->audio = ptr_audio;
733 input_data->length_per_channel = ptr_frame->samples_per_channel_;
734 input_data->audio_channel = current_num_channels;
735
736 return 0;
737}
738
739// Perform a resampling and down-mix if required. We down-mix only if
740// encoder is mono and input is stereo. In case of dual-streaming, both
741// encoders has to be mono for down-mix to take place.
742// |*ptr_out| will point to the pre-processed audio-frame. If no pre-processing
743// is required, |*ptr_out| points to |in_frame|.
yujo36b1a5f2017-06-12 12:45:32 -0700744// TODO(yujo): Make this more efficient for muted frames.
kwibergc13ded52016-06-17 06:00:45 -0700745int AudioCodingModuleImpl::PreprocessToAddData(const AudioFrame& in_frame,
746 const AudioFrame** ptr_out) {
747 const bool resample =
748 in_frame.sample_rate_hz_ != encoder_stack_->SampleRateHz();
749
750 // This variable is true if primary codec and secondary codec (if exists)
751 // are both mono and input is stereo.
752 // TODO(henrik.lundin): This condition should probably be
753 // in_frame.num_channels_ > encoder_stack_->NumChannels()
754 const bool down_mix =
755 in_frame.num_channels_ == 2 && encoder_stack_->NumChannels() == 1;
756
757 if (!first_10ms_data_) {
758 expected_in_ts_ = in_frame.timestamp_;
759 expected_codec_ts_ = in_frame.timestamp_;
760 first_10ms_data_ = true;
761 } else if (in_frame.timestamp_ != expected_in_ts_) {
Mirko Bonadei675513b2017-11-09 11:09:25 +0100762 RTC_LOG(LS_WARNING) << "Unexpected input timestamp: " << in_frame.timestamp_
763 << ", expected: " << expected_in_ts_;
kwibergc13ded52016-06-17 06:00:45 -0700764 expected_codec_ts_ +=
765 (in_frame.timestamp_ - expected_in_ts_) *
766 static_cast<uint32_t>(
767 static_cast<double>(encoder_stack_->SampleRateHz()) /
768 static_cast<double>(in_frame.sample_rate_hz_));
769 expected_in_ts_ = in_frame.timestamp_;
770 }
771
kwibergc13ded52016-06-17 06:00:45 -0700772 if (!down_mix && !resample) {
773 // No pre-processing is required.
ossu63fb95a2016-07-06 09:34:22 -0700774 if (expected_in_ts_ == expected_codec_ts_) {
775 // If we've never resampled, we can use the input frame as-is
776 *ptr_out = &in_frame;
777 } else {
778 // Otherwise we'll need to alter the timestamp. Since in_frame is const,
779 // we'll have to make a copy of it.
780 preprocess_frame_.CopyFrom(in_frame);
781 preprocess_frame_.timestamp_ = expected_codec_ts_;
782 *ptr_out = &preprocess_frame_;
783 }
784
kwibergc13ded52016-06-17 06:00:45 -0700785 expected_in_ts_ += static_cast<uint32_t>(in_frame.samples_per_channel_);
786 expected_codec_ts_ += static_cast<uint32_t>(in_frame.samples_per_channel_);
kwibergc13ded52016-06-17 06:00:45 -0700787 return 0;
788 }
789
790 *ptr_out = &preprocess_frame_;
791 preprocess_frame_.num_channels_ = in_frame.num_channels_;
792 int16_t audio[WEBRTC_10MS_PCM_AUDIO];
yujo36b1a5f2017-06-12 12:45:32 -0700793 const int16_t* src_ptr_audio = in_frame.data();
kwibergc13ded52016-06-17 06:00:45 -0700794 if (down_mix) {
795 // If a resampling is required the output of a down-mix is written into a
796 // local buffer, otherwise, it will be written to the output frame.
Yves Gerey665174f2018-06-19 15:03:05 +0200797 int16_t* dest_ptr_audio =
798 resample ? audio : preprocess_frame_.mutable_data();
kwibergc13ded52016-06-17 06:00:45 -0700799 if (DownMix(in_frame, WEBRTC_10MS_PCM_AUDIO, dest_ptr_audio) < 0)
800 return -1;
801 preprocess_frame_.num_channels_ = 1;
802 // Set the input of the resampler is the down-mixed signal.
803 src_ptr_audio = audio;
804 }
805
806 preprocess_frame_.timestamp_ = expected_codec_ts_;
807 preprocess_frame_.samples_per_channel_ = in_frame.samples_per_channel_;
808 preprocess_frame_.sample_rate_hz_ = in_frame.sample_rate_hz_;
809 // If it is required, we have to do a resampling.
810 if (resample) {
811 // The result of the resampler is written to output frame.
yujo36b1a5f2017-06-12 12:45:32 -0700812 int16_t* dest_ptr_audio = preprocess_frame_.mutable_data();
kwibergc13ded52016-06-17 06:00:45 -0700813
814 int samples_per_channel = resampler_.Resample10Msec(
815 src_ptr_audio, in_frame.sample_rate_hz_, encoder_stack_->SampleRateHz(),
816 preprocess_frame_.num_channels_, AudioFrame::kMaxDataSizeSamples,
817 dest_ptr_audio);
818
819 if (samples_per_channel < 0) {
Mirko Bonadei675513b2017-11-09 11:09:25 +0100820 RTC_LOG(LS_ERROR) << "Cannot add 10 ms audio, resampling failed";
kwibergc13ded52016-06-17 06:00:45 -0700821 return -1;
822 }
823 preprocess_frame_.samples_per_channel_ =
824 static_cast<size_t>(samples_per_channel);
825 preprocess_frame_.sample_rate_hz_ = encoder_stack_->SampleRateHz();
826 }
827
828 expected_codec_ts_ +=
829 static_cast<uint32_t>(preprocess_frame_.samples_per_channel_);
830 expected_in_ts_ += static_cast<uint32_t>(in_frame.samples_per_channel_);
831
832 return 0;
833}
834
835/////////////////////////////////////////
836// (RED) Redundant Coding
837//
838
839bool AudioCodingModuleImpl::REDStatus() const {
840 rtc::CritScope lock(&acm_crit_sect_);
841 return encoder_factory_->codec_manager.GetStackParams()->use_red;
842}
843
844// Configure RED status i.e on/off.
845int AudioCodingModuleImpl::SetREDStatus(bool enable_red) {
846#ifdef WEBRTC_CODEC_RED
847 rtc::CritScope lock(&acm_crit_sect_);
848 CreateSpeechEncoderIfNecessary(encoder_factory_.get());
849 if (!encoder_factory_->codec_manager.SetCopyRed(enable_red)) {
850 return -1;
851 }
852 auto* sp = encoder_factory_->codec_manager.GetStackParams();
853 if (sp->speech_encoder)
854 encoder_stack_ = encoder_factory_->rent_a_codec.RentEncoderStack(sp);
855 return 0;
856#else
Mirko Bonadei675513b2017-11-09 11:09:25 +0100857 RTC_LOG(LS_WARNING) << " WEBRTC_CODEC_RED is undefined";
kwibergc13ded52016-06-17 06:00:45 -0700858 return -1;
859#endif
860}
861
862/////////////////////////////////////////
863// (FEC) Forward Error Correction (codec internal)
864//
865
866bool AudioCodingModuleImpl::CodecFEC() const {
867 rtc::CritScope lock(&acm_crit_sect_);
868 return encoder_factory_->codec_manager.GetStackParams()->use_codec_fec;
869}
870
871int AudioCodingModuleImpl::SetCodecFEC(bool enable_codec_fec) {
872 rtc::CritScope lock(&acm_crit_sect_);
873 CreateSpeechEncoderIfNecessary(encoder_factory_.get());
874 if (!encoder_factory_->codec_manager.SetCodecFEC(enable_codec_fec)) {
875 return -1;
876 }
877 auto* sp = encoder_factory_->codec_manager.GetStackParams();
878 if (sp->speech_encoder)
879 encoder_stack_ = encoder_factory_->rent_a_codec.RentEncoderStack(sp);
880 if (enable_codec_fec) {
881 return sp->use_codec_fec ? 0 : -1;
882 } else {
883 RTC_DCHECK(!sp->use_codec_fec);
884 return 0;
885 }
886}
887
888int AudioCodingModuleImpl::SetPacketLossRate(int loss_rate) {
889 rtc::CritScope lock(&acm_crit_sect_);
890 if (HaveValidEncoder("SetPacketLossRate")) {
minyue4b9a2cb2016-11-30 06:49:59 -0800891 encoder_stack_->OnReceivedUplinkPacketLossFraction(loss_rate / 100.0);
kwibergc13ded52016-06-17 06:00:45 -0700892 }
893 return 0;
894}
895
896/////////////////////////////////////////
897// (VAD) Voice Activity Detection
898//
899int AudioCodingModuleImpl::SetVAD(bool enable_dtx,
900 bool enable_vad,
901 ACMVADMode mode) {
902 // Note: |enable_vad| is not used; VAD is enabled based on the DTX setting.
903 RTC_DCHECK_EQ(enable_dtx, enable_vad);
904 rtc::CritScope lock(&acm_crit_sect_);
905 CreateSpeechEncoderIfNecessary(encoder_factory_.get());
906 if (!encoder_factory_->codec_manager.SetVAD(enable_dtx, mode)) {
907 return -1;
908 }
909 auto* sp = encoder_factory_->codec_manager.GetStackParams();
910 if (sp->speech_encoder)
911 encoder_stack_ = encoder_factory_->rent_a_codec.RentEncoderStack(sp);
912 return 0;
913}
914
915// Get VAD/DTX settings.
Yves Gerey665174f2018-06-19 15:03:05 +0200916int AudioCodingModuleImpl::VAD(bool* dtx_enabled,
917 bool* vad_enabled,
kwibergc13ded52016-06-17 06:00:45 -0700918 ACMVADMode* mode) const {
919 rtc::CritScope lock(&acm_crit_sect_);
920 const auto* sp = encoder_factory_->codec_manager.GetStackParams();
921 *dtx_enabled = *vad_enabled = sp->use_cng;
922 *mode = sp->vad_mode;
923 return 0;
924}
925
926/////////////////////////////////////////
927// Receiver
928//
929
930int AudioCodingModuleImpl::InitializeReceiver() {
931 rtc::CritScope lock(&acm_crit_sect_);
932 return InitializeReceiverSafe();
933}
934
935// Initialize receiver, resets codec database etc.
936int AudioCodingModuleImpl::InitializeReceiverSafe() {
937 // If the receiver is already initialized then we want to destroy any
938 // existing decoders. After a call to this function, we should have a clean
939 // start-up.
kwiberg6b19b562016-09-20 04:02:25 -0700940 if (receiver_initialized_)
941 receiver_.RemoveAllCodecs();
kwibergc13ded52016-06-17 06:00:45 -0700942 receiver_.ResetInitialDelay();
943 receiver_.SetMinimumDelay(0);
944 receiver_.SetMaximumDelay(0);
945 receiver_.FlushBuffers();
946
kwibergc13ded52016-06-17 06:00:45 -0700947 receiver_initialized_ = true;
948 return 0;
949}
950
951// Get current receive frequency.
952int AudioCodingModuleImpl::ReceiveFrequency() const {
953 const auto last_packet_sample_rate = receiver_.last_packet_sample_rate_hz();
954 return last_packet_sample_rate ? *last_packet_sample_rate
955 : receiver_.last_output_sample_rate_hz();
956}
957
958// Get current playout frequency.
959int AudioCodingModuleImpl::PlayoutFrequency() const {
kwibergc13ded52016-06-17 06:00:45 -0700960 return receiver_.last_output_sample_rate_hz();
961}
962
kwiberg1c07c702017-03-27 07:15:49 -0700963void AudioCodingModuleImpl::SetReceiveCodecs(
964 const std::map<int, SdpAudioFormat>& codecs) {
965 rtc::CritScope lock(&acm_crit_sect_);
966 receiver_.SetCodecs(codecs);
967}
968
kwiberg5adaf732016-10-04 09:33:27 -0700969bool AudioCodingModuleImpl::RegisterReceiveCodec(
970 int rtp_payload_type,
971 const SdpAudioFormat& audio_format) {
972 rtc::CritScope lock(&acm_crit_sect_);
973 RTC_DCHECK(receiver_initialized_);
974
975 if (!acm2::RentACodec::IsPayloadTypeValid(rtp_payload_type)) {
Mirko Bonadei675513b2017-11-09 11:09:25 +0100976 RTC_LOG_F(LS_ERROR) << "Invalid payload-type " << rtp_payload_type
977 << " for decoder.";
kwiberg5adaf732016-10-04 09:33:27 -0700978 return false;
979 }
980
981 return receiver_.AddCodec(rtp_payload_type, audio_format);
982}
983
kwibergc13ded52016-06-17 06:00:45 -0700984int AudioCodingModuleImpl::RegisterReceiveCodec(const CodecInst& codec) {
985 rtc::CritScope lock(&acm_crit_sect_);
986 auto* ef = encoder_factory_.get();
987 return RegisterReceiveCodecUnlocked(
988 codec, [&] { return ef->rent_a_codec.RentIsacDecoder(codec.plfreq); });
989}
990
991int AudioCodingModuleImpl::RegisterReceiveCodec(
992 const CodecInst& codec,
kwiberg24c7c122016-09-28 11:57:10 -0700993 rtc::FunctionView<std::unique_ptr<AudioDecoder>()> isac_factory) {
kwibergc13ded52016-06-17 06:00:45 -0700994 rtc::CritScope lock(&acm_crit_sect_);
995 return RegisterReceiveCodecUnlocked(codec, isac_factory);
996}
997
998int AudioCodingModuleImpl::RegisterReceiveCodecUnlocked(
999 const CodecInst& codec,
kwiberg24c7c122016-09-28 11:57:10 -07001000 rtc::FunctionView<std::unique_ptr<AudioDecoder>()> isac_factory) {
kwibergc13ded52016-06-17 06:00:45 -07001001 RTC_DCHECK(receiver_initialized_);
1002 if (codec.channels > 2) {
Mirko Bonadei675513b2017-11-09 11:09:25 +01001003 RTC_LOG_F(LS_ERROR) << "Unsupported number of channels: " << codec.channels;
kwibergc13ded52016-06-17 06:00:45 -07001004 return -1;
1005 }
1006
1007 auto codec_id = acm2::RentACodec::CodecIdByParams(codec.plname, codec.plfreq,
1008 codec.channels);
1009 if (!codec_id) {
Mirko Bonadei675513b2017-11-09 11:09:25 +01001010 RTC_LOG_F(LS_ERROR)
1011 << "Wrong codec params to be registered as receive codec";
kwibergc13ded52016-06-17 06:00:45 -07001012 return -1;
1013 }
1014 auto codec_index = acm2::RentACodec::CodecIndexFromId(*codec_id);
1015 RTC_CHECK(codec_index) << "Invalid codec ID: " << static_cast<int>(*codec_id);
1016
1017 // Check if the payload-type is valid.
1018 if (!acm2::RentACodec::IsPayloadTypeValid(codec.pltype)) {
Mirko Bonadei675513b2017-11-09 11:09:25 +01001019 RTC_LOG_F(LS_ERROR) << "Invalid payload type " << codec.pltype << " for "
1020 << codec.plname;
kwibergc13ded52016-06-17 06:00:45 -07001021 return -1;
1022 }
1023
1024 AudioDecoder* isac_decoder = nullptr;
1025 if (STR_CASE_CMP(codec.plname, "isac") == 0) {
1026 std::unique_ptr<AudioDecoder>& saved_isac_decoder =
1027 codec.plfreq == 16000 ? isac_decoder_16k_ : isac_decoder_32k_;
1028 if (!saved_isac_decoder) {
1029 saved_isac_decoder = isac_factory();
1030 }
1031 isac_decoder = saved_isac_decoder.get();
1032 }
1033 return receiver_.AddCodec(*codec_index, codec.pltype, codec.channels,
1034 codec.plfreq, isac_decoder, codec.plname);
1035}
1036
1037int AudioCodingModuleImpl::RegisterExternalReceiveCodec(
1038 int rtp_payload_type,
1039 AudioDecoder* external_decoder,
1040 int sample_rate_hz,
1041 int num_channels,
1042 const std::string& name) {
1043 rtc::CritScope lock(&acm_crit_sect_);
1044 RTC_DCHECK(receiver_initialized_);
1045 if (num_channels > 2 || num_channels < 0) {
Mirko Bonadei675513b2017-11-09 11:09:25 +01001046 RTC_LOG_F(LS_ERROR) << "Unsupported number of channels: " << num_channels;
kwibergc13ded52016-06-17 06:00:45 -07001047 return -1;
1048 }
1049
1050 // Check if the payload-type is valid.
1051 if (!acm2::RentACodec::IsPayloadTypeValid(rtp_payload_type)) {
Mirko Bonadei675513b2017-11-09 11:09:25 +01001052 RTC_LOG_F(LS_ERROR) << "Invalid payload-type " << rtp_payload_type
1053 << " for external decoder.";
kwibergc13ded52016-06-17 06:00:45 -07001054 return -1;
1055 }
1056
1057 return receiver_.AddCodec(-1 /* external */, rtp_payload_type, num_channels,
1058 sample_rate_hz, external_decoder, name);
1059}
1060
1061// Get current received codec.
1062int AudioCodingModuleImpl::ReceiveCodec(CodecInst* current_codec) const {
1063 rtc::CritScope lock(&acm_crit_sect_);
1064 return receiver_.LastAudioCodec(current_codec);
1065}
1066
Danil Chapovalovb6021232018-06-19 13:26:36 +02001067absl::optional<SdpAudioFormat> AudioCodingModuleImpl::ReceiveFormat() const {
ossue280cde2016-10-12 11:04:10 -07001068 rtc::CritScope lock(&acm_crit_sect_);
1069 return receiver_.LastAudioFormat();
1070}
1071
kwibergc13ded52016-06-17 06:00:45 -07001072// Incoming packet from network parsed and ready for decode.
1073int AudioCodingModuleImpl::IncomingPacket(const uint8_t* incoming_payload,
1074 const size_t payload_length,
1075 const WebRtcRTPHeader& rtp_header) {
henrik.lundinb8c55b12017-05-10 07:38:01 -07001076 RTC_DCHECK_EQ(payload_length == 0, incoming_payload == nullptr);
kwibergc13ded52016-06-17 06:00:45 -07001077 return receiver_.InsertPacket(
1078 rtp_header,
1079 rtc::ArrayView<const uint8_t>(incoming_payload, payload_length));
1080}
1081
1082// Minimum playout delay (Used for lip-sync).
1083int AudioCodingModuleImpl::SetMinimumPlayoutDelay(int time_ms) {
1084 if ((time_ms < 0) || (time_ms > 10000)) {
Mirko Bonadei675513b2017-11-09 11:09:25 +01001085 RTC_LOG(LS_ERROR) << "Delay must be in the range of 0-10000 milliseconds.";
kwibergc13ded52016-06-17 06:00:45 -07001086 return -1;
1087 }
1088 return receiver_.SetMinimumDelay(time_ms);
1089}
1090
1091int AudioCodingModuleImpl::SetMaximumPlayoutDelay(int time_ms) {
1092 if ((time_ms < 0) || (time_ms > 10000)) {
Mirko Bonadei675513b2017-11-09 11:09:25 +01001093 RTC_LOG(LS_ERROR) << "Delay must be in the range of 0-10000 milliseconds.";
kwibergc13ded52016-06-17 06:00:45 -07001094 return -1;
1095 }
1096 return receiver_.SetMaximumDelay(time_ms);
1097}
1098
1099// Get 10 milliseconds of raw audio data to play out.
1100// Automatic resample to the requested frequency.
1101int AudioCodingModuleImpl::PlayoutData10Ms(int desired_freq_hz,
1102 AudioFrame* audio_frame,
1103 bool* muted) {
1104 // GetAudio always returns 10 ms, at the requested sample rate.
1105 if (receiver_.GetAudio(desired_freq_hz, audio_frame, muted) != 0) {
Mirko Bonadei675513b2017-11-09 11:09:25 +01001106 RTC_LOG(LS_ERROR) << "PlayoutData failed, RecOut Failed";
kwibergc13ded52016-06-17 06:00:45 -07001107 return -1;
1108 }
kwibergc13ded52016-06-17 06:00:45 -07001109 return 0;
1110}
1111
1112int AudioCodingModuleImpl::PlayoutData10Ms(int desired_freq_hz,
1113 AudioFrame* audio_frame) {
1114 bool muted;
1115 int ret = PlayoutData10Ms(desired_freq_hz, audio_frame, &muted);
1116 RTC_DCHECK(!muted);
1117 return ret;
1118}
1119
1120/////////////////////////////////////////
1121// Statistics
1122//
1123
1124// TODO(turajs) change the return value to void. Also change the corresponding
1125// NetEq function.
1126int AudioCodingModuleImpl::GetNetworkStatistics(NetworkStatistics* statistics) {
1127 receiver_.GetNetworkStatistics(statistics);
1128 return 0;
1129}
1130
1131int AudioCodingModuleImpl::RegisterVADCallback(ACMVADCallback* vad_callback) {
Mirko Bonadei675513b2017-11-09 11:09:25 +01001132 RTC_LOG(LS_VERBOSE) << "RegisterVADCallback()";
kwibergc13ded52016-06-17 06:00:45 -07001133 rtc::CritScope lock(&callback_crit_sect_);
1134 vad_callback_ = vad_callback;
1135 return 0;
1136}
1137
kwibergc13ded52016-06-17 06:00:45 -07001138int AudioCodingModuleImpl::SetOpusApplication(OpusApplicationMode application) {
1139 rtc::CritScope lock(&acm_crit_sect_);
1140 if (!HaveValidEncoder("SetOpusApplication")) {
1141 return -1;
1142 }
1143 AudioEncoder::Application app;
1144 switch (application) {
1145 case kVoip:
1146 app = AudioEncoder::Application::kSpeech;
1147 break;
1148 case kAudio:
1149 app = AudioEncoder::Application::kAudio;
1150 break;
1151 default:
1152 FATAL();
1153 return 0;
1154 }
1155 return encoder_stack_->SetApplication(app) ? 0 : -1;
1156}
1157
1158// Informs Opus encoder of the maximum playback rate the receiver will render.
1159int AudioCodingModuleImpl::SetOpusMaxPlaybackRate(int frequency_hz) {
1160 rtc::CritScope lock(&acm_crit_sect_);
1161 if (!HaveValidEncoder("SetOpusMaxPlaybackRate")) {
1162 return -1;
1163 }
1164 encoder_stack_->SetMaxPlaybackRate(frequency_hz);
1165 return 0;
1166}
1167
1168int AudioCodingModuleImpl::EnableOpusDtx() {
1169 rtc::CritScope lock(&acm_crit_sect_);
1170 if (!HaveValidEncoder("EnableOpusDtx")) {
1171 return -1;
1172 }
1173 return encoder_stack_->SetDtx(true) ? 0 : -1;
1174}
1175
1176int AudioCodingModuleImpl::DisableOpusDtx() {
1177 rtc::CritScope lock(&acm_crit_sect_);
1178 if (!HaveValidEncoder("DisableOpusDtx")) {
1179 return -1;
1180 }
1181 return encoder_stack_->SetDtx(false) ? 0 : -1;
1182}
1183
1184int32_t AudioCodingModuleImpl::PlayoutTimestamp(uint32_t* timestamp) {
Danil Chapovalovb6021232018-06-19 13:26:36 +02001185 absl::optional<uint32_t> ts = PlayoutTimestamp();
kwibergc13ded52016-06-17 06:00:45 -07001186 if (!ts)
1187 return -1;
1188 *timestamp = *ts;
1189 return 0;
1190}
1191
Danil Chapovalovb6021232018-06-19 13:26:36 +02001192absl::optional<uint32_t> AudioCodingModuleImpl::PlayoutTimestamp() {
kwibergc13ded52016-06-17 06:00:45 -07001193 return receiver_.GetPlayoutTimestamp();
1194}
1195
henrik.lundinb3f1c5d2016-08-22 15:39:53 -07001196int AudioCodingModuleImpl::FilteredCurrentDelayMs() const {
1197 return receiver_.FilteredCurrentDelayMs();
1198}
1199
Henrik Lundinabbff892017-11-29 09:14:04 +01001200int AudioCodingModuleImpl::TargetDelayMs() const {
1201 return receiver_.TargetDelayMs();
1202}
1203
kwibergc13ded52016-06-17 06:00:45 -07001204bool AudioCodingModuleImpl::HaveValidEncoder(const char* caller_name) const {
1205 if (!encoder_stack_) {
Mirko Bonadei675513b2017-11-09 11:09:25 +01001206 RTC_LOG(LS_ERROR) << caller_name << " failed: No send codec is registered.";
kwibergc13ded52016-06-17 06:00:45 -07001207 return false;
1208 }
1209 return true;
1210}
1211
1212int AudioCodingModuleImpl::UnregisterReceiveCodec(uint8_t payload_type) {
1213 return receiver_.RemoveCodec(payload_type);
1214}
1215
1216int AudioCodingModuleImpl::EnableNack(size_t max_nack_list_size) {
1217 return receiver_.EnableNack(max_nack_list_size);
1218}
1219
1220void AudioCodingModuleImpl::DisableNack() {
1221 receiver_.DisableNack();
1222}
1223
1224std::vector<uint16_t> AudioCodingModuleImpl::GetNackList(
1225 int64_t round_trip_time_ms) const {
1226 return receiver_.GetNackList(round_trip_time_ms);
1227}
1228
1229int AudioCodingModuleImpl::LeastRequiredDelayMs() const {
1230 return receiver_.LeastRequiredDelayMs();
1231}
1232
1233void AudioCodingModuleImpl::GetDecodingCallStatistics(
Yves Gerey665174f2018-06-19 15:03:05 +02001234 AudioDecodingCallStats* call_stats) const {
kwibergc13ded52016-06-17 06:00:45 -07001235 receiver_.GetDecodingCallStatistics(call_stats);
1236}
1237
ivoce1198e02017-09-08 08:13:19 -07001238ANAStats AudioCodingModuleImpl::GetANAStats() const {
1239 rtc::CritScope lock(&acm_crit_sect_);
1240 if (encoder_stack_)
1241 return encoder_stack_->GetANAStats();
1242 // If no encoder is set, return default stats.
1243 return ANAStats();
1244}
1245
kwibergc13ded52016-06-17 06:00:45 -07001246} // namespace
1247
Karl Wiberg5817d3d2018-04-06 10:06:42 +02001248AudioCodingModule::Config::Config(
1249 rtc::scoped_refptr<AudioDecoderFactory> decoder_factory)
1250 : neteq_config(),
1251 clock(Clock::GetRealTimeClock()),
1252 decoder_factory(decoder_factory) {
kwiberg36a43882016-08-29 05:33:32 -07001253 // Post-decode VAD is disabled by default in NetEq, however, Audio
1254 // Conference Mixer relies on VAD decisions and fails without them.
1255 neteq_config.enable_post_decode_vad = true;
1256}
1257
1258AudioCodingModule::Config::Config(const Config&) = default;
1259AudioCodingModule::Config::~Config() = default;
1260
Henrik Lundin64dad832015-05-11 12:44:23 +02001261AudioCodingModule* AudioCodingModule::Create(const Config& config) {
kwibergc13ded52016-06-17 06:00:45 -07001262 return new AudioCodingModuleImpl(config);
turaj@webrtc.org7959e162013-09-12 18:30:26 +00001263}
1264
turaj@webrtc.org7959e162013-09-12 18:30:26 +00001265int AudioCodingModule::NumberOfCodecs() {
kwibergfce4a942015-10-27 11:40:24 -07001266 return static_cast<int>(acm2::RentACodec::NumberOfCodecs());
turaj@webrtc.org7959e162013-09-12 18:30:26 +00001267}
1268
turaj@webrtc.org7959e162013-09-12 18:30:26 +00001269int AudioCodingModule::Codec(int list_id, CodecInst* codec) {
kwibergfce4a942015-10-27 11:40:24 -07001270 auto codec_id = acm2::RentACodec::CodecIdFromIndex(list_id);
1271 if (!codec_id)
1272 return -1;
1273 auto ci = acm2::RentACodec::CodecInstById(*codec_id);
1274 if (!ci)
1275 return -1;
1276 *codec = *ci;
1277 return 0;
turaj@webrtc.org7959e162013-09-12 18:30:26 +00001278}
1279
turaj@webrtc.org7959e162013-09-12 18:30:26 +00001280int AudioCodingModule::Codec(const char* payload_name,
1281 CodecInst* codec,
1282 int sampling_freq_hz,
Peter Kasting69558702016-01-12 16:26:35 -08001283 size_t channels) {
Danil Chapovalovb6021232018-06-19 13:26:36 +02001284 absl::optional<CodecInst> ci = acm2::RentACodec::CodecInstByParams(
turaj@webrtc.org6d5d2482013-10-06 04:47:28 +00001285 payload_name, sampling_freq_hz, channels);
kwibergfce4a942015-10-27 11:40:24 -07001286 if (ci) {
1287 *codec = *ci;
1288 return 0;
1289 } else {
1290 // We couldn't find a matching codec, so set the parameters to unacceptable
turaj@webrtc.org7959e162013-09-12 18:30:26 +00001291 // values and return.
1292 codec->plname[0] = '\0';
1293 codec->pltype = -1;
1294 codec->pacsize = 0;
1295 codec->rate = 0;
1296 codec->plfreq = 0;
1297 return -1;
1298 }
turaj@webrtc.org7959e162013-09-12 18:30:26 +00001299}
1300
turaj@webrtc.org7959e162013-09-12 18:30:26 +00001301int AudioCodingModule::Codec(const char* payload_name,
1302 int sampling_freq_hz,
Peter Kasting69558702016-01-12 16:26:35 -08001303 size_t channels) {
Danil Chapovalovb6021232018-06-19 13:26:36 +02001304 absl::optional<acm2::RentACodec::CodecId> ci =
Karl Wibergbe579832015-11-10 22:34:18 +01001305 acm2::RentACodec::CodecIdByParams(payload_name, sampling_freq_hz,
1306 channels);
kwibergfce4a942015-10-27 11:40:24 -07001307 if (!ci)
1308 return -1;
Danil Chapovalovb6021232018-06-19 13:26:36 +02001309 absl::optional<int> i = acm2::RentACodec::CodecIndexFromId(*ci);
kwibergfce4a942015-10-27 11:40:24 -07001310 return i ? *i : -1;
turaj@webrtc.org7959e162013-09-12 18:30:26 +00001311}
1312
1313// Checks the validity of the parameters of the given codec
1314bool AudioCodingModule::IsCodecValid(const CodecInst& codec) {
kwibergfce4a942015-10-27 11:40:24 -07001315 bool valid = acm2::RentACodec::IsCodecValid(codec);
1316 if (!valid)
Mirko Bonadei675513b2017-11-09 11:09:25 +01001317 RTC_LOG(LS_ERROR) << "Invalid codec setting";
kwibergfce4a942015-10-27 11:40:24 -07001318 return valid;
turaj@webrtc.org7959e162013-09-12 18:30:26 +00001319}
1320
turaj@webrtc.org7959e162013-09-12 18:30:26 +00001321} // namespace webrtc