blob: 1f108f5ed9dc78a94fed7acbfd6a1d3bf2ce7a29 [file] [log] [blame]
turaj@webrtc.org7959e162013-09-12 18:30:26 +00001/*
2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020011#include "modules/audio_coding/include/audio_coding_module.h"
turaj@webrtc.org7959e162013-09-12 18:30:26 +000012
Jonathan Yu36344a02017-07-30 01:55:34 -070013#include <algorithm>
14
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020015#include "api/audio_codecs/builtin_audio_decoder_factory.h"
16#include "modules/audio_coding/acm2/acm_receiver.h"
17#include "modules/audio_coding/acm2/acm_resampler.h"
18#include "modules/audio_coding/acm2/codec_manager.h"
19#include "modules/audio_coding/acm2/rent_a_codec.h"
20#include "rtc_base/checks.h"
21#include "rtc_base/logging.h"
22#include "rtc_base/safe_conversions.h"
23#include "system_wrappers/include/metrics.h"
turaj@webrtc.org7959e162013-09-12 18:30:26 +000024
25namespace webrtc {
26
kwibergc13ded52016-06-17 06:00:45 -070027namespace {
28
29struct EncoderFactory {
30 AudioEncoder* external_speech_encoder = nullptr;
31 acm2::CodecManager codec_manager;
32 acm2::RentACodec rent_a_codec;
33};
34
35class AudioCodingModuleImpl final : public AudioCodingModule {
36 public:
37 explicit AudioCodingModuleImpl(const AudioCodingModule::Config& config);
38 ~AudioCodingModuleImpl() override;
39
40 /////////////////////////////////////////
41 // Sender
42 //
43
44 // Can be called multiple times for Codec, CNG, RED.
45 int RegisterSendCodec(const CodecInst& send_codec) override;
46
47 void RegisterExternalSendCodec(
48 AudioEncoder* external_speech_encoder) override;
49
kwiberg24c7c122016-09-28 11:57:10 -070050 void ModifyEncoder(rtc::FunctionView<void(std::unique_ptr<AudioEncoder>*)>
51 modifier) override;
kwibergc13ded52016-06-17 06:00:45 -070052
kwiberg24c7c122016-09-28 11:57:10 -070053 void QueryEncoder(
54 rtc::FunctionView<void(const AudioEncoder*)> query) override;
ivoc85228d62016-07-27 04:53:47 -070055
kwibergc13ded52016-06-17 06:00:45 -070056 // Get current send codec.
57 rtc::Optional<CodecInst> SendCodec() const override;
58
59 // Get current send frequency.
60 int SendFrequency() const override;
61
62 // Sets the bitrate to the specified value in bits/sec. In case the codec does
63 // not support the requested value it will choose an appropriate value
64 // instead.
65 void SetBitRate(int bitrate_bps) override;
66
67 // Register a transport callback which will be
68 // called to deliver the encoded buffers.
69 int RegisterTransportCallback(AudioPacketizationCallback* transport) override;
70
71 // Add 10 ms of raw (PCM) audio data to the encoder.
72 int Add10MsData(const AudioFrame& audio_frame) override;
73
74 /////////////////////////////////////////
75 // (RED) Redundant Coding
76 //
77
78 // Configure RED status i.e. on/off.
79 int SetREDStatus(bool enable_red) override;
80
81 // Get RED status.
82 bool REDStatus() const override;
83
84 /////////////////////////////////////////
85 // (FEC) Forward Error Correction (codec internal)
86 //
87
88 // Configure FEC status i.e. on/off.
89 int SetCodecFEC(bool enabled_codec_fec) override;
90
91 // Get FEC status.
92 bool CodecFEC() const override;
93
94 // Set target packet loss rate
95 int SetPacketLossRate(int loss_rate) override;
96
97 /////////////////////////////////////////
98 // (VAD) Voice Activity Detection
99 // and
100 // (CNG) Comfort Noise Generation
101 //
102
103 int SetVAD(bool enable_dtx = true,
104 bool enable_vad = false,
105 ACMVADMode mode = VADNormal) override;
106
107 int VAD(bool* dtx_enabled,
108 bool* vad_enabled,
109 ACMVADMode* mode) const override;
110
111 int RegisterVADCallback(ACMVADCallback* vad_callback) override;
112
113 /////////////////////////////////////////
114 // Receiver
115 //
116
117 // Initialize receiver, resets codec database etc.
118 int InitializeReceiver() override;
119
120 // Get current receive frequency.
121 int ReceiveFrequency() const override;
122
123 // Get current playout frequency.
124 int PlayoutFrequency() const override;
125
kwiberg1c07c702017-03-27 07:15:49 -0700126 void SetReceiveCodecs(const std::map<int, SdpAudioFormat>& codecs) override;
127
kwiberg5adaf732016-10-04 09:33:27 -0700128 bool RegisterReceiveCodec(int rtp_payload_type,
129 const SdpAudioFormat& audio_format) override;
130
kwibergc13ded52016-06-17 06:00:45 -0700131 int RegisterReceiveCodec(const CodecInst& receive_codec) override;
132 int RegisterReceiveCodec(
133 const CodecInst& receive_codec,
kwiberg24c7c122016-09-28 11:57:10 -0700134 rtc::FunctionView<std::unique_ptr<AudioDecoder>()> isac_factory) override;
kwibergc13ded52016-06-17 06:00:45 -0700135
136 int RegisterExternalReceiveCodec(int rtp_payload_type,
137 AudioDecoder* external_decoder,
138 int sample_rate_hz,
139 int num_channels,
140 const std::string& name) override;
141
142 // Get current received codec.
143 int ReceiveCodec(CodecInst* current_codec) const override;
144
ossue280cde2016-10-12 11:04:10 -0700145 rtc::Optional<SdpAudioFormat> ReceiveFormat() const override;
146
kwibergc13ded52016-06-17 06:00:45 -0700147 // Incoming packet from network parsed and ready for decode.
148 int IncomingPacket(const uint8_t* incoming_payload,
149 const size_t payload_length,
150 const WebRtcRTPHeader& rtp_info) override;
151
152 // Incoming payloads, without rtp-info, the rtp-info will be created in ACM.
153 // One usage for this API is when pre-encoded files are pushed in ACM.
154 int IncomingPayload(const uint8_t* incoming_payload,
155 const size_t payload_length,
156 uint8_t payload_type,
157 uint32_t timestamp) override;
158
159 // Minimum playout delay.
160 int SetMinimumPlayoutDelay(int time_ms) override;
161
162 // Maximum playout delay.
163 int SetMaximumPlayoutDelay(int time_ms) override;
164
165 // Smallest latency NetEq will maintain.
166 int LeastRequiredDelayMs() const override;
167
168 RTC_DEPRECATED int32_t PlayoutTimestamp(uint32_t* timestamp) override;
169
170 rtc::Optional<uint32_t> PlayoutTimestamp() override;
171
henrik.lundinb3f1c5d2016-08-22 15:39:53 -0700172 int FilteredCurrentDelayMs() const override;
173
kwibergc13ded52016-06-17 06:00:45 -0700174 // Get 10 milliseconds of raw audio data to play out, and
175 // automatic resample to the requested frequency if > 0.
176 int PlayoutData10Ms(int desired_freq_hz,
177 AudioFrame* audio_frame,
178 bool* muted) override;
179 int PlayoutData10Ms(int desired_freq_hz, AudioFrame* audio_frame) override;
180
181 /////////////////////////////////////////
182 // Statistics
183 //
184
185 int GetNetworkStatistics(NetworkStatistics* statistics) override;
186
187 int SetOpusApplication(OpusApplicationMode application) override;
188
189 // If current send codec is Opus, informs it about the maximum playback rate
190 // the receiver will render.
191 int SetOpusMaxPlaybackRate(int frequency_hz) override;
192
193 int EnableOpusDtx() override;
194
195 int DisableOpusDtx() override;
196
197 int UnregisterReceiveCodec(uint8_t payload_type) override;
198
199 int EnableNack(size_t max_nack_list_size) override;
200
201 void DisableNack() override;
202
203 std::vector<uint16_t> GetNackList(int64_t round_trip_time_ms) const override;
204
205 void GetDecodingCallStatistics(AudioDecodingCallStats* stats) const override;
206
ivoce1198e02017-09-08 08:13:19 -0700207 ANAStats GetANAStats() const override;
208
kwibergc13ded52016-06-17 06:00:45 -0700209 private:
210 struct InputData {
211 uint32_t input_timestamp;
212 const int16_t* audio;
213 size_t length_per_channel;
214 size_t audio_channel;
215 // If a re-mix is required (up or down), this buffer will store a re-mixed
216 // version of the input.
217 int16_t buffer[WEBRTC_10MS_PCM_AUDIO];
218 };
219
220 // This member class writes values to the named UMA histogram, but only if
221 // the value has changed since the last time (and always for the first call).
222 class ChangeLogger {
223 public:
224 explicit ChangeLogger(const std::string& histogram_name)
225 : histogram_name_(histogram_name) {}
226 // Logs the new value if it is different from the last logged value, or if
227 // this is the first call.
228 void MaybeLog(int value);
229
230 private:
231 int last_value_ = 0;
232 int first_time_ = true;
233 const std::string histogram_name_;
234 };
235
236 int RegisterReceiveCodecUnlocked(
237 const CodecInst& codec,
kwiberg24c7c122016-09-28 11:57:10 -0700238 rtc::FunctionView<std::unique_ptr<AudioDecoder>()> isac_factory)
danilchap56359be2017-09-07 07:53:45 -0700239 RTC_EXCLUSIVE_LOCKS_REQUIRED(acm_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700240
241 int Add10MsDataInternal(const AudioFrame& audio_frame, InputData* input_data)
danilchap56359be2017-09-07 07:53:45 -0700242 RTC_EXCLUSIVE_LOCKS_REQUIRED(acm_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700243 int Encode(const InputData& input_data)
danilchap56359be2017-09-07 07:53:45 -0700244 RTC_EXCLUSIVE_LOCKS_REQUIRED(acm_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700245
danilchap56359be2017-09-07 07:53:45 -0700246 int InitializeReceiverSafe() RTC_EXCLUSIVE_LOCKS_REQUIRED(acm_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700247
248 bool HaveValidEncoder(const char* caller_name) const
danilchap56359be2017-09-07 07:53:45 -0700249 RTC_EXCLUSIVE_LOCKS_REQUIRED(acm_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700250
251 // Preprocessing of input audio, including resampling and down-mixing if
252 // required, before pushing audio into encoder's buffer.
253 //
254 // in_frame: input audio-frame
255 // ptr_out: pointer to output audio_frame. If no preprocessing is required
256 // |ptr_out| will be pointing to |in_frame|, otherwise pointing to
257 // |preprocess_frame_|.
258 //
259 // Return value:
260 // -1: if encountering an error.
261 // 0: otherwise.
262 int PreprocessToAddData(const AudioFrame& in_frame,
263 const AudioFrame** ptr_out)
danilchap56359be2017-09-07 07:53:45 -0700264 RTC_EXCLUSIVE_LOCKS_REQUIRED(acm_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700265
266 // Change required states after starting to receive the codec corresponding
267 // to |index|.
268 int UpdateUponReceivingCodec(int index);
269
270 rtc::CriticalSection acm_crit_sect_;
danilchap56359be2017-09-07 07:53:45 -0700271 rtc::Buffer encode_buffer_ RTC_GUARDED_BY(acm_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700272 int id_; // TODO(henrik.lundin) Make const.
danilchap56359be2017-09-07 07:53:45 -0700273 uint32_t expected_codec_ts_ RTC_GUARDED_BY(acm_crit_sect_);
274 uint32_t expected_in_ts_ RTC_GUARDED_BY(acm_crit_sect_);
275 acm2::ACMResampler resampler_ RTC_GUARDED_BY(acm_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700276 acm2::AcmReceiver receiver_; // AcmReceiver has it's own internal lock.
danilchap56359be2017-09-07 07:53:45 -0700277 ChangeLogger bitrate_logger_ RTC_GUARDED_BY(acm_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700278
danilchap56359be2017-09-07 07:53:45 -0700279 std::unique_ptr<EncoderFactory> encoder_factory_
280 RTC_GUARDED_BY(acm_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700281
282 // Current encoder stack, either obtained from
283 // encoder_factory_->rent_a_codec.RentEncoderStack or provided by a call to
284 // RegisterEncoder.
danilchap56359be2017-09-07 07:53:45 -0700285 std::unique_ptr<AudioEncoder> encoder_stack_ RTC_GUARDED_BY(acm_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700286
danilchap56359be2017-09-07 07:53:45 -0700287 std::unique_ptr<AudioDecoder> isac_decoder_16k_
288 RTC_GUARDED_BY(acm_crit_sect_);
289 std::unique_ptr<AudioDecoder> isac_decoder_32k_
290 RTC_GUARDED_BY(acm_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700291
292 // This is to keep track of CN instances where we can send DTMFs.
danilchap56359be2017-09-07 07:53:45 -0700293 uint8_t previous_pltype_ RTC_GUARDED_BY(acm_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700294
295 // Used when payloads are pushed into ACM without any RTP info
296 // One example is when pre-encoded bit-stream is pushed from
297 // a file.
298 // IMPORTANT: this variable is only used in IncomingPayload(), therefore,
299 // no lock acquired when interacting with this variable. If it is going to
300 // be used in other methods, locks need to be taken.
301 std::unique_ptr<WebRtcRTPHeader> aux_rtp_header_;
302
danilchap56359be2017-09-07 07:53:45 -0700303 bool receiver_initialized_ RTC_GUARDED_BY(acm_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700304
danilchap56359be2017-09-07 07:53:45 -0700305 AudioFrame preprocess_frame_ RTC_GUARDED_BY(acm_crit_sect_);
306 bool first_10ms_data_ RTC_GUARDED_BY(acm_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700307
danilchap56359be2017-09-07 07:53:45 -0700308 bool first_frame_ RTC_GUARDED_BY(acm_crit_sect_);
309 uint32_t last_timestamp_ RTC_GUARDED_BY(acm_crit_sect_);
310 uint32_t last_rtp_timestamp_ RTC_GUARDED_BY(acm_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700311
312 rtc::CriticalSection callback_crit_sect_;
313 AudioPacketizationCallback* packetization_callback_
danilchap56359be2017-09-07 07:53:45 -0700314 RTC_GUARDED_BY(callback_crit_sect_);
315 ACMVADCallback* vad_callback_ RTC_GUARDED_BY(callback_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700316
317 int codec_histogram_bins_log_[static_cast<size_t>(
318 AudioEncoder::CodecType::kMaxLoggedAudioCodecTypes)];
319 int number_of_consecutive_empty_packets_;
320};
321
322// Adds a codec usage sample to the histogram.
323void UpdateCodecTypeHistogram(size_t codec_type) {
324 RTC_HISTOGRAM_ENUMERATION(
325 "WebRTC.Audio.Encoder.CodecType", static_cast<int>(codec_type),
326 static_cast<int>(
327 webrtc::AudioEncoder::CodecType::kMaxLoggedAudioCodecTypes));
328}
329
kwibergc13ded52016-06-17 06:00:45 -0700330// Stereo-to-mono can be used as in-place.
331int DownMix(const AudioFrame& frame,
332 size_t length_out_buff,
333 int16_t* out_buff) {
yujo36b1a5f2017-06-12 12:45:32 -0700334 RTC_DCHECK_EQ(frame.num_channels_, 2);
335 RTC_DCHECK_GE(length_out_buff, frame.samples_per_channel_);
336
337 if (!frame.muted()) {
338 const int16_t* frame_data = frame.data();
339 for (size_t n = 0; n < frame.samples_per_channel_; ++n) {
340 out_buff[n] = static_cast<int16_t>(
341 (static_cast<int32_t>(frame_data[2 * n]) +
342 static_cast<int32_t>(frame_data[2 * n + 1])) >> 1);
343 }
344 } else {
Jonathan Yu36344a02017-07-30 01:55:34 -0700345 std::fill(out_buff, out_buff + frame.samples_per_channel_, 0);
kwibergc13ded52016-06-17 06:00:45 -0700346 }
kwibergc13ded52016-06-17 06:00:45 -0700347 return 0;
348}
349
350// Mono-to-stereo can be used as in-place.
351int UpMix(const AudioFrame& frame, size_t length_out_buff, int16_t* out_buff) {
yujo36b1a5f2017-06-12 12:45:32 -0700352 RTC_DCHECK_EQ(frame.num_channels_, 1);
353 RTC_DCHECK_GE(length_out_buff, 2 * frame.samples_per_channel_);
354
355 if (!frame.muted()) {
356 const int16_t* frame_data = frame.data();
357 for (size_t n = frame.samples_per_channel_; n != 0; --n) {
358 size_t i = n - 1;
359 int16_t sample = frame_data[i];
360 out_buff[2 * i + 1] = sample;
361 out_buff[2 * i] = sample;
362 }
363 } else {
Jonathan Yu36344a02017-07-30 01:55:34 -0700364 std::fill(out_buff, out_buff + frame.samples_per_channel_ * 2, 0);
kwibergc13ded52016-06-17 06:00:45 -0700365 }
366 return 0;
367}
368
369void ConvertEncodedInfoToFragmentationHeader(
370 const AudioEncoder::EncodedInfo& info,
371 RTPFragmentationHeader* frag) {
372 if (info.redundant.empty()) {
373 frag->fragmentationVectorSize = 0;
374 return;
375 }
376
377 frag->VerifyAndAllocateFragmentationHeader(
378 static_cast<uint16_t>(info.redundant.size()));
379 frag->fragmentationVectorSize = static_cast<uint16_t>(info.redundant.size());
380 size_t offset = 0;
381 for (size_t i = 0; i < info.redundant.size(); ++i) {
382 frag->fragmentationOffset[i] = offset;
383 offset += info.redundant[i].encoded_bytes;
384 frag->fragmentationLength[i] = info.redundant[i].encoded_bytes;
kwibergd3edd772017-03-01 18:52:48 -0800385 frag->fragmentationTimeDiff[i] = rtc::dchecked_cast<uint16_t>(
kwibergc13ded52016-06-17 06:00:45 -0700386 info.encoded_timestamp - info.redundant[i].encoded_timestamp);
387 frag->fragmentationPlType[i] = info.redundant[i].payload_type;
388 }
389}
390
391// Wraps a raw AudioEncoder pointer. The idea is that you can put one of these
392// in a unique_ptr, to protect the contained raw pointer from being deleted
393// when the unique_ptr expires. (This is of course a bad idea in general, but
394// backwards compatibility.)
395class RawAudioEncoderWrapper final : public AudioEncoder {
396 public:
397 RawAudioEncoderWrapper(AudioEncoder* enc) : enc_(enc) {}
398 int SampleRateHz() const override { return enc_->SampleRateHz(); }
399 size_t NumChannels() const override { return enc_->NumChannels(); }
400 int RtpTimestampRateHz() const override { return enc_->RtpTimestampRateHz(); }
401 size_t Num10MsFramesInNextPacket() const override {
402 return enc_->Num10MsFramesInNextPacket();
403 }
404 size_t Max10MsFramesInAPacket() const override {
405 return enc_->Max10MsFramesInAPacket();
406 }
407 int GetTargetBitrate() const override { return enc_->GetTargetBitrate(); }
408 EncodedInfo EncodeImpl(uint32_t rtp_timestamp,
409 rtc::ArrayView<const int16_t> audio,
410 rtc::Buffer* encoded) override {
411 return enc_->Encode(rtp_timestamp, audio, encoded);
412 }
413 void Reset() override { return enc_->Reset(); }
414 bool SetFec(bool enable) override { return enc_->SetFec(enable); }
415 bool SetDtx(bool enable) override { return enc_->SetDtx(enable); }
416 bool SetApplication(Application application) override {
417 return enc_->SetApplication(application);
418 }
419 void SetMaxPlaybackRate(int frequency_hz) override {
420 return enc_->SetMaxPlaybackRate(frequency_hz);
421 }
kwibergc13ded52016-06-17 06:00:45 -0700422
423 private:
424 AudioEncoder* enc_;
425};
426
427// Return false on error.
428bool CreateSpeechEncoderIfNecessary(EncoderFactory* ef) {
429 auto* sp = ef->codec_manager.GetStackParams();
430 if (sp->speech_encoder) {
431 // Do nothing; we already have a speech encoder.
432 } else if (ef->codec_manager.GetCodecInst()) {
433 RTC_DCHECK(!ef->external_speech_encoder);
434 // We have no speech encoder, but we have a specification for making one.
435 std::unique_ptr<AudioEncoder> enc =
436 ef->rent_a_codec.RentEncoder(*ef->codec_manager.GetCodecInst());
437 if (!enc)
438 return false; // Encoder spec was bad.
439 sp->speech_encoder = std::move(enc);
440 } else if (ef->external_speech_encoder) {
441 RTC_DCHECK(!ef->codec_manager.GetCodecInst());
442 // We have an external speech encoder.
443 sp->speech_encoder = std::unique_ptr<AudioEncoder>(
444 new RawAudioEncoderWrapper(ef->external_speech_encoder));
445 }
446 return true;
447}
448
449void AudioCodingModuleImpl::ChangeLogger::MaybeLog(int value) {
450 if (value != last_value_ || first_time_) {
451 first_time_ = false;
452 last_value_ = value;
453 RTC_HISTOGRAM_COUNTS_SPARSE_100(histogram_name_, value);
454 }
455}
456
457AudioCodingModuleImpl::AudioCodingModuleImpl(
458 const AudioCodingModule::Config& config)
459 : id_(config.id),
460 expected_codec_ts_(0xD87F3F9F),
461 expected_in_ts_(0xD87F3F9F),
462 receiver_(config),
463 bitrate_logger_("WebRTC.Audio.TargetBitrateInKbps"),
464 encoder_factory_(new EncoderFactory),
465 encoder_stack_(nullptr),
466 previous_pltype_(255),
467 receiver_initialized_(false),
468 first_10ms_data_(false),
469 first_frame_(true),
470 packetization_callback_(NULL),
471 vad_callback_(NULL),
472 codec_histogram_bins_log_(),
473 number_of_consecutive_empty_packets_(0) {
474 if (InitializeReceiverSafe() < 0) {
Alex Loiko300ec8c2017-05-30 17:23:28 +0200475 LOG(LS_ERROR) << "Cannot initialize receiver";
kwibergc13ded52016-06-17 06:00:45 -0700476 }
Alex Loiko300ec8c2017-05-30 17:23:28 +0200477 LOG(LS_INFO) << "Created";
kwibergc13ded52016-06-17 06:00:45 -0700478}
479
480AudioCodingModuleImpl::~AudioCodingModuleImpl() = default;
481
482int32_t AudioCodingModuleImpl::Encode(const InputData& input_data) {
483 AudioEncoder::EncodedInfo encoded_info;
484 uint8_t previous_pltype;
485
486 // Check if there is an encoder before.
487 if (!HaveValidEncoder("Process"))
488 return -1;
489
ossu63fb95a2016-07-06 09:34:22 -0700490 if(!first_frame_) {
deadbeeffcada902016-08-24 12:45:13 -0700491 RTC_DCHECK(IsNewerTimestamp(input_data.input_timestamp, last_timestamp_))
ossu63fb95a2016-07-06 09:34:22 -0700492 << "Time should not move backwards";
493 }
494
kwibergc13ded52016-06-17 06:00:45 -0700495 // Scale the timestamp to the codec's RTP timestamp rate.
496 uint32_t rtp_timestamp =
497 first_frame_ ? input_data.input_timestamp
498 : last_rtp_timestamp_ +
499 rtc::CheckedDivExact(
500 input_data.input_timestamp - last_timestamp_,
501 static_cast<uint32_t>(rtc::CheckedDivExact(
502 encoder_stack_->SampleRateHz(),
503 encoder_stack_->RtpTimestampRateHz())));
504 last_timestamp_ = input_data.input_timestamp;
505 last_rtp_timestamp_ = rtp_timestamp;
506 first_frame_ = false;
507
508 // Clear the buffer before reuse - encoded data will get appended.
509 encode_buffer_.Clear();
510 encoded_info = encoder_stack_->Encode(
511 rtp_timestamp, rtc::ArrayView<const int16_t>(
512 input_data.audio, input_data.audio_channel *
513 input_data.length_per_channel),
514 &encode_buffer_);
515
516 bitrate_logger_.MaybeLog(encoder_stack_->GetTargetBitrate() / 1000);
517 if (encode_buffer_.size() == 0 && !encoded_info.send_even_if_empty) {
518 // Not enough data.
519 return 0;
520 }
521 previous_pltype = previous_pltype_; // Read it while we have the critsect.
522
523 // Log codec type to histogram once every 500 packets.
524 if (encoded_info.encoded_bytes == 0) {
525 ++number_of_consecutive_empty_packets_;
526 } else {
527 size_t codec_type = static_cast<size_t>(encoded_info.encoder_type);
528 codec_histogram_bins_log_[codec_type] +=
529 number_of_consecutive_empty_packets_ + 1;
530 number_of_consecutive_empty_packets_ = 0;
531 if (codec_histogram_bins_log_[codec_type] >= 500) {
532 codec_histogram_bins_log_[codec_type] -= 500;
533 UpdateCodecTypeHistogram(codec_type);
534 }
535 }
536
537 RTPFragmentationHeader my_fragmentation;
538 ConvertEncodedInfoToFragmentationHeader(encoded_info, &my_fragmentation);
539 FrameType frame_type;
540 if (encode_buffer_.size() == 0 && encoded_info.send_even_if_empty) {
541 frame_type = kEmptyFrame;
542 encoded_info.payload_type = previous_pltype;
543 } else {
kwibergaf476c72016-11-28 15:21:39 -0800544 RTC_DCHECK_GT(encode_buffer_.size(), 0);
kwibergc13ded52016-06-17 06:00:45 -0700545 frame_type = encoded_info.speech ? kAudioFrameSpeech : kAudioFrameCN;
546 }
547
548 {
549 rtc::CritScope lock(&callback_crit_sect_);
550 if (packetization_callback_) {
551 packetization_callback_->SendData(
552 frame_type, encoded_info.payload_type, encoded_info.encoded_timestamp,
553 encode_buffer_.data(), encode_buffer_.size(),
554 my_fragmentation.fragmentationVectorSize > 0 ? &my_fragmentation
555 : nullptr);
556 }
557
558 if (vad_callback_) {
559 // Callback with VAD decision.
560 vad_callback_->InFrameType(frame_type);
561 }
562 }
563 previous_pltype_ = encoded_info.payload_type;
564 return static_cast<int32_t>(encode_buffer_.size());
565}
566
567/////////////////////////////////////////
568// Sender
569//
570
571// Can be called multiple times for Codec, CNG, RED.
572int AudioCodingModuleImpl::RegisterSendCodec(const CodecInst& send_codec) {
573 rtc::CritScope lock(&acm_crit_sect_);
574 if (!encoder_factory_->codec_manager.RegisterEncoder(send_codec)) {
575 return -1;
576 }
577 if (encoder_factory_->codec_manager.GetCodecInst()) {
578 encoder_factory_->external_speech_encoder = nullptr;
579 }
580 if (!CreateSpeechEncoderIfNecessary(encoder_factory_.get())) {
581 return -1;
582 }
583 auto* sp = encoder_factory_->codec_manager.GetStackParams();
584 if (sp->speech_encoder)
585 encoder_stack_ = encoder_factory_->rent_a_codec.RentEncoderStack(sp);
586 return 0;
587}
588
589void AudioCodingModuleImpl::RegisterExternalSendCodec(
590 AudioEncoder* external_speech_encoder) {
591 rtc::CritScope lock(&acm_crit_sect_);
592 encoder_factory_->codec_manager.UnsetCodecInst();
593 encoder_factory_->external_speech_encoder = external_speech_encoder;
594 RTC_CHECK(CreateSpeechEncoderIfNecessary(encoder_factory_.get()));
595 auto* sp = encoder_factory_->codec_manager.GetStackParams();
596 RTC_CHECK(sp->speech_encoder);
597 encoder_stack_ = encoder_factory_->rent_a_codec.RentEncoderStack(sp);
598}
599
600void AudioCodingModuleImpl::ModifyEncoder(
kwiberg24c7c122016-09-28 11:57:10 -0700601 rtc::FunctionView<void(std::unique_ptr<AudioEncoder>*)> modifier) {
kwibergc13ded52016-06-17 06:00:45 -0700602 rtc::CritScope lock(&acm_crit_sect_);
603
604 // Wipe the encoder factory, so that everything that relies on it will fail.
605 // We don't want the complexity of supporting swapping back and forth.
606 if (encoder_factory_) {
607 encoder_factory_.reset();
608 RTC_CHECK(!encoder_stack_); // Ensure we hadn't started using the factory.
609 }
610
611 modifier(&encoder_stack_);
612}
613
ivoc85228d62016-07-27 04:53:47 -0700614void AudioCodingModuleImpl::QueryEncoder(
kwiberg24c7c122016-09-28 11:57:10 -0700615 rtc::FunctionView<void(const AudioEncoder*)> query) {
ivoc85228d62016-07-27 04:53:47 -0700616 rtc::CritScope lock(&acm_crit_sect_);
617 query(encoder_stack_.get());
618}
619
kwibergc13ded52016-06-17 06:00:45 -0700620// Get current send codec.
621rtc::Optional<CodecInst> AudioCodingModuleImpl::SendCodec() const {
622 rtc::CritScope lock(&acm_crit_sect_);
623 if (encoder_factory_) {
624 auto* ci = encoder_factory_->codec_manager.GetCodecInst();
625 if (ci) {
626 return rtc::Optional<CodecInst>(*ci);
627 }
628 CreateSpeechEncoderIfNecessary(encoder_factory_.get());
629 const std::unique_ptr<AudioEncoder>& enc =
630 encoder_factory_->codec_manager.GetStackParams()->speech_encoder;
631 if (enc) {
632 return rtc::Optional<CodecInst>(
633 acm2::CodecManager::ForgeCodecInst(enc.get()));
634 }
635 return rtc::Optional<CodecInst>();
636 } else {
637 return encoder_stack_
638 ? rtc::Optional<CodecInst>(
639 acm2::CodecManager::ForgeCodecInst(encoder_stack_.get()))
640 : rtc::Optional<CodecInst>();
641 }
642}
643
644// Get current send frequency.
645int AudioCodingModuleImpl::SendFrequency() const {
kwibergc13ded52016-06-17 06:00:45 -0700646 rtc::CritScope lock(&acm_crit_sect_);
647
648 if (!encoder_stack_) {
Noah Richardsbc8ee332017-07-07 13:22:45 -0700649 LOG(LS_ERROR) << "SendFrequency Failed, no codec is registered";
kwibergc13ded52016-06-17 06:00:45 -0700650 return -1;
651 }
652
653 return encoder_stack_->SampleRateHz();
654}
655
656void AudioCodingModuleImpl::SetBitRate(int bitrate_bps) {
657 rtc::CritScope lock(&acm_crit_sect_);
658 if (encoder_stack_) {
michaelt566d8202017-01-12 10:17:38 -0800659 encoder_stack_->OnReceivedUplinkBandwidth(bitrate_bps,
660 rtc::Optional<int64_t>());
kwibergc13ded52016-06-17 06:00:45 -0700661 }
662}
663
664// Register a transport callback which will be called to deliver
665// the encoded buffers.
666int AudioCodingModuleImpl::RegisterTransportCallback(
667 AudioPacketizationCallback* transport) {
668 rtc::CritScope lock(&callback_crit_sect_);
669 packetization_callback_ = transport;
670 return 0;
671}
672
673// Add 10MS of raw (PCM) audio data to the encoder.
674int AudioCodingModuleImpl::Add10MsData(const AudioFrame& audio_frame) {
675 InputData input_data;
676 rtc::CritScope lock(&acm_crit_sect_);
677 int r = Add10MsDataInternal(audio_frame, &input_data);
678 return r < 0 ? r : Encode(input_data);
679}
680
681int AudioCodingModuleImpl::Add10MsDataInternal(const AudioFrame& audio_frame,
682 InputData* input_data) {
683 if (audio_frame.samples_per_channel_ == 0) {
684 assert(false);
Alex Loiko300ec8c2017-05-30 17:23:28 +0200685 LOG(LS_ERROR) << "Cannot Add 10 ms audio, payload length is zero";
kwibergc13ded52016-06-17 06:00:45 -0700686 return -1;
687 }
688
689 if (audio_frame.sample_rate_hz_ > 48000) {
690 assert(false);
Alex Loiko300ec8c2017-05-30 17:23:28 +0200691 LOG(LS_ERROR) << "Cannot Add 10 ms audio, input frequency not valid";
kwibergc13ded52016-06-17 06:00:45 -0700692 return -1;
693 }
694
695 // If the length and frequency matches. We currently just support raw PCM.
696 if (static_cast<size_t>(audio_frame.sample_rate_hz_ / 100) !=
697 audio_frame.samples_per_channel_) {
Alex Loiko300ec8c2017-05-30 17:23:28 +0200698 LOG(LS_ERROR)
699 << "Cannot Add 10 ms audio, input frequency and length doesn't match";
kwibergc13ded52016-06-17 06:00:45 -0700700 return -1;
701 }
702
703 if (audio_frame.num_channels_ != 1 && audio_frame.num_channels_ != 2) {
Alex Loiko300ec8c2017-05-30 17:23:28 +0200704 LOG(LS_ERROR) << "Cannot Add 10 ms audio, invalid number of channels.";
kwibergc13ded52016-06-17 06:00:45 -0700705 return -1;
706 }
707
708 // Do we have a codec registered?
709 if (!HaveValidEncoder("Add10MsData")) {
710 return -1;
711 }
712
713 const AudioFrame* ptr_frame;
714 // Perform a resampling, also down-mix if it is required and can be
715 // performed before resampling (a down mix prior to resampling will take
716 // place if both primary and secondary encoders are mono and input is in
717 // stereo).
718 if (PreprocessToAddData(audio_frame, &ptr_frame) < 0) {
719 return -1;
720 }
721
722 // Check whether we need an up-mix or down-mix?
723 const size_t current_num_channels = encoder_stack_->NumChannels();
724 const bool same_num_channels =
725 ptr_frame->num_channels_ == current_num_channels;
726
727 if (!same_num_channels) {
728 if (ptr_frame->num_channels_ == 1) {
729 if (UpMix(*ptr_frame, WEBRTC_10MS_PCM_AUDIO, input_data->buffer) < 0)
730 return -1;
731 } else {
732 if (DownMix(*ptr_frame, WEBRTC_10MS_PCM_AUDIO, input_data->buffer) < 0)
733 return -1;
734 }
735 }
736
737 // When adding data to encoders this pointer is pointing to an audio buffer
738 // with correct number of channels.
yujo36b1a5f2017-06-12 12:45:32 -0700739 const int16_t* ptr_audio = ptr_frame->data();
kwibergc13ded52016-06-17 06:00:45 -0700740
741 // For pushing data to primary, point the |ptr_audio| to correct buffer.
742 if (!same_num_channels)
743 ptr_audio = input_data->buffer;
744
yujo36b1a5f2017-06-12 12:45:32 -0700745 // TODO(yujo): Skip encode of muted frames.
kwibergc13ded52016-06-17 06:00:45 -0700746 input_data->input_timestamp = ptr_frame->timestamp_;
747 input_data->audio = ptr_audio;
748 input_data->length_per_channel = ptr_frame->samples_per_channel_;
749 input_data->audio_channel = current_num_channels;
750
751 return 0;
752}
753
754// Perform a resampling and down-mix if required. We down-mix only if
755// encoder is mono and input is stereo. In case of dual-streaming, both
756// encoders has to be mono for down-mix to take place.
757// |*ptr_out| will point to the pre-processed audio-frame. If no pre-processing
758// is required, |*ptr_out| points to |in_frame|.
yujo36b1a5f2017-06-12 12:45:32 -0700759// TODO(yujo): Make this more efficient for muted frames.
kwibergc13ded52016-06-17 06:00:45 -0700760int AudioCodingModuleImpl::PreprocessToAddData(const AudioFrame& in_frame,
761 const AudioFrame** ptr_out) {
762 const bool resample =
763 in_frame.sample_rate_hz_ != encoder_stack_->SampleRateHz();
764
765 // This variable is true if primary codec and secondary codec (if exists)
766 // are both mono and input is stereo.
767 // TODO(henrik.lundin): This condition should probably be
768 // in_frame.num_channels_ > encoder_stack_->NumChannels()
769 const bool down_mix =
770 in_frame.num_channels_ == 2 && encoder_stack_->NumChannels() == 1;
771
772 if (!first_10ms_data_) {
773 expected_in_ts_ = in_frame.timestamp_;
774 expected_codec_ts_ = in_frame.timestamp_;
775 first_10ms_data_ = true;
776 } else if (in_frame.timestamp_ != expected_in_ts_) {
ossu63fb95a2016-07-06 09:34:22 -0700777 LOG(LS_WARNING) << "Unexpected input timestamp: " << in_frame.timestamp_
778 << ", expected: " << expected_in_ts_;
kwibergc13ded52016-06-17 06:00:45 -0700779 expected_codec_ts_ +=
780 (in_frame.timestamp_ - expected_in_ts_) *
781 static_cast<uint32_t>(
782 static_cast<double>(encoder_stack_->SampleRateHz()) /
783 static_cast<double>(in_frame.sample_rate_hz_));
784 expected_in_ts_ = in_frame.timestamp_;
785 }
786
787
788 if (!down_mix && !resample) {
789 // No pre-processing is required.
ossu63fb95a2016-07-06 09:34:22 -0700790 if (expected_in_ts_ == expected_codec_ts_) {
791 // If we've never resampled, we can use the input frame as-is
792 *ptr_out = &in_frame;
793 } else {
794 // Otherwise we'll need to alter the timestamp. Since in_frame is const,
795 // we'll have to make a copy of it.
796 preprocess_frame_.CopyFrom(in_frame);
797 preprocess_frame_.timestamp_ = expected_codec_ts_;
798 *ptr_out = &preprocess_frame_;
799 }
800
kwibergc13ded52016-06-17 06:00:45 -0700801 expected_in_ts_ += static_cast<uint32_t>(in_frame.samples_per_channel_);
802 expected_codec_ts_ += static_cast<uint32_t>(in_frame.samples_per_channel_);
kwibergc13ded52016-06-17 06:00:45 -0700803 return 0;
804 }
805
806 *ptr_out = &preprocess_frame_;
807 preprocess_frame_.num_channels_ = in_frame.num_channels_;
808 int16_t audio[WEBRTC_10MS_PCM_AUDIO];
yujo36b1a5f2017-06-12 12:45:32 -0700809 const int16_t* src_ptr_audio = in_frame.data();
kwibergc13ded52016-06-17 06:00:45 -0700810 if (down_mix) {
811 // If a resampling is required the output of a down-mix is written into a
812 // local buffer, otherwise, it will be written to the output frame.
yujo36b1a5f2017-06-12 12:45:32 -0700813 int16_t* dest_ptr_audio = resample ?
814 audio : preprocess_frame_.mutable_data();
kwibergc13ded52016-06-17 06:00:45 -0700815 if (DownMix(in_frame, WEBRTC_10MS_PCM_AUDIO, dest_ptr_audio) < 0)
816 return -1;
817 preprocess_frame_.num_channels_ = 1;
818 // Set the input of the resampler is the down-mixed signal.
819 src_ptr_audio = audio;
820 }
821
822 preprocess_frame_.timestamp_ = expected_codec_ts_;
823 preprocess_frame_.samples_per_channel_ = in_frame.samples_per_channel_;
824 preprocess_frame_.sample_rate_hz_ = in_frame.sample_rate_hz_;
825 // If it is required, we have to do a resampling.
826 if (resample) {
827 // The result of the resampler is written to output frame.
yujo36b1a5f2017-06-12 12:45:32 -0700828 int16_t* dest_ptr_audio = preprocess_frame_.mutable_data();
kwibergc13ded52016-06-17 06:00:45 -0700829
830 int samples_per_channel = resampler_.Resample10Msec(
831 src_ptr_audio, in_frame.sample_rate_hz_, encoder_stack_->SampleRateHz(),
832 preprocess_frame_.num_channels_, AudioFrame::kMaxDataSizeSamples,
833 dest_ptr_audio);
834
835 if (samples_per_channel < 0) {
Alex Loiko300ec8c2017-05-30 17:23:28 +0200836 LOG(LS_ERROR) << "Cannot add 10 ms audio, resampling failed";
kwibergc13ded52016-06-17 06:00:45 -0700837 return -1;
838 }
839 preprocess_frame_.samples_per_channel_ =
840 static_cast<size_t>(samples_per_channel);
841 preprocess_frame_.sample_rate_hz_ = encoder_stack_->SampleRateHz();
842 }
843
844 expected_codec_ts_ +=
845 static_cast<uint32_t>(preprocess_frame_.samples_per_channel_);
846 expected_in_ts_ += static_cast<uint32_t>(in_frame.samples_per_channel_);
847
848 return 0;
849}
850
851/////////////////////////////////////////
852// (RED) Redundant Coding
853//
854
855bool AudioCodingModuleImpl::REDStatus() const {
856 rtc::CritScope lock(&acm_crit_sect_);
857 return encoder_factory_->codec_manager.GetStackParams()->use_red;
858}
859
860// Configure RED status i.e on/off.
861int AudioCodingModuleImpl::SetREDStatus(bool enable_red) {
862#ifdef WEBRTC_CODEC_RED
863 rtc::CritScope lock(&acm_crit_sect_);
864 CreateSpeechEncoderIfNecessary(encoder_factory_.get());
865 if (!encoder_factory_->codec_manager.SetCopyRed(enable_red)) {
866 return -1;
867 }
868 auto* sp = encoder_factory_->codec_manager.GetStackParams();
869 if (sp->speech_encoder)
870 encoder_stack_ = encoder_factory_->rent_a_codec.RentEncoderStack(sp);
871 return 0;
872#else
Alex Loiko300ec8c2017-05-30 17:23:28 +0200873 LOG(LS_WARNING) << " WEBRTC_CODEC_RED is undefined";
kwibergc13ded52016-06-17 06:00:45 -0700874 return -1;
875#endif
876}
877
878/////////////////////////////////////////
879// (FEC) Forward Error Correction (codec internal)
880//
881
882bool AudioCodingModuleImpl::CodecFEC() const {
883 rtc::CritScope lock(&acm_crit_sect_);
884 return encoder_factory_->codec_manager.GetStackParams()->use_codec_fec;
885}
886
887int AudioCodingModuleImpl::SetCodecFEC(bool enable_codec_fec) {
888 rtc::CritScope lock(&acm_crit_sect_);
889 CreateSpeechEncoderIfNecessary(encoder_factory_.get());
890 if (!encoder_factory_->codec_manager.SetCodecFEC(enable_codec_fec)) {
891 return -1;
892 }
893 auto* sp = encoder_factory_->codec_manager.GetStackParams();
894 if (sp->speech_encoder)
895 encoder_stack_ = encoder_factory_->rent_a_codec.RentEncoderStack(sp);
896 if (enable_codec_fec) {
897 return sp->use_codec_fec ? 0 : -1;
898 } else {
899 RTC_DCHECK(!sp->use_codec_fec);
900 return 0;
901 }
902}
903
904int AudioCodingModuleImpl::SetPacketLossRate(int loss_rate) {
905 rtc::CritScope lock(&acm_crit_sect_);
906 if (HaveValidEncoder("SetPacketLossRate")) {
minyue4b9a2cb2016-11-30 06:49:59 -0800907 encoder_stack_->OnReceivedUplinkPacketLossFraction(loss_rate / 100.0);
kwibergc13ded52016-06-17 06:00:45 -0700908 }
909 return 0;
910}
911
912/////////////////////////////////////////
913// (VAD) Voice Activity Detection
914//
915int AudioCodingModuleImpl::SetVAD(bool enable_dtx,
916 bool enable_vad,
917 ACMVADMode mode) {
918 // Note: |enable_vad| is not used; VAD is enabled based on the DTX setting.
919 RTC_DCHECK_EQ(enable_dtx, enable_vad);
920 rtc::CritScope lock(&acm_crit_sect_);
921 CreateSpeechEncoderIfNecessary(encoder_factory_.get());
922 if (!encoder_factory_->codec_manager.SetVAD(enable_dtx, mode)) {
923 return -1;
924 }
925 auto* sp = encoder_factory_->codec_manager.GetStackParams();
926 if (sp->speech_encoder)
927 encoder_stack_ = encoder_factory_->rent_a_codec.RentEncoderStack(sp);
928 return 0;
929}
930
931// Get VAD/DTX settings.
932int AudioCodingModuleImpl::VAD(bool* dtx_enabled, bool* vad_enabled,
933 ACMVADMode* mode) const {
934 rtc::CritScope lock(&acm_crit_sect_);
935 const auto* sp = encoder_factory_->codec_manager.GetStackParams();
936 *dtx_enabled = *vad_enabled = sp->use_cng;
937 *mode = sp->vad_mode;
938 return 0;
939}
940
941/////////////////////////////////////////
942// Receiver
943//
944
945int AudioCodingModuleImpl::InitializeReceiver() {
946 rtc::CritScope lock(&acm_crit_sect_);
947 return InitializeReceiverSafe();
948}
949
950// Initialize receiver, resets codec database etc.
951int AudioCodingModuleImpl::InitializeReceiverSafe() {
952 // If the receiver is already initialized then we want to destroy any
953 // existing decoders. After a call to this function, we should have a clean
954 // start-up.
kwiberg6b19b562016-09-20 04:02:25 -0700955 if (receiver_initialized_)
956 receiver_.RemoveAllCodecs();
kwibergc13ded52016-06-17 06:00:45 -0700957 receiver_.ResetInitialDelay();
958 receiver_.SetMinimumDelay(0);
959 receiver_.SetMaximumDelay(0);
960 receiver_.FlushBuffers();
961
kwibergc13ded52016-06-17 06:00:45 -0700962 receiver_initialized_ = true;
963 return 0;
964}
965
966// Get current receive frequency.
967int AudioCodingModuleImpl::ReceiveFrequency() const {
968 const auto last_packet_sample_rate = receiver_.last_packet_sample_rate_hz();
969 return last_packet_sample_rate ? *last_packet_sample_rate
970 : receiver_.last_output_sample_rate_hz();
971}
972
973// Get current playout frequency.
974int AudioCodingModuleImpl::PlayoutFrequency() const {
kwibergc13ded52016-06-17 06:00:45 -0700975 return receiver_.last_output_sample_rate_hz();
976}
977
kwiberg1c07c702017-03-27 07:15:49 -0700978void AudioCodingModuleImpl::SetReceiveCodecs(
979 const std::map<int, SdpAudioFormat>& codecs) {
980 rtc::CritScope lock(&acm_crit_sect_);
981 receiver_.SetCodecs(codecs);
982}
983
kwiberg5adaf732016-10-04 09:33:27 -0700984bool AudioCodingModuleImpl::RegisterReceiveCodec(
985 int rtp_payload_type,
986 const SdpAudioFormat& audio_format) {
987 rtc::CritScope lock(&acm_crit_sect_);
988 RTC_DCHECK(receiver_initialized_);
989
990 if (!acm2::RentACodec::IsPayloadTypeValid(rtp_payload_type)) {
991 LOG_F(LS_ERROR) << "Invalid payload-type " << rtp_payload_type
992 << " for decoder.";
993 return false;
994 }
995
996 return receiver_.AddCodec(rtp_payload_type, audio_format);
997}
998
kwibergc13ded52016-06-17 06:00:45 -0700999int AudioCodingModuleImpl::RegisterReceiveCodec(const CodecInst& codec) {
1000 rtc::CritScope lock(&acm_crit_sect_);
1001 auto* ef = encoder_factory_.get();
1002 return RegisterReceiveCodecUnlocked(
1003 codec, [&] { return ef->rent_a_codec.RentIsacDecoder(codec.plfreq); });
1004}
1005
1006int AudioCodingModuleImpl::RegisterReceiveCodec(
1007 const CodecInst& codec,
kwiberg24c7c122016-09-28 11:57:10 -07001008 rtc::FunctionView<std::unique_ptr<AudioDecoder>()> isac_factory) {
kwibergc13ded52016-06-17 06:00:45 -07001009 rtc::CritScope lock(&acm_crit_sect_);
1010 return RegisterReceiveCodecUnlocked(codec, isac_factory);
1011}
1012
1013int AudioCodingModuleImpl::RegisterReceiveCodecUnlocked(
1014 const CodecInst& codec,
kwiberg24c7c122016-09-28 11:57:10 -07001015 rtc::FunctionView<std::unique_ptr<AudioDecoder>()> isac_factory) {
kwibergc13ded52016-06-17 06:00:45 -07001016 RTC_DCHECK(receiver_initialized_);
1017 if (codec.channels > 2) {
1018 LOG_F(LS_ERROR) << "Unsupported number of channels: " << codec.channels;
1019 return -1;
1020 }
1021
1022 auto codec_id = acm2::RentACodec::CodecIdByParams(codec.plname, codec.plfreq,
1023 codec.channels);
1024 if (!codec_id) {
1025 LOG_F(LS_ERROR) << "Wrong codec params to be registered as receive codec";
1026 return -1;
1027 }
1028 auto codec_index = acm2::RentACodec::CodecIndexFromId(*codec_id);
1029 RTC_CHECK(codec_index) << "Invalid codec ID: " << static_cast<int>(*codec_id);
1030
1031 // Check if the payload-type is valid.
1032 if (!acm2::RentACodec::IsPayloadTypeValid(codec.pltype)) {
1033 LOG_F(LS_ERROR) << "Invalid payload type " << codec.pltype << " for "
1034 << codec.plname;
1035 return -1;
1036 }
1037
1038 AudioDecoder* isac_decoder = nullptr;
1039 if (STR_CASE_CMP(codec.plname, "isac") == 0) {
1040 std::unique_ptr<AudioDecoder>& saved_isac_decoder =
1041 codec.plfreq == 16000 ? isac_decoder_16k_ : isac_decoder_32k_;
1042 if (!saved_isac_decoder) {
1043 saved_isac_decoder = isac_factory();
1044 }
1045 isac_decoder = saved_isac_decoder.get();
1046 }
1047 return receiver_.AddCodec(*codec_index, codec.pltype, codec.channels,
1048 codec.plfreq, isac_decoder, codec.plname);
1049}
1050
1051int AudioCodingModuleImpl::RegisterExternalReceiveCodec(
1052 int rtp_payload_type,
1053 AudioDecoder* external_decoder,
1054 int sample_rate_hz,
1055 int num_channels,
1056 const std::string& name) {
1057 rtc::CritScope lock(&acm_crit_sect_);
1058 RTC_DCHECK(receiver_initialized_);
1059 if (num_channels > 2 || num_channels < 0) {
1060 LOG_F(LS_ERROR) << "Unsupported number of channels: " << num_channels;
1061 return -1;
1062 }
1063
1064 // Check if the payload-type is valid.
1065 if (!acm2::RentACodec::IsPayloadTypeValid(rtp_payload_type)) {
1066 LOG_F(LS_ERROR) << "Invalid payload-type " << rtp_payload_type
1067 << " for external decoder.";
1068 return -1;
1069 }
1070
1071 return receiver_.AddCodec(-1 /* external */, rtp_payload_type, num_channels,
1072 sample_rate_hz, external_decoder, name);
1073}
1074
1075// Get current received codec.
1076int AudioCodingModuleImpl::ReceiveCodec(CodecInst* current_codec) const {
1077 rtc::CritScope lock(&acm_crit_sect_);
1078 return receiver_.LastAudioCodec(current_codec);
1079}
1080
ossue280cde2016-10-12 11:04:10 -07001081rtc::Optional<SdpAudioFormat> AudioCodingModuleImpl::ReceiveFormat() const {
1082 rtc::CritScope lock(&acm_crit_sect_);
1083 return receiver_.LastAudioFormat();
1084}
1085
kwibergc13ded52016-06-17 06:00:45 -07001086// Incoming packet from network parsed and ready for decode.
1087int AudioCodingModuleImpl::IncomingPacket(const uint8_t* incoming_payload,
1088 const size_t payload_length,
1089 const WebRtcRTPHeader& rtp_header) {
henrik.lundinb8c55b12017-05-10 07:38:01 -07001090 RTC_DCHECK_EQ(payload_length == 0, incoming_payload == nullptr);
kwibergc13ded52016-06-17 06:00:45 -07001091 return receiver_.InsertPacket(
1092 rtp_header,
1093 rtc::ArrayView<const uint8_t>(incoming_payload, payload_length));
1094}
1095
1096// Minimum playout delay (Used for lip-sync).
1097int AudioCodingModuleImpl::SetMinimumPlayoutDelay(int time_ms) {
1098 if ((time_ms < 0) || (time_ms > 10000)) {
Alex Loiko300ec8c2017-05-30 17:23:28 +02001099 LOG(LS_ERROR) << "Delay must be in the range of 0-1000 milliseconds.";
kwibergc13ded52016-06-17 06:00:45 -07001100 return -1;
1101 }
1102 return receiver_.SetMinimumDelay(time_ms);
1103}
1104
1105int AudioCodingModuleImpl::SetMaximumPlayoutDelay(int time_ms) {
1106 if ((time_ms < 0) || (time_ms > 10000)) {
Alex Loiko300ec8c2017-05-30 17:23:28 +02001107 LOG(LS_ERROR) << "Delay must be in the range of 0-1000 milliseconds.";
kwibergc13ded52016-06-17 06:00:45 -07001108 return -1;
1109 }
1110 return receiver_.SetMaximumDelay(time_ms);
1111}
1112
1113// Get 10 milliseconds of raw audio data to play out.
1114// Automatic resample to the requested frequency.
1115int AudioCodingModuleImpl::PlayoutData10Ms(int desired_freq_hz,
1116 AudioFrame* audio_frame,
1117 bool* muted) {
1118 // GetAudio always returns 10 ms, at the requested sample rate.
1119 if (receiver_.GetAudio(desired_freq_hz, audio_frame, muted) != 0) {
Alex Loiko300ec8c2017-05-30 17:23:28 +02001120 LOG(LS_ERROR) << "PlayoutData failed, RecOut Failed";
kwibergc13ded52016-06-17 06:00:45 -07001121 return -1;
1122 }
1123 audio_frame->id_ = id_;
1124 return 0;
1125}
1126
1127int AudioCodingModuleImpl::PlayoutData10Ms(int desired_freq_hz,
1128 AudioFrame* audio_frame) {
1129 bool muted;
1130 int ret = PlayoutData10Ms(desired_freq_hz, audio_frame, &muted);
1131 RTC_DCHECK(!muted);
1132 return ret;
1133}
1134
1135/////////////////////////////////////////
1136// Statistics
1137//
1138
1139// TODO(turajs) change the return value to void. Also change the corresponding
1140// NetEq function.
1141int AudioCodingModuleImpl::GetNetworkStatistics(NetworkStatistics* statistics) {
1142 receiver_.GetNetworkStatistics(statistics);
1143 return 0;
1144}
1145
1146int AudioCodingModuleImpl::RegisterVADCallback(ACMVADCallback* vad_callback) {
Alex Loiko300ec8c2017-05-30 17:23:28 +02001147 LOG(LS_VERBOSE) << "RegisterVADCallback()";
kwibergc13ded52016-06-17 06:00:45 -07001148 rtc::CritScope lock(&callback_crit_sect_);
1149 vad_callback_ = vad_callback;
1150 return 0;
1151}
1152
1153// TODO(kwiberg): Remove this method, and have callers call IncomingPacket
1154// instead. The translation logic and state belong with them, not with
1155// AudioCodingModuleImpl.
1156int AudioCodingModuleImpl::IncomingPayload(const uint8_t* incoming_payload,
1157 size_t payload_length,
1158 uint8_t payload_type,
1159 uint32_t timestamp) {
1160 // We are not acquiring any lock when interacting with |aux_rtp_header_| no
1161 // other method uses this member variable.
1162 if (!aux_rtp_header_) {
1163 // This is the first time that we are using |dummy_rtp_header_|
1164 // so we have to create it.
1165 aux_rtp_header_.reset(new WebRtcRTPHeader);
1166 aux_rtp_header_->header.payloadType = payload_type;
1167 // Don't matter in this case.
1168 aux_rtp_header_->header.ssrc = 0;
1169 aux_rtp_header_->header.markerBit = false;
1170 // Start with random numbers.
1171 aux_rtp_header_->header.sequenceNumber = 0x1234; // Arbitrary.
1172 aux_rtp_header_->type.Audio.channel = 1;
1173 }
1174
1175 aux_rtp_header_->header.timestamp = timestamp;
1176 IncomingPacket(incoming_payload, payload_length, *aux_rtp_header_);
1177 // Get ready for the next payload.
1178 aux_rtp_header_->header.sequenceNumber++;
1179 return 0;
1180}
1181
1182int AudioCodingModuleImpl::SetOpusApplication(OpusApplicationMode application) {
1183 rtc::CritScope lock(&acm_crit_sect_);
1184 if (!HaveValidEncoder("SetOpusApplication")) {
1185 return -1;
1186 }
1187 AudioEncoder::Application app;
1188 switch (application) {
1189 case kVoip:
1190 app = AudioEncoder::Application::kSpeech;
1191 break;
1192 case kAudio:
1193 app = AudioEncoder::Application::kAudio;
1194 break;
1195 default:
1196 FATAL();
1197 return 0;
1198 }
1199 return encoder_stack_->SetApplication(app) ? 0 : -1;
1200}
1201
1202// Informs Opus encoder of the maximum playback rate the receiver will render.
1203int AudioCodingModuleImpl::SetOpusMaxPlaybackRate(int frequency_hz) {
1204 rtc::CritScope lock(&acm_crit_sect_);
1205 if (!HaveValidEncoder("SetOpusMaxPlaybackRate")) {
1206 return -1;
1207 }
1208 encoder_stack_->SetMaxPlaybackRate(frequency_hz);
1209 return 0;
1210}
1211
1212int AudioCodingModuleImpl::EnableOpusDtx() {
1213 rtc::CritScope lock(&acm_crit_sect_);
1214 if (!HaveValidEncoder("EnableOpusDtx")) {
1215 return -1;
1216 }
1217 return encoder_stack_->SetDtx(true) ? 0 : -1;
1218}
1219
1220int AudioCodingModuleImpl::DisableOpusDtx() {
1221 rtc::CritScope lock(&acm_crit_sect_);
1222 if (!HaveValidEncoder("DisableOpusDtx")) {
1223 return -1;
1224 }
1225 return encoder_stack_->SetDtx(false) ? 0 : -1;
1226}
1227
1228int32_t AudioCodingModuleImpl::PlayoutTimestamp(uint32_t* timestamp) {
1229 rtc::Optional<uint32_t> ts = PlayoutTimestamp();
1230 if (!ts)
1231 return -1;
1232 *timestamp = *ts;
1233 return 0;
1234}
1235
1236rtc::Optional<uint32_t> AudioCodingModuleImpl::PlayoutTimestamp() {
1237 return receiver_.GetPlayoutTimestamp();
1238}
1239
henrik.lundinb3f1c5d2016-08-22 15:39:53 -07001240int AudioCodingModuleImpl::FilteredCurrentDelayMs() const {
1241 return receiver_.FilteredCurrentDelayMs();
1242}
1243
kwibergc13ded52016-06-17 06:00:45 -07001244bool AudioCodingModuleImpl::HaveValidEncoder(const char* caller_name) const {
1245 if (!encoder_stack_) {
Alex Loiko300ec8c2017-05-30 17:23:28 +02001246 LOG(LS_ERROR) << caller_name << " failed: No send codec is registered.";
kwibergc13ded52016-06-17 06:00:45 -07001247 return false;
1248 }
1249 return true;
1250}
1251
1252int AudioCodingModuleImpl::UnregisterReceiveCodec(uint8_t payload_type) {
1253 return receiver_.RemoveCodec(payload_type);
1254}
1255
1256int AudioCodingModuleImpl::EnableNack(size_t max_nack_list_size) {
1257 return receiver_.EnableNack(max_nack_list_size);
1258}
1259
1260void AudioCodingModuleImpl::DisableNack() {
1261 receiver_.DisableNack();
1262}
1263
1264std::vector<uint16_t> AudioCodingModuleImpl::GetNackList(
1265 int64_t round_trip_time_ms) const {
1266 return receiver_.GetNackList(round_trip_time_ms);
1267}
1268
1269int AudioCodingModuleImpl::LeastRequiredDelayMs() const {
1270 return receiver_.LeastRequiredDelayMs();
1271}
1272
1273void AudioCodingModuleImpl::GetDecodingCallStatistics(
1274 AudioDecodingCallStats* call_stats) const {
1275 receiver_.GetDecodingCallStatistics(call_stats);
1276}
1277
ivoce1198e02017-09-08 08:13:19 -07001278ANAStats AudioCodingModuleImpl::GetANAStats() const {
1279 rtc::CritScope lock(&acm_crit_sect_);
1280 if (encoder_stack_)
1281 return encoder_stack_->GetANAStats();
1282 // If no encoder is set, return default stats.
1283 return ANAStats();
1284}
1285
kwibergc13ded52016-06-17 06:00:45 -07001286} // namespace
1287
kwiberg36a43882016-08-29 05:33:32 -07001288AudioCodingModule::Config::Config()
1289 : id(0), neteq_config(), clock(Clock::GetRealTimeClock()) {
1290 // Post-decode VAD is disabled by default in NetEq, however, Audio
1291 // Conference Mixer relies on VAD decisions and fails without them.
1292 neteq_config.enable_post_decode_vad = true;
1293}
1294
1295AudioCodingModule::Config::Config(const Config&) = default;
1296AudioCodingModule::Config::~Config() = default;
1297
turaj@webrtc.org7959e162013-09-12 18:30:26 +00001298// Create module
1299AudioCodingModule* AudioCodingModule::Create(int id) {
Henrik Lundin64dad832015-05-11 12:44:23 +02001300 Config config;
1301 config.id = id;
1302 config.clock = Clock::GetRealTimeClock();
ossue3525782016-05-25 07:37:43 -07001303 config.decoder_factory = CreateBuiltinAudioDecoderFactory();
Henrik Lundin64dad832015-05-11 12:44:23 +02001304 return Create(config);
turaj@webrtc.org532f3dc2013-09-19 00:12:23 +00001305}
1306
1307AudioCodingModule* AudioCodingModule::Create(int id, Clock* clock) {
Henrik Lundin64dad832015-05-11 12:44:23 +02001308 Config config;
henrik.lundin@webrtc.orge772c712014-04-28 10:16:57 +00001309 config.id = id;
henrik.lundin@webrtc.org0bc9b5a2014-04-29 08:09:31 +00001310 config.clock = clock;
ossue3525782016-05-25 07:37:43 -07001311 config.decoder_factory = CreateBuiltinAudioDecoderFactory();
Henrik Lundin64dad832015-05-11 12:44:23 +02001312 return Create(config);
1313}
1314
1315AudioCodingModule* AudioCodingModule::Create(const Config& config) {
ossue3525782016-05-25 07:37:43 -07001316 if (!config.decoder_factory) {
1317 // TODO(ossu): Backwards compatibility. Will be removed after a deprecation
1318 // cycle.
1319 Config config_copy = config;
1320 config_copy.decoder_factory = CreateBuiltinAudioDecoderFactory();
kwibergc13ded52016-06-17 06:00:45 -07001321 return new AudioCodingModuleImpl(config_copy);
ossue3525782016-05-25 07:37:43 -07001322 }
kwibergc13ded52016-06-17 06:00:45 -07001323 return new AudioCodingModuleImpl(config);
turaj@webrtc.org7959e162013-09-12 18:30:26 +00001324}
1325
turaj@webrtc.org7959e162013-09-12 18:30:26 +00001326int AudioCodingModule::NumberOfCodecs() {
kwibergfce4a942015-10-27 11:40:24 -07001327 return static_cast<int>(acm2::RentACodec::NumberOfCodecs());
turaj@webrtc.org7959e162013-09-12 18:30:26 +00001328}
1329
turaj@webrtc.org7959e162013-09-12 18:30:26 +00001330int AudioCodingModule::Codec(int list_id, CodecInst* codec) {
kwibergfce4a942015-10-27 11:40:24 -07001331 auto codec_id = acm2::RentACodec::CodecIdFromIndex(list_id);
1332 if (!codec_id)
1333 return -1;
1334 auto ci = acm2::RentACodec::CodecInstById(*codec_id);
1335 if (!ci)
1336 return -1;
1337 *codec = *ci;
1338 return 0;
turaj@webrtc.org7959e162013-09-12 18:30:26 +00001339}
1340
turaj@webrtc.org7959e162013-09-12 18:30:26 +00001341int AudioCodingModule::Codec(const char* payload_name,
1342 CodecInst* codec,
1343 int sampling_freq_hz,
Peter Kasting69558702016-01-12 16:26:35 -08001344 size_t channels) {
Karl Wibergbe579832015-11-10 22:34:18 +01001345 rtc::Optional<CodecInst> ci = acm2::RentACodec::CodecInstByParams(
turaj@webrtc.org6d5d2482013-10-06 04:47:28 +00001346 payload_name, sampling_freq_hz, channels);
kwibergfce4a942015-10-27 11:40:24 -07001347 if (ci) {
1348 *codec = *ci;
1349 return 0;
1350 } else {
1351 // We couldn't find a matching codec, so set the parameters to unacceptable
turaj@webrtc.org7959e162013-09-12 18:30:26 +00001352 // values and return.
1353 codec->plname[0] = '\0';
1354 codec->pltype = -1;
1355 codec->pacsize = 0;
1356 codec->rate = 0;
1357 codec->plfreq = 0;
1358 return -1;
1359 }
turaj@webrtc.org7959e162013-09-12 18:30:26 +00001360}
1361
turaj@webrtc.org7959e162013-09-12 18:30:26 +00001362int AudioCodingModule::Codec(const char* payload_name,
1363 int sampling_freq_hz,
Peter Kasting69558702016-01-12 16:26:35 -08001364 size_t channels) {
Karl Wibergbe579832015-11-10 22:34:18 +01001365 rtc::Optional<acm2::RentACodec::CodecId> ci =
1366 acm2::RentACodec::CodecIdByParams(payload_name, sampling_freq_hz,
1367 channels);
kwibergfce4a942015-10-27 11:40:24 -07001368 if (!ci)
1369 return -1;
Karl Wibergbe579832015-11-10 22:34:18 +01001370 rtc::Optional<int> i = acm2::RentACodec::CodecIndexFromId(*ci);
kwibergfce4a942015-10-27 11:40:24 -07001371 return i ? *i : -1;
turaj@webrtc.org7959e162013-09-12 18:30:26 +00001372}
1373
1374// Checks the validity of the parameters of the given codec
1375bool AudioCodingModule::IsCodecValid(const CodecInst& codec) {
kwibergfce4a942015-10-27 11:40:24 -07001376 bool valid = acm2::RentACodec::IsCodecValid(codec);
1377 if (!valid)
Alex Loiko300ec8c2017-05-30 17:23:28 +02001378 LOG(LS_ERROR) << "Invalid codec setting";
kwibergfce4a942015-10-27 11:40:24 -07001379 return valid;
turaj@webrtc.org7959e162013-09-12 18:30:26 +00001380}
1381
turaj@webrtc.org7959e162013-09-12 18:30:26 +00001382} // namespace webrtc