blob: 6c0e31edc2693647edf472200be556b82095ab8b [file] [log] [blame]
turaj@webrtc.org7959e162013-09-12 18:30:26 +00001/*
2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020011#include "modules/audio_coding/include/audio_coding_module.h"
turaj@webrtc.org7959e162013-09-12 18:30:26 +000012
Yves Gerey988cc082018-10-23 12:03:01 +020013#include <assert.h>
Jonas Olssona4d87372019-07-05 19:08:33 +020014
Jonathan Yu36344a02017-07-30 01:55:34 -070015#include <algorithm>
Yves Gerey988cc082018-10-23 12:03:01 +020016#include <cstdint>
Jonathan Yu36344a02017-07-30 01:55:34 -070017
Niels Möller2edab4c2018-10-22 09:48:08 +020018#include "absl/strings/match.h"
Yves Gerey988cc082018-10-23 12:03:01 +020019#include "api/array_view.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020020#include "modules/audio_coding/acm2/acm_receiver.h"
21#include "modules/audio_coding/acm2/acm_resampler.h"
Fredrik Solenbergbbf21a32018-04-12 22:44:09 +020022#include "modules/include/module_common_types.h"
Yves Gerey988cc082018-10-23 12:03:01 +020023#include "modules/include/module_common_types_public.h"
24#include "rtc_base/buffer.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020025#include "rtc_base/checks.h"
Steve Anton10542f22019-01-11 09:11:00 -080026#include "rtc_base/critical_section.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020027#include "rtc_base/logging.h"
Karl Wiberge40468b2017-11-22 10:42:26 +010028#include "rtc_base/numerics/safe_conversions.h"
Yves Gerey988cc082018-10-23 12:03:01 +020029#include "rtc_base/thread_annotations.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020030#include "system_wrappers/include/metrics.h"
turaj@webrtc.org7959e162013-09-12 18:30:26 +000031
32namespace webrtc {
33
kwibergc13ded52016-06-17 06:00:45 -070034namespace {
35
kwibergc13ded52016-06-17 06:00:45 -070036class AudioCodingModuleImpl final : public AudioCodingModule {
37 public:
38 explicit AudioCodingModuleImpl(const AudioCodingModule::Config& config);
39 ~AudioCodingModuleImpl() override;
40
41 /////////////////////////////////////////
42 // Sender
43 //
44
kwiberg24c7c122016-09-28 11:57:10 -070045 void ModifyEncoder(rtc::FunctionView<void(std::unique_ptr<AudioEncoder>*)>
46 modifier) override;
kwibergc13ded52016-06-17 06:00:45 -070047
kwibergc13ded52016-06-17 06:00:45 -070048 // Register a transport callback which will be
49 // called to deliver the encoded buffers.
50 int RegisterTransportCallback(AudioPacketizationCallback* transport) override;
51
52 // Add 10 ms of raw (PCM) audio data to the encoder.
53 int Add10MsData(const AudioFrame& audio_frame) override;
54
55 /////////////////////////////////////////
kwibergc13ded52016-06-17 06:00:45 -070056 // (FEC) Forward Error Correction (codec internal)
57 //
58
kwibergc13ded52016-06-17 06:00:45 -070059 // Set target packet loss rate
60 int SetPacketLossRate(int loss_rate) override;
61
62 /////////////////////////////////////////
63 // (VAD) Voice Activity Detection
64 // and
65 // (CNG) Comfort Noise Generation
66 //
67
kwibergc13ded52016-06-17 06:00:45 -070068 int RegisterVADCallback(ACMVADCallback* vad_callback) override;
69
70 /////////////////////////////////////////
71 // Receiver
72 //
73
74 // Initialize receiver, resets codec database etc.
75 int InitializeReceiver() override;
76
77 // Get current receive frequency.
78 int ReceiveFrequency() const override;
79
80 // Get current playout frequency.
81 int PlayoutFrequency() const override;
82
kwiberg1c07c702017-03-27 07:15:49 -070083 void SetReceiveCodecs(const std::map<int, SdpAudioFormat>& codecs) override;
84
kwibergc13ded52016-06-17 06:00:45 -070085 // Get current received codec.
Fredrik Solenbergf693bfa2018-12-11 12:22:10 +010086 absl::optional<std::pair<int, SdpAudioFormat>> ReceiveCodec() const override;
ossue280cde2016-10-12 11:04:10 -070087
kwibergc13ded52016-06-17 06:00:45 -070088 // Incoming packet from network parsed and ready for decode.
89 int IncomingPacket(const uint8_t* incoming_payload,
90 const size_t payload_length,
Niels Möllerafb5dbb2019-02-15 15:21:47 +010091 const RTPHeader& rtp_info) override;
kwibergc13ded52016-06-17 06:00:45 -070092
kwibergc13ded52016-06-17 06:00:45 -070093 // Minimum playout delay.
94 int SetMinimumPlayoutDelay(int time_ms) override;
95
96 // Maximum playout delay.
97 int SetMaximumPlayoutDelay(int time_ms) override;
98
Ruslan Burakov3b50f9f2019-02-06 09:45:56 +010099 bool SetBaseMinimumPlayoutDelayMs(int delay_ms) override;
100
101 int GetBaseMinimumPlayoutDelayMs() const override;
102
Danil Chapovalovb6021232018-06-19 13:26:36 +0200103 absl::optional<uint32_t> PlayoutTimestamp() override;
kwibergc13ded52016-06-17 06:00:45 -0700104
henrik.lundinb3f1c5d2016-08-22 15:39:53 -0700105 int FilteredCurrentDelayMs() const override;
106
Henrik Lundinabbff892017-11-29 09:14:04 +0100107 int TargetDelayMs() const override;
108
kwibergc13ded52016-06-17 06:00:45 -0700109 // Get 10 milliseconds of raw audio data to play out, and
110 // automatic resample to the requested frequency if > 0.
111 int PlayoutData10Ms(int desired_freq_hz,
112 AudioFrame* audio_frame,
113 bool* muted) override;
kwibergc13ded52016-06-17 06:00:45 -0700114
115 /////////////////////////////////////////
116 // Statistics
117 //
118
119 int GetNetworkStatistics(NetworkStatistics* statistics) override;
120
kwibergc13ded52016-06-17 06:00:45 -0700121 int EnableNack(size_t max_nack_list_size) override;
122
123 void DisableNack() override;
124
125 std::vector<uint16_t> GetNackList(int64_t round_trip_time_ms) const override;
126
127 void GetDecodingCallStatistics(AudioDecodingCallStats* stats) const override;
128
ivoce1198e02017-09-08 08:13:19 -0700129 ANAStats GetANAStats() const override;
130
kwibergc13ded52016-06-17 06:00:45 -0700131 private:
132 struct InputData {
133 uint32_t input_timestamp;
134 const int16_t* audio;
135 size_t length_per_channel;
136 size_t audio_channel;
137 // If a re-mix is required (up or down), this buffer will store a re-mixed
138 // version of the input.
139 int16_t buffer[WEBRTC_10MS_PCM_AUDIO];
140 };
141
142 // This member class writes values to the named UMA histogram, but only if
143 // the value has changed since the last time (and always for the first call).
144 class ChangeLogger {
145 public:
146 explicit ChangeLogger(const std::string& histogram_name)
147 : histogram_name_(histogram_name) {}
148 // Logs the new value if it is different from the last logged value, or if
149 // this is the first call.
150 void MaybeLog(int value);
151
152 private:
153 int last_value_ = 0;
154 int first_time_ = true;
155 const std::string histogram_name_;
156 };
157
kwibergc13ded52016-06-17 06:00:45 -0700158 int Add10MsDataInternal(const AudioFrame& audio_frame, InputData* input_data)
danilchap56359be2017-09-07 07:53:45 -0700159 RTC_EXCLUSIVE_LOCKS_REQUIRED(acm_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700160 int Encode(const InputData& input_data)
danilchap56359be2017-09-07 07:53:45 -0700161 RTC_EXCLUSIVE_LOCKS_REQUIRED(acm_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700162
danilchap56359be2017-09-07 07:53:45 -0700163 int InitializeReceiverSafe() RTC_EXCLUSIVE_LOCKS_REQUIRED(acm_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700164
165 bool HaveValidEncoder(const char* caller_name) const
danilchap56359be2017-09-07 07:53:45 -0700166 RTC_EXCLUSIVE_LOCKS_REQUIRED(acm_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700167
168 // Preprocessing of input audio, including resampling and down-mixing if
169 // required, before pushing audio into encoder's buffer.
170 //
171 // in_frame: input audio-frame
172 // ptr_out: pointer to output audio_frame. If no preprocessing is required
173 // |ptr_out| will be pointing to |in_frame|, otherwise pointing to
174 // |preprocess_frame_|.
175 //
176 // Return value:
177 // -1: if encountering an error.
178 // 0: otherwise.
179 int PreprocessToAddData(const AudioFrame& in_frame,
180 const AudioFrame** ptr_out)
danilchap56359be2017-09-07 07:53:45 -0700181 RTC_EXCLUSIVE_LOCKS_REQUIRED(acm_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700182
183 // Change required states after starting to receive the codec corresponding
184 // to |index|.
185 int UpdateUponReceivingCodec(int index);
186
187 rtc::CriticalSection acm_crit_sect_;
danilchap56359be2017-09-07 07:53:45 -0700188 rtc::Buffer encode_buffer_ RTC_GUARDED_BY(acm_crit_sect_);
danilchap56359be2017-09-07 07:53:45 -0700189 uint32_t expected_codec_ts_ RTC_GUARDED_BY(acm_crit_sect_);
190 uint32_t expected_in_ts_ RTC_GUARDED_BY(acm_crit_sect_);
191 acm2::ACMResampler resampler_ RTC_GUARDED_BY(acm_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700192 acm2::AcmReceiver receiver_; // AcmReceiver has it's own internal lock.
danilchap56359be2017-09-07 07:53:45 -0700193 ChangeLogger bitrate_logger_ RTC_GUARDED_BY(acm_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700194
Karl Wiberg49c33ce2018-11-12 14:21:58 +0100195 // Current encoder stack, provided by a call to RegisterEncoder.
danilchap56359be2017-09-07 07:53:45 -0700196 std::unique_ptr<AudioEncoder> encoder_stack_ RTC_GUARDED_BY(acm_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700197
kwibergc13ded52016-06-17 06:00:45 -0700198 // This is to keep track of CN instances where we can send DTMFs.
danilchap56359be2017-09-07 07:53:45 -0700199 uint8_t previous_pltype_ RTC_GUARDED_BY(acm_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700200
danilchap56359be2017-09-07 07:53:45 -0700201 bool receiver_initialized_ RTC_GUARDED_BY(acm_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700202
danilchap56359be2017-09-07 07:53:45 -0700203 AudioFrame preprocess_frame_ RTC_GUARDED_BY(acm_crit_sect_);
204 bool first_10ms_data_ RTC_GUARDED_BY(acm_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700205
danilchap56359be2017-09-07 07:53:45 -0700206 bool first_frame_ RTC_GUARDED_BY(acm_crit_sect_);
207 uint32_t last_timestamp_ RTC_GUARDED_BY(acm_crit_sect_);
208 uint32_t last_rtp_timestamp_ RTC_GUARDED_BY(acm_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700209
210 rtc::CriticalSection callback_crit_sect_;
211 AudioPacketizationCallback* packetization_callback_
danilchap56359be2017-09-07 07:53:45 -0700212 RTC_GUARDED_BY(callback_crit_sect_);
213 ACMVADCallback* vad_callback_ RTC_GUARDED_BY(callback_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700214
215 int codec_histogram_bins_log_[static_cast<size_t>(
216 AudioEncoder::CodecType::kMaxLoggedAudioCodecTypes)];
217 int number_of_consecutive_empty_packets_;
218};
219
220// Adds a codec usage sample to the histogram.
221void UpdateCodecTypeHistogram(size_t codec_type) {
222 RTC_HISTOGRAM_ENUMERATION(
223 "WebRTC.Audio.Encoder.CodecType", static_cast<int>(codec_type),
224 static_cast<int>(
225 webrtc::AudioEncoder::CodecType::kMaxLoggedAudioCodecTypes));
226}
227
kwibergc13ded52016-06-17 06:00:45 -0700228// Stereo-to-mono can be used as in-place.
229int DownMix(const AudioFrame& frame,
230 size_t length_out_buff,
231 int16_t* out_buff) {
yujo36b1a5f2017-06-12 12:45:32 -0700232 RTC_DCHECK_EQ(frame.num_channels_, 2);
233 RTC_DCHECK_GE(length_out_buff, frame.samples_per_channel_);
234
235 if (!frame.muted()) {
236 const int16_t* frame_data = frame.data();
237 for (size_t n = 0; n < frame.samples_per_channel_; ++n) {
Yves Gerey665174f2018-06-19 15:03:05 +0200238 out_buff[n] =
239 static_cast<int16_t>((static_cast<int32_t>(frame_data[2 * n]) +
240 static_cast<int32_t>(frame_data[2 * n + 1])) >>
241 1);
yujo36b1a5f2017-06-12 12:45:32 -0700242 }
243 } else {
Jonathan Yu36344a02017-07-30 01:55:34 -0700244 std::fill(out_buff, out_buff + frame.samples_per_channel_, 0);
kwibergc13ded52016-06-17 06:00:45 -0700245 }
kwibergc13ded52016-06-17 06:00:45 -0700246 return 0;
247}
248
249// Mono-to-stereo can be used as in-place.
250int UpMix(const AudioFrame& frame, size_t length_out_buff, int16_t* out_buff) {
yujo36b1a5f2017-06-12 12:45:32 -0700251 RTC_DCHECK_EQ(frame.num_channels_, 1);
252 RTC_DCHECK_GE(length_out_buff, 2 * frame.samples_per_channel_);
253
254 if (!frame.muted()) {
255 const int16_t* frame_data = frame.data();
256 for (size_t n = frame.samples_per_channel_; n != 0; --n) {
257 size_t i = n - 1;
258 int16_t sample = frame_data[i];
259 out_buff[2 * i + 1] = sample;
260 out_buff[2 * i] = sample;
261 }
262 } else {
Jonathan Yu36344a02017-07-30 01:55:34 -0700263 std::fill(out_buff, out_buff + frame.samples_per_channel_ * 2, 0);
kwibergc13ded52016-06-17 06:00:45 -0700264 }
265 return 0;
266}
267
kwibergc13ded52016-06-17 06:00:45 -0700268void AudioCodingModuleImpl::ChangeLogger::MaybeLog(int value) {
269 if (value != last_value_ || first_time_) {
270 first_time_ = false;
271 last_value_ = value;
272 RTC_HISTOGRAM_COUNTS_SPARSE_100(histogram_name_, value);
273 }
274}
275
276AudioCodingModuleImpl::AudioCodingModuleImpl(
277 const AudioCodingModule::Config& config)
solenbergc7b4a452017-09-28 07:37:11 -0700278 : expected_codec_ts_(0xD87F3F9F),
kwibergc13ded52016-06-17 06:00:45 -0700279 expected_in_ts_(0xD87F3F9F),
280 receiver_(config),
281 bitrate_logger_("WebRTC.Audio.TargetBitrateInKbps"),
kwibergc13ded52016-06-17 06:00:45 -0700282 encoder_stack_(nullptr),
283 previous_pltype_(255),
284 receiver_initialized_(false),
285 first_10ms_data_(false),
286 first_frame_(true),
287 packetization_callback_(NULL),
288 vad_callback_(NULL),
289 codec_histogram_bins_log_(),
290 number_of_consecutive_empty_packets_(0) {
291 if (InitializeReceiverSafe() < 0) {
Mirko Bonadei675513b2017-11-09 11:09:25 +0100292 RTC_LOG(LS_ERROR) << "Cannot initialize receiver";
kwibergc13ded52016-06-17 06:00:45 -0700293 }
Mirko Bonadei675513b2017-11-09 11:09:25 +0100294 RTC_LOG(LS_INFO) << "Created";
kwibergc13ded52016-06-17 06:00:45 -0700295}
296
297AudioCodingModuleImpl::~AudioCodingModuleImpl() = default;
298
299int32_t AudioCodingModuleImpl::Encode(const InputData& input_data) {
300 AudioEncoder::EncodedInfo encoded_info;
301 uint8_t previous_pltype;
302
303 // Check if there is an encoder before.
304 if (!HaveValidEncoder("Process"))
305 return -1;
306
Yves Gerey665174f2018-06-19 15:03:05 +0200307 if (!first_frame_) {
deadbeeffcada902016-08-24 12:45:13 -0700308 RTC_DCHECK(IsNewerTimestamp(input_data.input_timestamp, last_timestamp_))
ossu63fb95a2016-07-06 09:34:22 -0700309 << "Time should not move backwards";
310 }
311
kwibergc13ded52016-06-17 06:00:45 -0700312 // Scale the timestamp to the codec's RTP timestamp rate.
313 uint32_t rtp_timestamp =
Karl Wiberg053c3712019-05-16 15:24:17 +0200314 first_frame_
315 ? input_data.input_timestamp
316 : last_rtp_timestamp_ +
317 rtc::dchecked_cast<uint32_t>(rtc::CheckedDivExact(
318 int64_t{input_data.input_timestamp - last_timestamp_} *
319 encoder_stack_->RtpTimestampRateHz(),
320 int64_t{encoder_stack_->SampleRateHz()}));
kwibergc13ded52016-06-17 06:00:45 -0700321 last_timestamp_ = input_data.input_timestamp;
322 last_rtp_timestamp_ = rtp_timestamp;
323 first_frame_ = false;
324
325 // Clear the buffer before reuse - encoded data will get appended.
326 encode_buffer_.Clear();
327 encoded_info = encoder_stack_->Encode(
Yves Gerey665174f2018-06-19 15:03:05 +0200328 rtp_timestamp,
329 rtc::ArrayView<const int16_t>(
330 input_data.audio,
331 input_data.audio_channel * input_data.length_per_channel),
kwibergc13ded52016-06-17 06:00:45 -0700332 &encode_buffer_);
333
334 bitrate_logger_.MaybeLog(encoder_stack_->GetTargetBitrate() / 1000);
335 if (encode_buffer_.size() == 0 && !encoded_info.send_even_if_empty) {
336 // Not enough data.
337 return 0;
338 }
339 previous_pltype = previous_pltype_; // Read it while we have the critsect.
340
341 // Log codec type to histogram once every 500 packets.
342 if (encoded_info.encoded_bytes == 0) {
343 ++number_of_consecutive_empty_packets_;
344 } else {
345 size_t codec_type = static_cast<size_t>(encoded_info.encoder_type);
346 codec_histogram_bins_log_[codec_type] +=
347 number_of_consecutive_empty_packets_ + 1;
348 number_of_consecutive_empty_packets_ = 0;
349 if (codec_histogram_bins_log_[codec_type] >= 500) {
350 codec_histogram_bins_log_[codec_type] -= 500;
351 UpdateCodecTypeHistogram(codec_type);
352 }
353 }
354
Niels Möller87e2d782019-03-07 10:18:23 +0100355 AudioFrameType frame_type;
kwibergc13ded52016-06-17 06:00:45 -0700356 if (encode_buffer_.size() == 0 && encoded_info.send_even_if_empty) {
Niels Möllerc936cb62019-03-19 14:10:16 +0100357 frame_type = AudioFrameType::kEmptyFrame;
kwibergc13ded52016-06-17 06:00:45 -0700358 encoded_info.payload_type = previous_pltype;
359 } else {
kwibergaf476c72016-11-28 15:21:39 -0800360 RTC_DCHECK_GT(encode_buffer_.size(), 0);
Niels Möllerc936cb62019-03-19 14:10:16 +0100361 frame_type = encoded_info.speech ? AudioFrameType::kAudioFrameSpeech
362 : AudioFrameType::kAudioFrameCN;
kwibergc13ded52016-06-17 06:00:45 -0700363 }
364
365 {
366 rtc::CritScope lock(&callback_crit_sect_);
367 if (packetization_callback_) {
368 packetization_callback_->SendData(
369 frame_type, encoded_info.payload_type, encoded_info.encoded_timestamp,
Niels Möllerc35b6e62019-04-25 16:31:18 +0200370 encode_buffer_.data(), encode_buffer_.size());
kwibergc13ded52016-06-17 06:00:45 -0700371 }
372
373 if (vad_callback_) {
374 // Callback with VAD decision.
375 vad_callback_->InFrameType(frame_type);
376 }
377 }
378 previous_pltype_ = encoded_info.payload_type;
379 return static_cast<int32_t>(encode_buffer_.size());
380}
381
382/////////////////////////////////////////
383// Sender
384//
385
kwibergc13ded52016-06-17 06:00:45 -0700386void AudioCodingModuleImpl::ModifyEncoder(
kwiberg24c7c122016-09-28 11:57:10 -0700387 rtc::FunctionView<void(std::unique_ptr<AudioEncoder>*)> modifier) {
kwibergc13ded52016-06-17 06:00:45 -0700388 rtc::CritScope lock(&acm_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700389 modifier(&encoder_stack_);
390}
391
kwibergc13ded52016-06-17 06:00:45 -0700392// Register a transport callback which will be called to deliver
393// the encoded buffers.
394int AudioCodingModuleImpl::RegisterTransportCallback(
395 AudioPacketizationCallback* transport) {
396 rtc::CritScope lock(&callback_crit_sect_);
397 packetization_callback_ = transport;
398 return 0;
399}
400
401// Add 10MS of raw (PCM) audio data to the encoder.
402int AudioCodingModuleImpl::Add10MsData(const AudioFrame& audio_frame) {
403 InputData input_data;
404 rtc::CritScope lock(&acm_crit_sect_);
405 int r = Add10MsDataInternal(audio_frame, &input_data);
406 return r < 0 ? r : Encode(input_data);
407}
408
409int AudioCodingModuleImpl::Add10MsDataInternal(const AudioFrame& audio_frame,
410 InputData* input_data) {
411 if (audio_frame.samples_per_channel_ == 0) {
412 assert(false);
Mirko Bonadei675513b2017-11-09 11:09:25 +0100413 RTC_LOG(LS_ERROR) << "Cannot Add 10 ms audio, payload length is zero";
kwibergc13ded52016-06-17 06:00:45 -0700414 return -1;
415 }
416
417 if (audio_frame.sample_rate_hz_ > 48000) {
418 assert(false);
Mirko Bonadei675513b2017-11-09 11:09:25 +0100419 RTC_LOG(LS_ERROR) << "Cannot Add 10 ms audio, input frequency not valid";
kwibergc13ded52016-06-17 06:00:45 -0700420 return -1;
421 }
422
423 // If the length and frequency matches. We currently just support raw PCM.
424 if (static_cast<size_t>(audio_frame.sample_rate_hz_ / 100) !=
425 audio_frame.samples_per_channel_) {
Mirko Bonadei675513b2017-11-09 11:09:25 +0100426 RTC_LOG(LS_ERROR)
Alex Loiko300ec8c2017-05-30 17:23:28 +0200427 << "Cannot Add 10 ms audio, input frequency and length doesn't match";
kwibergc13ded52016-06-17 06:00:45 -0700428 return -1;
429 }
430
Alex Loiko65438812019-02-22 10:13:44 +0100431 if (audio_frame.num_channels_ != 1 && audio_frame.num_channels_ != 2 &&
432 audio_frame.num_channels_ != 4 && audio_frame.num_channels_ != 6 &&
433 audio_frame.num_channels_ != 8) {
Mirko Bonadei675513b2017-11-09 11:09:25 +0100434 RTC_LOG(LS_ERROR) << "Cannot Add 10 ms audio, invalid number of channels.";
kwibergc13ded52016-06-17 06:00:45 -0700435 return -1;
436 }
437
438 // Do we have a codec registered?
439 if (!HaveValidEncoder("Add10MsData")) {
440 return -1;
441 }
442
443 const AudioFrame* ptr_frame;
444 // Perform a resampling, also down-mix if it is required and can be
445 // performed before resampling (a down mix prior to resampling will take
446 // place if both primary and secondary encoders are mono and input is in
447 // stereo).
448 if (PreprocessToAddData(audio_frame, &ptr_frame) < 0) {
449 return -1;
450 }
451
452 // Check whether we need an up-mix or down-mix?
453 const size_t current_num_channels = encoder_stack_->NumChannels();
454 const bool same_num_channels =
455 ptr_frame->num_channels_ == current_num_channels;
456
457 if (!same_num_channels) {
458 if (ptr_frame->num_channels_ == 1) {
459 if (UpMix(*ptr_frame, WEBRTC_10MS_PCM_AUDIO, input_data->buffer) < 0)
460 return -1;
461 } else {
462 if (DownMix(*ptr_frame, WEBRTC_10MS_PCM_AUDIO, input_data->buffer) < 0)
463 return -1;
464 }
465 }
466
467 // When adding data to encoders this pointer is pointing to an audio buffer
468 // with correct number of channels.
yujo36b1a5f2017-06-12 12:45:32 -0700469 const int16_t* ptr_audio = ptr_frame->data();
kwibergc13ded52016-06-17 06:00:45 -0700470
471 // For pushing data to primary, point the |ptr_audio| to correct buffer.
472 if (!same_num_channels)
473 ptr_audio = input_data->buffer;
474
yujo36b1a5f2017-06-12 12:45:32 -0700475 // TODO(yujo): Skip encode of muted frames.
kwibergc13ded52016-06-17 06:00:45 -0700476 input_data->input_timestamp = ptr_frame->timestamp_;
477 input_data->audio = ptr_audio;
478 input_data->length_per_channel = ptr_frame->samples_per_channel_;
479 input_data->audio_channel = current_num_channels;
480
481 return 0;
482}
483
484// Perform a resampling and down-mix if required. We down-mix only if
485// encoder is mono and input is stereo. In case of dual-streaming, both
486// encoders has to be mono for down-mix to take place.
487// |*ptr_out| will point to the pre-processed audio-frame. If no pre-processing
488// is required, |*ptr_out| points to |in_frame|.
yujo36b1a5f2017-06-12 12:45:32 -0700489// TODO(yujo): Make this more efficient for muted frames.
kwibergc13ded52016-06-17 06:00:45 -0700490int AudioCodingModuleImpl::PreprocessToAddData(const AudioFrame& in_frame,
491 const AudioFrame** ptr_out) {
492 const bool resample =
493 in_frame.sample_rate_hz_ != encoder_stack_->SampleRateHz();
494
495 // This variable is true if primary codec and secondary codec (if exists)
496 // are both mono and input is stereo.
497 // TODO(henrik.lundin): This condition should probably be
498 // in_frame.num_channels_ > encoder_stack_->NumChannels()
499 const bool down_mix =
500 in_frame.num_channels_ == 2 && encoder_stack_->NumChannels() == 1;
501
502 if (!first_10ms_data_) {
503 expected_in_ts_ = in_frame.timestamp_;
504 expected_codec_ts_ = in_frame.timestamp_;
505 first_10ms_data_ = true;
506 } else if (in_frame.timestamp_ != expected_in_ts_) {
Mirko Bonadei675513b2017-11-09 11:09:25 +0100507 RTC_LOG(LS_WARNING) << "Unexpected input timestamp: " << in_frame.timestamp_
508 << ", expected: " << expected_in_ts_;
kwibergc13ded52016-06-17 06:00:45 -0700509 expected_codec_ts_ +=
510 (in_frame.timestamp_ - expected_in_ts_) *
511 static_cast<uint32_t>(
512 static_cast<double>(encoder_stack_->SampleRateHz()) /
513 static_cast<double>(in_frame.sample_rate_hz_));
514 expected_in_ts_ = in_frame.timestamp_;
515 }
516
kwibergc13ded52016-06-17 06:00:45 -0700517 if (!down_mix && !resample) {
518 // No pre-processing is required.
ossu63fb95a2016-07-06 09:34:22 -0700519 if (expected_in_ts_ == expected_codec_ts_) {
520 // If we've never resampled, we can use the input frame as-is
521 *ptr_out = &in_frame;
522 } else {
523 // Otherwise we'll need to alter the timestamp. Since in_frame is const,
524 // we'll have to make a copy of it.
525 preprocess_frame_.CopyFrom(in_frame);
526 preprocess_frame_.timestamp_ = expected_codec_ts_;
527 *ptr_out = &preprocess_frame_;
528 }
529
kwibergc13ded52016-06-17 06:00:45 -0700530 expected_in_ts_ += static_cast<uint32_t>(in_frame.samples_per_channel_);
531 expected_codec_ts_ += static_cast<uint32_t>(in_frame.samples_per_channel_);
kwibergc13ded52016-06-17 06:00:45 -0700532 return 0;
533 }
534
535 *ptr_out = &preprocess_frame_;
536 preprocess_frame_.num_channels_ = in_frame.num_channels_;
537 int16_t audio[WEBRTC_10MS_PCM_AUDIO];
yujo36b1a5f2017-06-12 12:45:32 -0700538 const int16_t* src_ptr_audio = in_frame.data();
kwibergc13ded52016-06-17 06:00:45 -0700539 if (down_mix) {
540 // If a resampling is required the output of a down-mix is written into a
541 // local buffer, otherwise, it will be written to the output frame.
Yves Gerey665174f2018-06-19 15:03:05 +0200542 int16_t* dest_ptr_audio =
543 resample ? audio : preprocess_frame_.mutable_data();
kwibergc13ded52016-06-17 06:00:45 -0700544 if (DownMix(in_frame, WEBRTC_10MS_PCM_AUDIO, dest_ptr_audio) < 0)
545 return -1;
546 preprocess_frame_.num_channels_ = 1;
547 // Set the input of the resampler is the down-mixed signal.
548 src_ptr_audio = audio;
549 }
550
551 preprocess_frame_.timestamp_ = expected_codec_ts_;
552 preprocess_frame_.samples_per_channel_ = in_frame.samples_per_channel_;
553 preprocess_frame_.sample_rate_hz_ = in_frame.sample_rate_hz_;
554 // If it is required, we have to do a resampling.
555 if (resample) {
556 // The result of the resampler is written to output frame.
yujo36b1a5f2017-06-12 12:45:32 -0700557 int16_t* dest_ptr_audio = preprocess_frame_.mutable_data();
kwibergc13ded52016-06-17 06:00:45 -0700558
559 int samples_per_channel = resampler_.Resample10Msec(
560 src_ptr_audio, in_frame.sample_rate_hz_, encoder_stack_->SampleRateHz(),
561 preprocess_frame_.num_channels_, AudioFrame::kMaxDataSizeSamples,
562 dest_ptr_audio);
563
564 if (samples_per_channel < 0) {
Mirko Bonadei675513b2017-11-09 11:09:25 +0100565 RTC_LOG(LS_ERROR) << "Cannot add 10 ms audio, resampling failed";
kwibergc13ded52016-06-17 06:00:45 -0700566 return -1;
567 }
568 preprocess_frame_.samples_per_channel_ =
569 static_cast<size_t>(samples_per_channel);
570 preprocess_frame_.sample_rate_hz_ = encoder_stack_->SampleRateHz();
571 }
572
573 expected_codec_ts_ +=
574 static_cast<uint32_t>(preprocess_frame_.samples_per_channel_);
575 expected_in_ts_ += static_cast<uint32_t>(in_frame.samples_per_channel_);
576
577 return 0;
578}
579
580/////////////////////////////////////////
kwibergc13ded52016-06-17 06:00:45 -0700581// (FEC) Forward Error Correction (codec internal)
582//
583
kwibergc13ded52016-06-17 06:00:45 -0700584int AudioCodingModuleImpl::SetPacketLossRate(int loss_rate) {
585 rtc::CritScope lock(&acm_crit_sect_);
586 if (HaveValidEncoder("SetPacketLossRate")) {
minyue4b9a2cb2016-11-30 06:49:59 -0800587 encoder_stack_->OnReceivedUplinkPacketLossFraction(loss_rate / 100.0);
kwibergc13ded52016-06-17 06:00:45 -0700588 }
589 return 0;
590}
591
592/////////////////////////////////////////
kwibergc13ded52016-06-17 06:00:45 -0700593// Receiver
594//
595
596int AudioCodingModuleImpl::InitializeReceiver() {
597 rtc::CritScope lock(&acm_crit_sect_);
598 return InitializeReceiverSafe();
599}
600
601// Initialize receiver, resets codec database etc.
602int AudioCodingModuleImpl::InitializeReceiverSafe() {
603 // If the receiver is already initialized then we want to destroy any
604 // existing decoders. After a call to this function, we should have a clean
605 // start-up.
kwiberg6b19b562016-09-20 04:02:25 -0700606 if (receiver_initialized_)
607 receiver_.RemoveAllCodecs();
kwibergc13ded52016-06-17 06:00:45 -0700608 receiver_.FlushBuffers();
609
kwibergc13ded52016-06-17 06:00:45 -0700610 receiver_initialized_ = true;
611 return 0;
612}
613
614// Get current receive frequency.
615int AudioCodingModuleImpl::ReceiveFrequency() const {
616 const auto last_packet_sample_rate = receiver_.last_packet_sample_rate_hz();
617 return last_packet_sample_rate ? *last_packet_sample_rate
618 : receiver_.last_output_sample_rate_hz();
619}
620
621// Get current playout frequency.
622int AudioCodingModuleImpl::PlayoutFrequency() const {
kwibergc13ded52016-06-17 06:00:45 -0700623 return receiver_.last_output_sample_rate_hz();
624}
625
kwiberg1c07c702017-03-27 07:15:49 -0700626void AudioCodingModuleImpl::SetReceiveCodecs(
627 const std::map<int, SdpAudioFormat>& codecs) {
628 rtc::CritScope lock(&acm_crit_sect_);
629 receiver_.SetCodecs(codecs);
630}
631
Fredrik Solenbergf693bfa2018-12-11 12:22:10 +0100632absl::optional<std::pair<int, SdpAudioFormat>>
Jonas Olssona4d87372019-07-05 19:08:33 +0200633AudioCodingModuleImpl::ReceiveCodec() const {
kwiberg5adaf732016-10-04 09:33:27 -0700634 rtc::CritScope lock(&acm_crit_sect_);
Fredrik Solenbergf693bfa2018-12-11 12:22:10 +0100635 return receiver_.LastDecoder();
ossue280cde2016-10-12 11:04:10 -0700636}
637
kwibergc13ded52016-06-17 06:00:45 -0700638// Incoming packet from network parsed and ready for decode.
639int AudioCodingModuleImpl::IncomingPacket(const uint8_t* incoming_payload,
640 const size_t payload_length,
Niels Möllerafb5dbb2019-02-15 15:21:47 +0100641 const RTPHeader& rtp_header) {
henrik.lundinb8c55b12017-05-10 07:38:01 -0700642 RTC_DCHECK_EQ(payload_length == 0, incoming_payload == nullptr);
kwibergc13ded52016-06-17 06:00:45 -0700643 return receiver_.InsertPacket(
644 rtp_header,
645 rtc::ArrayView<const uint8_t>(incoming_payload, payload_length));
646}
647
648// Minimum playout delay (Used for lip-sync).
649int AudioCodingModuleImpl::SetMinimumPlayoutDelay(int time_ms) {
650 if ((time_ms < 0) || (time_ms > 10000)) {
Mirko Bonadei675513b2017-11-09 11:09:25 +0100651 RTC_LOG(LS_ERROR) << "Delay must be in the range of 0-10000 milliseconds.";
kwibergc13ded52016-06-17 06:00:45 -0700652 return -1;
653 }
654 return receiver_.SetMinimumDelay(time_ms);
655}
656
657int AudioCodingModuleImpl::SetMaximumPlayoutDelay(int time_ms) {
658 if ((time_ms < 0) || (time_ms > 10000)) {
Mirko Bonadei675513b2017-11-09 11:09:25 +0100659 RTC_LOG(LS_ERROR) << "Delay must be in the range of 0-10000 milliseconds.";
kwibergc13ded52016-06-17 06:00:45 -0700660 return -1;
661 }
662 return receiver_.SetMaximumDelay(time_ms);
663}
664
Ruslan Burakov3b50f9f2019-02-06 09:45:56 +0100665bool AudioCodingModuleImpl::SetBaseMinimumPlayoutDelayMs(int delay_ms) {
666 // All necessary validation happens on NetEq level.
667 return receiver_.SetBaseMinimumDelayMs(delay_ms);
668}
669
670int AudioCodingModuleImpl::GetBaseMinimumPlayoutDelayMs() const {
671 return receiver_.GetBaseMinimumDelayMs();
672}
673
kwibergc13ded52016-06-17 06:00:45 -0700674// Get 10 milliseconds of raw audio data to play out.
675// Automatic resample to the requested frequency.
676int AudioCodingModuleImpl::PlayoutData10Ms(int desired_freq_hz,
677 AudioFrame* audio_frame,
678 bool* muted) {
679 // GetAudio always returns 10 ms, at the requested sample rate.
680 if (receiver_.GetAudio(desired_freq_hz, audio_frame, muted) != 0) {
Mirko Bonadei675513b2017-11-09 11:09:25 +0100681 RTC_LOG(LS_ERROR) << "PlayoutData failed, RecOut Failed";
kwibergc13ded52016-06-17 06:00:45 -0700682 return -1;
683 }
kwibergc13ded52016-06-17 06:00:45 -0700684 return 0;
685}
686
kwibergc13ded52016-06-17 06:00:45 -0700687/////////////////////////////////////////
688// Statistics
689//
690
691// TODO(turajs) change the return value to void. Also change the corresponding
692// NetEq function.
693int AudioCodingModuleImpl::GetNetworkStatistics(NetworkStatistics* statistics) {
694 receiver_.GetNetworkStatistics(statistics);
695 return 0;
696}
697
698int AudioCodingModuleImpl::RegisterVADCallback(ACMVADCallback* vad_callback) {
Mirko Bonadei675513b2017-11-09 11:09:25 +0100699 RTC_LOG(LS_VERBOSE) << "RegisterVADCallback()";
kwibergc13ded52016-06-17 06:00:45 -0700700 rtc::CritScope lock(&callback_crit_sect_);
701 vad_callback_ = vad_callback;
702 return 0;
703}
704
Danil Chapovalovb6021232018-06-19 13:26:36 +0200705absl::optional<uint32_t> AudioCodingModuleImpl::PlayoutTimestamp() {
kwibergc13ded52016-06-17 06:00:45 -0700706 return receiver_.GetPlayoutTimestamp();
707}
708
henrik.lundinb3f1c5d2016-08-22 15:39:53 -0700709int AudioCodingModuleImpl::FilteredCurrentDelayMs() const {
710 return receiver_.FilteredCurrentDelayMs();
711}
712
Henrik Lundinabbff892017-11-29 09:14:04 +0100713int AudioCodingModuleImpl::TargetDelayMs() const {
714 return receiver_.TargetDelayMs();
715}
716
kwibergc13ded52016-06-17 06:00:45 -0700717bool AudioCodingModuleImpl::HaveValidEncoder(const char* caller_name) const {
718 if (!encoder_stack_) {
Mirko Bonadei675513b2017-11-09 11:09:25 +0100719 RTC_LOG(LS_ERROR) << caller_name << " failed: No send codec is registered.";
kwibergc13ded52016-06-17 06:00:45 -0700720 return false;
721 }
722 return true;
723}
724
kwibergc13ded52016-06-17 06:00:45 -0700725int AudioCodingModuleImpl::EnableNack(size_t max_nack_list_size) {
726 return receiver_.EnableNack(max_nack_list_size);
727}
728
729void AudioCodingModuleImpl::DisableNack() {
730 receiver_.DisableNack();
731}
732
733std::vector<uint16_t> AudioCodingModuleImpl::GetNackList(
734 int64_t round_trip_time_ms) const {
735 return receiver_.GetNackList(round_trip_time_ms);
736}
737
kwibergc13ded52016-06-17 06:00:45 -0700738void AudioCodingModuleImpl::GetDecodingCallStatistics(
Yves Gerey665174f2018-06-19 15:03:05 +0200739 AudioDecodingCallStats* call_stats) const {
kwibergc13ded52016-06-17 06:00:45 -0700740 receiver_.GetDecodingCallStatistics(call_stats);
741}
742
ivoce1198e02017-09-08 08:13:19 -0700743ANAStats AudioCodingModuleImpl::GetANAStats() const {
744 rtc::CritScope lock(&acm_crit_sect_);
745 if (encoder_stack_)
746 return encoder_stack_->GetANAStats();
747 // If no encoder is set, return default stats.
748 return ANAStats();
749}
750
kwibergc13ded52016-06-17 06:00:45 -0700751} // namespace
752
Karl Wiberg5817d3d2018-04-06 10:06:42 +0200753AudioCodingModule::Config::Config(
754 rtc::scoped_refptr<AudioDecoderFactory> decoder_factory)
755 : neteq_config(),
756 clock(Clock::GetRealTimeClock()),
757 decoder_factory(decoder_factory) {
kwiberg36a43882016-08-29 05:33:32 -0700758 // Post-decode VAD is disabled by default in NetEq, however, Audio
759 // Conference Mixer relies on VAD decisions and fails without them.
760 neteq_config.enable_post_decode_vad = true;
761}
762
763AudioCodingModule::Config::Config(const Config&) = default;
764AudioCodingModule::Config::~Config() = default;
765
Henrik Lundin64dad832015-05-11 12:44:23 +0200766AudioCodingModule* AudioCodingModule::Create(const Config& config) {
kwibergc13ded52016-06-17 06:00:45 -0700767 return new AudioCodingModuleImpl(config);
turaj@webrtc.org7959e162013-09-12 18:30:26 +0000768}
769
turaj@webrtc.org7959e162013-09-12 18:30:26 +0000770} // namespace webrtc