blob: b0421f50d2d42fae85174ccc6a183e5676498f5e [file] [log] [blame]
turaj@webrtc.org7959e162013-09-12 18:30:26 +00001/*
2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020011#include "modules/audio_coding/include/audio_coding_module.h"
turaj@webrtc.org7959e162013-09-12 18:30:26 +000012
Yves Gerey988cc082018-10-23 12:03:01 +020013#include <assert.h>
Jonas Olssona4d87372019-07-05 19:08:33 +020014
Jonathan Yu36344a02017-07-30 01:55:34 -070015#include <algorithm>
Yves Gerey988cc082018-10-23 12:03:01 +020016#include <cstdint>
Jonathan Yu36344a02017-07-30 01:55:34 -070017
Niels Möller2edab4c2018-10-22 09:48:08 +020018#include "absl/strings/match.h"
Yves Gerey988cc082018-10-23 12:03:01 +020019#include "api/array_view.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020020#include "modules/audio_coding/acm2/acm_receiver.h"
21#include "modules/audio_coding/acm2/acm_resampler.h"
Fredrik Solenbergbbf21a32018-04-12 22:44:09 +020022#include "modules/include/module_common_types.h"
Yves Gerey988cc082018-10-23 12:03:01 +020023#include "modules/include/module_common_types_public.h"
24#include "rtc_base/buffer.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020025#include "rtc_base/checks.h"
Steve Anton10542f22019-01-11 09:11:00 -080026#include "rtc_base/critical_section.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020027#include "rtc_base/logging.h"
Karl Wiberge40468b2017-11-22 10:42:26 +010028#include "rtc_base/numerics/safe_conversions.h"
Yves Gerey988cc082018-10-23 12:03:01 +020029#include "rtc_base/thread_annotations.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020030#include "system_wrappers/include/metrics.h"
turaj@webrtc.org7959e162013-09-12 18:30:26 +000031
32namespace webrtc {
33
kwibergc13ded52016-06-17 06:00:45 -070034namespace {
35
kwibergc13ded52016-06-17 06:00:45 -070036class AudioCodingModuleImpl final : public AudioCodingModule {
37 public:
38 explicit AudioCodingModuleImpl(const AudioCodingModule::Config& config);
39 ~AudioCodingModuleImpl() override;
40
41 /////////////////////////////////////////
42 // Sender
43 //
44
kwiberg24c7c122016-09-28 11:57:10 -070045 void ModifyEncoder(rtc::FunctionView<void(std::unique_ptr<AudioEncoder>*)>
46 modifier) override;
kwibergc13ded52016-06-17 06:00:45 -070047
kwibergc13ded52016-06-17 06:00:45 -070048 // Register a transport callback which will be
49 // called to deliver the encoded buffers.
50 int RegisterTransportCallback(AudioPacketizationCallback* transport) override;
51
52 // Add 10 ms of raw (PCM) audio data to the encoder.
53 int Add10MsData(const AudioFrame& audio_frame) override;
54
55 /////////////////////////////////////////
kwibergc13ded52016-06-17 06:00:45 -070056 // (FEC) Forward Error Correction (codec internal)
57 //
58
kwibergc13ded52016-06-17 06:00:45 -070059 // Set target packet loss rate
60 int SetPacketLossRate(int loss_rate) override;
61
62 /////////////////////////////////////////
63 // (VAD) Voice Activity Detection
64 // and
65 // (CNG) Comfort Noise Generation
66 //
67
kwibergc13ded52016-06-17 06:00:45 -070068 int RegisterVADCallback(ACMVADCallback* vad_callback) override;
69
70 /////////////////////////////////////////
71 // Receiver
72 //
73
74 // Initialize receiver, resets codec database etc.
75 int InitializeReceiver() override;
76
77 // Get current receive frequency.
78 int ReceiveFrequency() const override;
79
80 // Get current playout frequency.
81 int PlayoutFrequency() const override;
82
kwiberg1c07c702017-03-27 07:15:49 -070083 void SetReceiveCodecs(const std::map<int, SdpAudioFormat>& codecs) override;
84
kwibergc13ded52016-06-17 06:00:45 -070085 // Get current received codec.
Fredrik Solenbergf693bfa2018-12-11 12:22:10 +010086 absl::optional<std::pair<int, SdpAudioFormat>> ReceiveCodec() const override;
ossue280cde2016-10-12 11:04:10 -070087
kwibergc13ded52016-06-17 06:00:45 -070088 // Incoming packet from network parsed and ready for decode.
89 int IncomingPacket(const uint8_t* incoming_payload,
90 const size_t payload_length,
Niels Möllerafb5dbb2019-02-15 15:21:47 +010091 const RTPHeader& rtp_info) override;
kwibergc13ded52016-06-17 06:00:45 -070092
kwibergc13ded52016-06-17 06:00:45 -070093 // Minimum playout delay.
94 int SetMinimumPlayoutDelay(int time_ms) override;
95
96 // Maximum playout delay.
97 int SetMaximumPlayoutDelay(int time_ms) override;
98
Ruslan Burakov3b50f9f2019-02-06 09:45:56 +010099 bool SetBaseMinimumPlayoutDelayMs(int delay_ms) override;
100
101 int GetBaseMinimumPlayoutDelayMs() const override;
102
Danil Chapovalovb6021232018-06-19 13:26:36 +0200103 absl::optional<uint32_t> PlayoutTimestamp() override;
kwibergc13ded52016-06-17 06:00:45 -0700104
henrik.lundinb3f1c5d2016-08-22 15:39:53 -0700105 int FilteredCurrentDelayMs() const override;
106
Henrik Lundinabbff892017-11-29 09:14:04 +0100107 int TargetDelayMs() const override;
108
kwibergc13ded52016-06-17 06:00:45 -0700109 // Get 10 milliseconds of raw audio data to play out, and
110 // automatic resample to the requested frequency if > 0.
111 int PlayoutData10Ms(int desired_freq_hz,
112 AudioFrame* audio_frame,
113 bool* muted) override;
kwibergc13ded52016-06-17 06:00:45 -0700114
115 /////////////////////////////////////////
116 // Statistics
117 //
118
119 int GetNetworkStatistics(NetworkStatistics* statistics) override;
120
kwibergc13ded52016-06-17 06:00:45 -0700121 void GetDecodingCallStatistics(AudioDecodingCallStats* stats) const override;
122
ivoce1198e02017-09-08 08:13:19 -0700123 ANAStats GetANAStats() const override;
124
kwibergc13ded52016-06-17 06:00:45 -0700125 private:
126 struct InputData {
127 uint32_t input_timestamp;
128 const int16_t* audio;
129 size_t length_per_channel;
130 size_t audio_channel;
131 // If a re-mix is required (up or down), this buffer will store a re-mixed
132 // version of the input.
133 int16_t buffer[WEBRTC_10MS_PCM_AUDIO];
134 };
135
136 // This member class writes values to the named UMA histogram, but only if
137 // the value has changed since the last time (and always for the first call).
138 class ChangeLogger {
139 public:
140 explicit ChangeLogger(const std::string& histogram_name)
141 : histogram_name_(histogram_name) {}
142 // Logs the new value if it is different from the last logged value, or if
143 // this is the first call.
144 void MaybeLog(int value);
145
146 private:
147 int last_value_ = 0;
148 int first_time_ = true;
149 const std::string histogram_name_;
150 };
151
kwibergc13ded52016-06-17 06:00:45 -0700152 int Add10MsDataInternal(const AudioFrame& audio_frame, InputData* input_data)
danilchap56359be2017-09-07 07:53:45 -0700153 RTC_EXCLUSIVE_LOCKS_REQUIRED(acm_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700154 int Encode(const InputData& input_data)
danilchap56359be2017-09-07 07:53:45 -0700155 RTC_EXCLUSIVE_LOCKS_REQUIRED(acm_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700156
danilchap56359be2017-09-07 07:53:45 -0700157 int InitializeReceiverSafe() RTC_EXCLUSIVE_LOCKS_REQUIRED(acm_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700158
159 bool HaveValidEncoder(const char* caller_name) const
danilchap56359be2017-09-07 07:53:45 -0700160 RTC_EXCLUSIVE_LOCKS_REQUIRED(acm_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700161
162 // Preprocessing of input audio, including resampling and down-mixing if
163 // required, before pushing audio into encoder's buffer.
164 //
165 // in_frame: input audio-frame
166 // ptr_out: pointer to output audio_frame. If no preprocessing is required
167 // |ptr_out| will be pointing to |in_frame|, otherwise pointing to
168 // |preprocess_frame_|.
169 //
170 // Return value:
171 // -1: if encountering an error.
172 // 0: otherwise.
173 int PreprocessToAddData(const AudioFrame& in_frame,
174 const AudioFrame** ptr_out)
danilchap56359be2017-09-07 07:53:45 -0700175 RTC_EXCLUSIVE_LOCKS_REQUIRED(acm_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700176
177 // Change required states after starting to receive the codec corresponding
178 // to |index|.
179 int UpdateUponReceivingCodec(int index);
180
181 rtc::CriticalSection acm_crit_sect_;
danilchap56359be2017-09-07 07:53:45 -0700182 rtc::Buffer encode_buffer_ RTC_GUARDED_BY(acm_crit_sect_);
danilchap56359be2017-09-07 07:53:45 -0700183 uint32_t expected_codec_ts_ RTC_GUARDED_BY(acm_crit_sect_);
184 uint32_t expected_in_ts_ RTC_GUARDED_BY(acm_crit_sect_);
185 acm2::ACMResampler resampler_ RTC_GUARDED_BY(acm_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700186 acm2::AcmReceiver receiver_; // AcmReceiver has it's own internal lock.
danilchap56359be2017-09-07 07:53:45 -0700187 ChangeLogger bitrate_logger_ RTC_GUARDED_BY(acm_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700188
Karl Wiberg49c33ce2018-11-12 14:21:58 +0100189 // Current encoder stack, provided by a call to RegisterEncoder.
danilchap56359be2017-09-07 07:53:45 -0700190 std::unique_ptr<AudioEncoder> encoder_stack_ RTC_GUARDED_BY(acm_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700191
kwibergc13ded52016-06-17 06:00:45 -0700192 // This is to keep track of CN instances where we can send DTMFs.
danilchap56359be2017-09-07 07:53:45 -0700193 uint8_t previous_pltype_ RTC_GUARDED_BY(acm_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700194
danilchap56359be2017-09-07 07:53:45 -0700195 bool receiver_initialized_ RTC_GUARDED_BY(acm_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700196
danilchap56359be2017-09-07 07:53:45 -0700197 AudioFrame preprocess_frame_ RTC_GUARDED_BY(acm_crit_sect_);
198 bool first_10ms_data_ RTC_GUARDED_BY(acm_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700199
danilchap56359be2017-09-07 07:53:45 -0700200 bool first_frame_ RTC_GUARDED_BY(acm_crit_sect_);
201 uint32_t last_timestamp_ RTC_GUARDED_BY(acm_crit_sect_);
202 uint32_t last_rtp_timestamp_ RTC_GUARDED_BY(acm_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700203
204 rtc::CriticalSection callback_crit_sect_;
205 AudioPacketizationCallback* packetization_callback_
danilchap56359be2017-09-07 07:53:45 -0700206 RTC_GUARDED_BY(callback_crit_sect_);
207 ACMVADCallback* vad_callback_ RTC_GUARDED_BY(callback_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700208
209 int codec_histogram_bins_log_[static_cast<size_t>(
210 AudioEncoder::CodecType::kMaxLoggedAudioCodecTypes)];
211 int number_of_consecutive_empty_packets_;
212};
213
214// Adds a codec usage sample to the histogram.
215void UpdateCodecTypeHistogram(size_t codec_type) {
216 RTC_HISTOGRAM_ENUMERATION(
217 "WebRTC.Audio.Encoder.CodecType", static_cast<int>(codec_type),
218 static_cast<int>(
219 webrtc::AudioEncoder::CodecType::kMaxLoggedAudioCodecTypes));
220}
221
kwibergc13ded52016-06-17 06:00:45 -0700222// Stereo-to-mono can be used as in-place.
223int DownMix(const AudioFrame& frame,
224 size_t length_out_buff,
225 int16_t* out_buff) {
yujo36b1a5f2017-06-12 12:45:32 -0700226 RTC_DCHECK_EQ(frame.num_channels_, 2);
227 RTC_DCHECK_GE(length_out_buff, frame.samples_per_channel_);
228
229 if (!frame.muted()) {
230 const int16_t* frame_data = frame.data();
231 for (size_t n = 0; n < frame.samples_per_channel_; ++n) {
Yves Gerey665174f2018-06-19 15:03:05 +0200232 out_buff[n] =
233 static_cast<int16_t>((static_cast<int32_t>(frame_data[2 * n]) +
234 static_cast<int32_t>(frame_data[2 * n + 1])) >>
235 1);
yujo36b1a5f2017-06-12 12:45:32 -0700236 }
237 } else {
Jonathan Yu36344a02017-07-30 01:55:34 -0700238 std::fill(out_buff, out_buff + frame.samples_per_channel_, 0);
kwibergc13ded52016-06-17 06:00:45 -0700239 }
kwibergc13ded52016-06-17 06:00:45 -0700240 return 0;
241}
242
243// Mono-to-stereo can be used as in-place.
244int UpMix(const AudioFrame& frame, size_t length_out_buff, int16_t* out_buff) {
yujo36b1a5f2017-06-12 12:45:32 -0700245 RTC_DCHECK_EQ(frame.num_channels_, 1);
246 RTC_DCHECK_GE(length_out_buff, 2 * frame.samples_per_channel_);
247
248 if (!frame.muted()) {
249 const int16_t* frame_data = frame.data();
250 for (size_t n = frame.samples_per_channel_; n != 0; --n) {
251 size_t i = n - 1;
252 int16_t sample = frame_data[i];
253 out_buff[2 * i + 1] = sample;
254 out_buff[2 * i] = sample;
255 }
256 } else {
Jonathan Yu36344a02017-07-30 01:55:34 -0700257 std::fill(out_buff, out_buff + frame.samples_per_channel_ * 2, 0);
kwibergc13ded52016-06-17 06:00:45 -0700258 }
259 return 0;
260}
261
kwibergc13ded52016-06-17 06:00:45 -0700262void AudioCodingModuleImpl::ChangeLogger::MaybeLog(int value) {
263 if (value != last_value_ || first_time_) {
264 first_time_ = false;
265 last_value_ = value;
266 RTC_HISTOGRAM_COUNTS_SPARSE_100(histogram_name_, value);
267 }
268}
269
270AudioCodingModuleImpl::AudioCodingModuleImpl(
271 const AudioCodingModule::Config& config)
solenbergc7b4a452017-09-28 07:37:11 -0700272 : expected_codec_ts_(0xD87F3F9F),
kwibergc13ded52016-06-17 06:00:45 -0700273 expected_in_ts_(0xD87F3F9F),
274 receiver_(config),
275 bitrate_logger_("WebRTC.Audio.TargetBitrateInKbps"),
kwibergc13ded52016-06-17 06:00:45 -0700276 encoder_stack_(nullptr),
277 previous_pltype_(255),
278 receiver_initialized_(false),
279 first_10ms_data_(false),
280 first_frame_(true),
281 packetization_callback_(NULL),
282 vad_callback_(NULL),
283 codec_histogram_bins_log_(),
284 number_of_consecutive_empty_packets_(0) {
285 if (InitializeReceiverSafe() < 0) {
Mirko Bonadei675513b2017-11-09 11:09:25 +0100286 RTC_LOG(LS_ERROR) << "Cannot initialize receiver";
kwibergc13ded52016-06-17 06:00:45 -0700287 }
Mirko Bonadei675513b2017-11-09 11:09:25 +0100288 RTC_LOG(LS_INFO) << "Created";
kwibergc13ded52016-06-17 06:00:45 -0700289}
290
291AudioCodingModuleImpl::~AudioCodingModuleImpl() = default;
292
293int32_t AudioCodingModuleImpl::Encode(const InputData& input_data) {
294 AudioEncoder::EncodedInfo encoded_info;
295 uint8_t previous_pltype;
296
297 // Check if there is an encoder before.
298 if (!HaveValidEncoder("Process"))
299 return -1;
300
Yves Gerey665174f2018-06-19 15:03:05 +0200301 if (!first_frame_) {
deadbeeffcada902016-08-24 12:45:13 -0700302 RTC_DCHECK(IsNewerTimestamp(input_data.input_timestamp, last_timestamp_))
ossu63fb95a2016-07-06 09:34:22 -0700303 << "Time should not move backwards";
304 }
305
kwibergc13ded52016-06-17 06:00:45 -0700306 // Scale the timestamp to the codec's RTP timestamp rate.
307 uint32_t rtp_timestamp =
Karl Wiberg053c3712019-05-16 15:24:17 +0200308 first_frame_
309 ? input_data.input_timestamp
310 : last_rtp_timestamp_ +
311 rtc::dchecked_cast<uint32_t>(rtc::CheckedDivExact(
312 int64_t{input_data.input_timestamp - last_timestamp_} *
313 encoder_stack_->RtpTimestampRateHz(),
314 int64_t{encoder_stack_->SampleRateHz()}));
kwibergc13ded52016-06-17 06:00:45 -0700315 last_timestamp_ = input_data.input_timestamp;
316 last_rtp_timestamp_ = rtp_timestamp;
317 first_frame_ = false;
318
319 // Clear the buffer before reuse - encoded data will get appended.
320 encode_buffer_.Clear();
321 encoded_info = encoder_stack_->Encode(
Yves Gerey665174f2018-06-19 15:03:05 +0200322 rtp_timestamp,
323 rtc::ArrayView<const int16_t>(
324 input_data.audio,
325 input_data.audio_channel * input_data.length_per_channel),
kwibergc13ded52016-06-17 06:00:45 -0700326 &encode_buffer_);
327
328 bitrate_logger_.MaybeLog(encoder_stack_->GetTargetBitrate() / 1000);
329 if (encode_buffer_.size() == 0 && !encoded_info.send_even_if_empty) {
330 // Not enough data.
331 return 0;
332 }
333 previous_pltype = previous_pltype_; // Read it while we have the critsect.
334
335 // Log codec type to histogram once every 500 packets.
336 if (encoded_info.encoded_bytes == 0) {
337 ++number_of_consecutive_empty_packets_;
338 } else {
339 size_t codec_type = static_cast<size_t>(encoded_info.encoder_type);
340 codec_histogram_bins_log_[codec_type] +=
341 number_of_consecutive_empty_packets_ + 1;
342 number_of_consecutive_empty_packets_ = 0;
343 if (codec_histogram_bins_log_[codec_type] >= 500) {
344 codec_histogram_bins_log_[codec_type] -= 500;
345 UpdateCodecTypeHistogram(codec_type);
346 }
347 }
348
Niels Möller87e2d782019-03-07 10:18:23 +0100349 AudioFrameType frame_type;
kwibergc13ded52016-06-17 06:00:45 -0700350 if (encode_buffer_.size() == 0 && encoded_info.send_even_if_empty) {
Niels Möllerc936cb62019-03-19 14:10:16 +0100351 frame_type = AudioFrameType::kEmptyFrame;
kwibergc13ded52016-06-17 06:00:45 -0700352 encoded_info.payload_type = previous_pltype;
353 } else {
kwibergaf476c72016-11-28 15:21:39 -0800354 RTC_DCHECK_GT(encode_buffer_.size(), 0);
Niels Möllerc936cb62019-03-19 14:10:16 +0100355 frame_type = encoded_info.speech ? AudioFrameType::kAudioFrameSpeech
356 : AudioFrameType::kAudioFrameCN;
kwibergc13ded52016-06-17 06:00:45 -0700357 }
358
359 {
360 rtc::CritScope lock(&callback_crit_sect_);
361 if (packetization_callback_) {
362 packetization_callback_->SendData(
363 frame_type, encoded_info.payload_type, encoded_info.encoded_timestamp,
Niels Möllerc35b6e62019-04-25 16:31:18 +0200364 encode_buffer_.data(), encode_buffer_.size());
kwibergc13ded52016-06-17 06:00:45 -0700365 }
366
367 if (vad_callback_) {
368 // Callback with VAD decision.
369 vad_callback_->InFrameType(frame_type);
370 }
371 }
372 previous_pltype_ = encoded_info.payload_type;
373 return static_cast<int32_t>(encode_buffer_.size());
374}
375
376/////////////////////////////////////////
377// Sender
378//
379
kwibergc13ded52016-06-17 06:00:45 -0700380void AudioCodingModuleImpl::ModifyEncoder(
kwiberg24c7c122016-09-28 11:57:10 -0700381 rtc::FunctionView<void(std::unique_ptr<AudioEncoder>*)> modifier) {
kwibergc13ded52016-06-17 06:00:45 -0700382 rtc::CritScope lock(&acm_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700383 modifier(&encoder_stack_);
384}
385
kwibergc13ded52016-06-17 06:00:45 -0700386// Register a transport callback which will be called to deliver
387// the encoded buffers.
388int AudioCodingModuleImpl::RegisterTransportCallback(
389 AudioPacketizationCallback* transport) {
390 rtc::CritScope lock(&callback_crit_sect_);
391 packetization_callback_ = transport;
392 return 0;
393}
394
395// Add 10MS of raw (PCM) audio data to the encoder.
396int AudioCodingModuleImpl::Add10MsData(const AudioFrame& audio_frame) {
397 InputData input_data;
398 rtc::CritScope lock(&acm_crit_sect_);
399 int r = Add10MsDataInternal(audio_frame, &input_data);
400 return r < 0 ? r : Encode(input_data);
401}
402
403int AudioCodingModuleImpl::Add10MsDataInternal(const AudioFrame& audio_frame,
404 InputData* input_data) {
405 if (audio_frame.samples_per_channel_ == 0) {
406 assert(false);
Mirko Bonadei675513b2017-11-09 11:09:25 +0100407 RTC_LOG(LS_ERROR) << "Cannot Add 10 ms audio, payload length is zero";
kwibergc13ded52016-06-17 06:00:45 -0700408 return -1;
409 }
410
411 if (audio_frame.sample_rate_hz_ > 48000) {
412 assert(false);
Mirko Bonadei675513b2017-11-09 11:09:25 +0100413 RTC_LOG(LS_ERROR) << "Cannot Add 10 ms audio, input frequency not valid";
kwibergc13ded52016-06-17 06:00:45 -0700414 return -1;
415 }
416
417 // If the length and frequency matches. We currently just support raw PCM.
418 if (static_cast<size_t>(audio_frame.sample_rate_hz_ / 100) !=
419 audio_frame.samples_per_channel_) {
Mirko Bonadei675513b2017-11-09 11:09:25 +0100420 RTC_LOG(LS_ERROR)
Alex Loiko300ec8c2017-05-30 17:23:28 +0200421 << "Cannot Add 10 ms audio, input frequency and length doesn't match";
kwibergc13ded52016-06-17 06:00:45 -0700422 return -1;
423 }
424
Alex Loiko65438812019-02-22 10:13:44 +0100425 if (audio_frame.num_channels_ != 1 && audio_frame.num_channels_ != 2 &&
426 audio_frame.num_channels_ != 4 && audio_frame.num_channels_ != 6 &&
427 audio_frame.num_channels_ != 8) {
Mirko Bonadei675513b2017-11-09 11:09:25 +0100428 RTC_LOG(LS_ERROR) << "Cannot Add 10 ms audio, invalid number of channels.";
kwibergc13ded52016-06-17 06:00:45 -0700429 return -1;
430 }
431
432 // Do we have a codec registered?
433 if (!HaveValidEncoder("Add10MsData")) {
434 return -1;
435 }
436
437 const AudioFrame* ptr_frame;
438 // Perform a resampling, also down-mix if it is required and can be
439 // performed before resampling (a down mix prior to resampling will take
440 // place if both primary and secondary encoders are mono and input is in
441 // stereo).
442 if (PreprocessToAddData(audio_frame, &ptr_frame) < 0) {
443 return -1;
444 }
445
446 // Check whether we need an up-mix or down-mix?
447 const size_t current_num_channels = encoder_stack_->NumChannels();
448 const bool same_num_channels =
449 ptr_frame->num_channels_ == current_num_channels;
450
451 if (!same_num_channels) {
452 if (ptr_frame->num_channels_ == 1) {
453 if (UpMix(*ptr_frame, WEBRTC_10MS_PCM_AUDIO, input_data->buffer) < 0)
454 return -1;
455 } else {
456 if (DownMix(*ptr_frame, WEBRTC_10MS_PCM_AUDIO, input_data->buffer) < 0)
457 return -1;
458 }
459 }
460
461 // When adding data to encoders this pointer is pointing to an audio buffer
462 // with correct number of channels.
yujo36b1a5f2017-06-12 12:45:32 -0700463 const int16_t* ptr_audio = ptr_frame->data();
kwibergc13ded52016-06-17 06:00:45 -0700464
465 // For pushing data to primary, point the |ptr_audio| to correct buffer.
466 if (!same_num_channels)
467 ptr_audio = input_data->buffer;
468
yujo36b1a5f2017-06-12 12:45:32 -0700469 // TODO(yujo): Skip encode of muted frames.
kwibergc13ded52016-06-17 06:00:45 -0700470 input_data->input_timestamp = ptr_frame->timestamp_;
471 input_data->audio = ptr_audio;
472 input_data->length_per_channel = ptr_frame->samples_per_channel_;
473 input_data->audio_channel = current_num_channels;
474
475 return 0;
476}
477
478// Perform a resampling and down-mix if required. We down-mix only if
479// encoder is mono and input is stereo. In case of dual-streaming, both
480// encoders has to be mono for down-mix to take place.
481// |*ptr_out| will point to the pre-processed audio-frame. If no pre-processing
482// is required, |*ptr_out| points to |in_frame|.
yujo36b1a5f2017-06-12 12:45:32 -0700483// TODO(yujo): Make this more efficient for muted frames.
kwibergc13ded52016-06-17 06:00:45 -0700484int AudioCodingModuleImpl::PreprocessToAddData(const AudioFrame& in_frame,
485 const AudioFrame** ptr_out) {
486 const bool resample =
487 in_frame.sample_rate_hz_ != encoder_stack_->SampleRateHz();
488
489 // This variable is true if primary codec and secondary codec (if exists)
490 // are both mono and input is stereo.
491 // TODO(henrik.lundin): This condition should probably be
492 // in_frame.num_channels_ > encoder_stack_->NumChannels()
493 const bool down_mix =
494 in_frame.num_channels_ == 2 && encoder_stack_->NumChannels() == 1;
495
496 if (!first_10ms_data_) {
497 expected_in_ts_ = in_frame.timestamp_;
498 expected_codec_ts_ = in_frame.timestamp_;
499 first_10ms_data_ = true;
500 } else if (in_frame.timestamp_ != expected_in_ts_) {
Mirko Bonadei675513b2017-11-09 11:09:25 +0100501 RTC_LOG(LS_WARNING) << "Unexpected input timestamp: " << in_frame.timestamp_
502 << ", expected: " << expected_in_ts_;
kwibergc13ded52016-06-17 06:00:45 -0700503 expected_codec_ts_ +=
504 (in_frame.timestamp_ - expected_in_ts_) *
505 static_cast<uint32_t>(
506 static_cast<double>(encoder_stack_->SampleRateHz()) /
507 static_cast<double>(in_frame.sample_rate_hz_));
508 expected_in_ts_ = in_frame.timestamp_;
509 }
510
kwibergc13ded52016-06-17 06:00:45 -0700511 if (!down_mix && !resample) {
512 // No pre-processing is required.
ossu63fb95a2016-07-06 09:34:22 -0700513 if (expected_in_ts_ == expected_codec_ts_) {
514 // If we've never resampled, we can use the input frame as-is
515 *ptr_out = &in_frame;
516 } else {
517 // Otherwise we'll need to alter the timestamp. Since in_frame is const,
518 // we'll have to make a copy of it.
519 preprocess_frame_.CopyFrom(in_frame);
520 preprocess_frame_.timestamp_ = expected_codec_ts_;
521 *ptr_out = &preprocess_frame_;
522 }
523
kwibergc13ded52016-06-17 06:00:45 -0700524 expected_in_ts_ += static_cast<uint32_t>(in_frame.samples_per_channel_);
525 expected_codec_ts_ += static_cast<uint32_t>(in_frame.samples_per_channel_);
kwibergc13ded52016-06-17 06:00:45 -0700526 return 0;
527 }
528
529 *ptr_out = &preprocess_frame_;
530 preprocess_frame_.num_channels_ = in_frame.num_channels_;
531 int16_t audio[WEBRTC_10MS_PCM_AUDIO];
yujo36b1a5f2017-06-12 12:45:32 -0700532 const int16_t* src_ptr_audio = in_frame.data();
kwibergc13ded52016-06-17 06:00:45 -0700533 if (down_mix) {
534 // If a resampling is required the output of a down-mix is written into a
535 // local buffer, otherwise, it will be written to the output frame.
Yves Gerey665174f2018-06-19 15:03:05 +0200536 int16_t* dest_ptr_audio =
537 resample ? audio : preprocess_frame_.mutable_data();
kwibergc13ded52016-06-17 06:00:45 -0700538 if (DownMix(in_frame, WEBRTC_10MS_PCM_AUDIO, dest_ptr_audio) < 0)
539 return -1;
540 preprocess_frame_.num_channels_ = 1;
541 // Set the input of the resampler is the down-mixed signal.
542 src_ptr_audio = audio;
543 }
544
545 preprocess_frame_.timestamp_ = expected_codec_ts_;
546 preprocess_frame_.samples_per_channel_ = in_frame.samples_per_channel_;
547 preprocess_frame_.sample_rate_hz_ = in_frame.sample_rate_hz_;
548 // If it is required, we have to do a resampling.
549 if (resample) {
550 // The result of the resampler is written to output frame.
yujo36b1a5f2017-06-12 12:45:32 -0700551 int16_t* dest_ptr_audio = preprocess_frame_.mutable_data();
kwibergc13ded52016-06-17 06:00:45 -0700552
553 int samples_per_channel = resampler_.Resample10Msec(
554 src_ptr_audio, in_frame.sample_rate_hz_, encoder_stack_->SampleRateHz(),
555 preprocess_frame_.num_channels_, AudioFrame::kMaxDataSizeSamples,
556 dest_ptr_audio);
557
558 if (samples_per_channel < 0) {
Mirko Bonadei675513b2017-11-09 11:09:25 +0100559 RTC_LOG(LS_ERROR) << "Cannot add 10 ms audio, resampling failed";
kwibergc13ded52016-06-17 06:00:45 -0700560 return -1;
561 }
562 preprocess_frame_.samples_per_channel_ =
563 static_cast<size_t>(samples_per_channel);
564 preprocess_frame_.sample_rate_hz_ = encoder_stack_->SampleRateHz();
565 }
566
567 expected_codec_ts_ +=
568 static_cast<uint32_t>(preprocess_frame_.samples_per_channel_);
569 expected_in_ts_ += static_cast<uint32_t>(in_frame.samples_per_channel_);
570
571 return 0;
572}
573
574/////////////////////////////////////////
kwibergc13ded52016-06-17 06:00:45 -0700575// (FEC) Forward Error Correction (codec internal)
576//
577
kwibergc13ded52016-06-17 06:00:45 -0700578int AudioCodingModuleImpl::SetPacketLossRate(int loss_rate) {
579 rtc::CritScope lock(&acm_crit_sect_);
580 if (HaveValidEncoder("SetPacketLossRate")) {
minyue4b9a2cb2016-11-30 06:49:59 -0800581 encoder_stack_->OnReceivedUplinkPacketLossFraction(loss_rate / 100.0);
kwibergc13ded52016-06-17 06:00:45 -0700582 }
583 return 0;
584}
585
586/////////////////////////////////////////
kwibergc13ded52016-06-17 06:00:45 -0700587// Receiver
588//
589
590int AudioCodingModuleImpl::InitializeReceiver() {
591 rtc::CritScope lock(&acm_crit_sect_);
592 return InitializeReceiverSafe();
593}
594
595// Initialize receiver, resets codec database etc.
596int AudioCodingModuleImpl::InitializeReceiverSafe() {
597 // If the receiver is already initialized then we want to destroy any
598 // existing decoders. After a call to this function, we should have a clean
599 // start-up.
kwiberg6b19b562016-09-20 04:02:25 -0700600 if (receiver_initialized_)
601 receiver_.RemoveAllCodecs();
kwibergc13ded52016-06-17 06:00:45 -0700602 receiver_.FlushBuffers();
603
kwibergc13ded52016-06-17 06:00:45 -0700604 receiver_initialized_ = true;
605 return 0;
606}
607
608// Get current receive frequency.
609int AudioCodingModuleImpl::ReceiveFrequency() const {
610 const auto last_packet_sample_rate = receiver_.last_packet_sample_rate_hz();
611 return last_packet_sample_rate ? *last_packet_sample_rate
612 : receiver_.last_output_sample_rate_hz();
613}
614
615// Get current playout frequency.
616int AudioCodingModuleImpl::PlayoutFrequency() const {
kwibergc13ded52016-06-17 06:00:45 -0700617 return receiver_.last_output_sample_rate_hz();
618}
619
kwiberg1c07c702017-03-27 07:15:49 -0700620void AudioCodingModuleImpl::SetReceiveCodecs(
621 const std::map<int, SdpAudioFormat>& codecs) {
622 rtc::CritScope lock(&acm_crit_sect_);
623 receiver_.SetCodecs(codecs);
624}
625
Fredrik Solenbergf693bfa2018-12-11 12:22:10 +0100626absl::optional<std::pair<int, SdpAudioFormat>>
Jonas Olssona4d87372019-07-05 19:08:33 +0200627AudioCodingModuleImpl::ReceiveCodec() const {
kwiberg5adaf732016-10-04 09:33:27 -0700628 rtc::CritScope lock(&acm_crit_sect_);
Fredrik Solenbergf693bfa2018-12-11 12:22:10 +0100629 return receiver_.LastDecoder();
ossue280cde2016-10-12 11:04:10 -0700630}
631
kwibergc13ded52016-06-17 06:00:45 -0700632// Incoming packet from network parsed and ready for decode.
633int AudioCodingModuleImpl::IncomingPacket(const uint8_t* incoming_payload,
634 const size_t payload_length,
Niels Möllerafb5dbb2019-02-15 15:21:47 +0100635 const RTPHeader& rtp_header) {
henrik.lundinb8c55b12017-05-10 07:38:01 -0700636 RTC_DCHECK_EQ(payload_length == 0, incoming_payload == nullptr);
kwibergc13ded52016-06-17 06:00:45 -0700637 return receiver_.InsertPacket(
638 rtp_header,
639 rtc::ArrayView<const uint8_t>(incoming_payload, payload_length));
640}
641
642// Minimum playout delay (Used for lip-sync).
643int AudioCodingModuleImpl::SetMinimumPlayoutDelay(int time_ms) {
644 if ((time_ms < 0) || (time_ms > 10000)) {
Mirko Bonadei675513b2017-11-09 11:09:25 +0100645 RTC_LOG(LS_ERROR) << "Delay must be in the range of 0-10000 milliseconds.";
kwibergc13ded52016-06-17 06:00:45 -0700646 return -1;
647 }
648 return receiver_.SetMinimumDelay(time_ms);
649}
650
651int AudioCodingModuleImpl::SetMaximumPlayoutDelay(int time_ms) {
652 if ((time_ms < 0) || (time_ms > 10000)) {
Mirko Bonadei675513b2017-11-09 11:09:25 +0100653 RTC_LOG(LS_ERROR) << "Delay must be in the range of 0-10000 milliseconds.";
kwibergc13ded52016-06-17 06:00:45 -0700654 return -1;
655 }
656 return receiver_.SetMaximumDelay(time_ms);
657}
658
Ruslan Burakov3b50f9f2019-02-06 09:45:56 +0100659bool AudioCodingModuleImpl::SetBaseMinimumPlayoutDelayMs(int delay_ms) {
660 // All necessary validation happens on NetEq level.
661 return receiver_.SetBaseMinimumDelayMs(delay_ms);
662}
663
664int AudioCodingModuleImpl::GetBaseMinimumPlayoutDelayMs() const {
665 return receiver_.GetBaseMinimumDelayMs();
666}
667
kwibergc13ded52016-06-17 06:00:45 -0700668// Get 10 milliseconds of raw audio data to play out.
669// Automatic resample to the requested frequency.
670int AudioCodingModuleImpl::PlayoutData10Ms(int desired_freq_hz,
671 AudioFrame* audio_frame,
672 bool* muted) {
673 // GetAudio always returns 10 ms, at the requested sample rate.
674 if (receiver_.GetAudio(desired_freq_hz, audio_frame, muted) != 0) {
Mirko Bonadei675513b2017-11-09 11:09:25 +0100675 RTC_LOG(LS_ERROR) << "PlayoutData failed, RecOut Failed";
kwibergc13ded52016-06-17 06:00:45 -0700676 return -1;
677 }
kwibergc13ded52016-06-17 06:00:45 -0700678 return 0;
679}
680
kwibergc13ded52016-06-17 06:00:45 -0700681/////////////////////////////////////////
682// Statistics
683//
684
685// TODO(turajs) change the return value to void. Also change the corresponding
686// NetEq function.
687int AudioCodingModuleImpl::GetNetworkStatistics(NetworkStatistics* statistics) {
688 receiver_.GetNetworkStatistics(statistics);
689 return 0;
690}
691
692int AudioCodingModuleImpl::RegisterVADCallback(ACMVADCallback* vad_callback) {
Mirko Bonadei675513b2017-11-09 11:09:25 +0100693 RTC_LOG(LS_VERBOSE) << "RegisterVADCallback()";
kwibergc13ded52016-06-17 06:00:45 -0700694 rtc::CritScope lock(&callback_crit_sect_);
695 vad_callback_ = vad_callback;
696 return 0;
697}
698
Danil Chapovalovb6021232018-06-19 13:26:36 +0200699absl::optional<uint32_t> AudioCodingModuleImpl::PlayoutTimestamp() {
kwibergc13ded52016-06-17 06:00:45 -0700700 return receiver_.GetPlayoutTimestamp();
701}
702
henrik.lundinb3f1c5d2016-08-22 15:39:53 -0700703int AudioCodingModuleImpl::FilteredCurrentDelayMs() const {
704 return receiver_.FilteredCurrentDelayMs();
705}
706
Henrik Lundinabbff892017-11-29 09:14:04 +0100707int AudioCodingModuleImpl::TargetDelayMs() const {
708 return receiver_.TargetDelayMs();
709}
710
kwibergc13ded52016-06-17 06:00:45 -0700711bool AudioCodingModuleImpl::HaveValidEncoder(const char* caller_name) const {
712 if (!encoder_stack_) {
Mirko Bonadei675513b2017-11-09 11:09:25 +0100713 RTC_LOG(LS_ERROR) << caller_name << " failed: No send codec is registered.";
kwibergc13ded52016-06-17 06:00:45 -0700714 return false;
715 }
716 return true;
717}
718
kwibergc13ded52016-06-17 06:00:45 -0700719void AudioCodingModuleImpl::GetDecodingCallStatistics(
Yves Gerey665174f2018-06-19 15:03:05 +0200720 AudioDecodingCallStats* call_stats) const {
kwibergc13ded52016-06-17 06:00:45 -0700721 receiver_.GetDecodingCallStatistics(call_stats);
722}
723
ivoce1198e02017-09-08 08:13:19 -0700724ANAStats AudioCodingModuleImpl::GetANAStats() const {
725 rtc::CritScope lock(&acm_crit_sect_);
726 if (encoder_stack_)
727 return encoder_stack_->GetANAStats();
728 // If no encoder is set, return default stats.
729 return ANAStats();
730}
731
kwibergc13ded52016-06-17 06:00:45 -0700732} // namespace
733
Karl Wiberg5817d3d2018-04-06 10:06:42 +0200734AudioCodingModule::Config::Config(
735 rtc::scoped_refptr<AudioDecoderFactory> decoder_factory)
736 : neteq_config(),
737 clock(Clock::GetRealTimeClock()),
738 decoder_factory(decoder_factory) {
kwiberg36a43882016-08-29 05:33:32 -0700739 // Post-decode VAD is disabled by default in NetEq, however, Audio
740 // Conference Mixer relies on VAD decisions and fails without them.
741 neteq_config.enable_post_decode_vad = true;
742}
743
744AudioCodingModule::Config::Config(const Config&) = default;
745AudioCodingModule::Config::~Config() = default;
746
Henrik Lundin64dad832015-05-11 12:44:23 +0200747AudioCodingModule* AudioCodingModule::Create(const Config& config) {
kwibergc13ded52016-06-17 06:00:45 -0700748 return new AudioCodingModuleImpl(config);
turaj@webrtc.org7959e162013-09-12 18:30:26 +0000749}
750
turaj@webrtc.org7959e162013-09-12 18:30:26 +0000751} // namespace webrtc