blob: 3f7a06a5fb9aef22cfc2ee3b90ded84d751c980e [file] [log] [blame]
turaj@webrtc.org7959e162013-09-12 18:30:26 +00001/*
2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020011#include "modules/audio_coding/include/audio_coding_module.h"
turaj@webrtc.org7959e162013-09-12 18:30:26 +000012
Yves Gerey988cc082018-10-23 12:03:01 +020013#include <assert.h>
Jonas Olssona4d87372019-07-05 19:08:33 +020014
Jonathan Yu36344a02017-07-30 01:55:34 -070015#include <algorithm>
Yves Gerey988cc082018-10-23 12:03:01 +020016#include <cstdint>
Jonathan Yu36344a02017-07-30 01:55:34 -070017
Niels Möller2edab4c2018-10-22 09:48:08 +020018#include "absl/strings/match.h"
Yves Gerey988cc082018-10-23 12:03:01 +020019#include "api/array_view.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020020#include "modules/audio_coding/acm2/acm_receiver.h"
21#include "modules/audio_coding/acm2/acm_resampler.h"
Fredrik Solenbergbbf21a32018-04-12 22:44:09 +020022#include "modules/include/module_common_types.h"
Yves Gerey988cc082018-10-23 12:03:01 +020023#include "modules/include/module_common_types_public.h"
24#include "rtc_base/buffer.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020025#include "rtc_base/checks.h"
Steve Anton10542f22019-01-11 09:11:00 -080026#include "rtc_base/critical_section.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020027#include "rtc_base/logging.h"
Karl Wiberge40468b2017-11-22 10:42:26 +010028#include "rtc_base/numerics/safe_conversions.h"
Yves Gerey988cc082018-10-23 12:03:01 +020029#include "rtc_base/thread_annotations.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020030#include "system_wrappers/include/metrics.h"
turaj@webrtc.org7959e162013-09-12 18:30:26 +000031
32namespace webrtc {
33
kwibergc13ded52016-06-17 06:00:45 -070034namespace {
35
kwibergc13ded52016-06-17 06:00:45 -070036class AudioCodingModuleImpl final : public AudioCodingModule {
37 public:
38 explicit AudioCodingModuleImpl(const AudioCodingModule::Config& config);
39 ~AudioCodingModuleImpl() override;
40
41 /////////////////////////////////////////
42 // Sender
43 //
44
kwiberg24c7c122016-09-28 11:57:10 -070045 void ModifyEncoder(rtc::FunctionView<void(std::unique_ptr<AudioEncoder>*)>
46 modifier) override;
kwibergc13ded52016-06-17 06:00:45 -070047
kwibergc13ded52016-06-17 06:00:45 -070048 // Register a transport callback which will be
49 // called to deliver the encoded buffers.
50 int RegisterTransportCallback(AudioPacketizationCallback* transport) override;
51
52 // Add 10 ms of raw (PCM) audio data to the encoder.
53 int Add10MsData(const AudioFrame& audio_frame) override;
54
55 /////////////////////////////////////////
kwibergc13ded52016-06-17 06:00:45 -070056 // (FEC) Forward Error Correction (codec internal)
57 //
58
kwibergc13ded52016-06-17 06:00:45 -070059 // Set target packet loss rate
60 int SetPacketLossRate(int loss_rate) override;
61
62 /////////////////////////////////////////
63 // (VAD) Voice Activity Detection
64 // and
65 // (CNG) Comfort Noise Generation
66 //
67
kwibergc13ded52016-06-17 06:00:45 -070068 int RegisterVADCallback(ACMVADCallback* vad_callback) override;
69
70 /////////////////////////////////////////
71 // Receiver
72 //
73
74 // Initialize receiver, resets codec database etc.
75 int InitializeReceiver() override;
76
kwiberg1c07c702017-03-27 07:15:49 -070077 void SetReceiveCodecs(const std::map<int, SdpAudioFormat>& codecs) override;
78
kwibergc13ded52016-06-17 06:00:45 -070079 // Incoming packet from network parsed and ready for decode.
80 int IncomingPacket(const uint8_t* incoming_payload,
81 const size_t payload_length,
Niels Möllerafb5dbb2019-02-15 15:21:47 +010082 const RTPHeader& rtp_info) override;
kwibergc13ded52016-06-17 06:00:45 -070083
kwibergc13ded52016-06-17 06:00:45 -070084 // Get 10 milliseconds of raw audio data to play out, and
85 // automatic resample to the requested frequency if > 0.
86 int PlayoutData10Ms(int desired_freq_hz,
87 AudioFrame* audio_frame,
88 bool* muted) override;
kwibergc13ded52016-06-17 06:00:45 -070089
90 /////////////////////////////////////////
91 // Statistics
92 //
93
94 int GetNetworkStatistics(NetworkStatistics* statistics) override;
95
ivoce1198e02017-09-08 08:13:19 -070096 ANAStats GetANAStats() const override;
97
kwibergc13ded52016-06-17 06:00:45 -070098 private:
99 struct InputData {
100 uint32_t input_timestamp;
101 const int16_t* audio;
102 size_t length_per_channel;
103 size_t audio_channel;
104 // If a re-mix is required (up or down), this buffer will store a re-mixed
105 // version of the input.
106 int16_t buffer[WEBRTC_10MS_PCM_AUDIO];
107 };
108
109 // This member class writes values to the named UMA histogram, but only if
110 // the value has changed since the last time (and always for the first call).
111 class ChangeLogger {
112 public:
113 explicit ChangeLogger(const std::string& histogram_name)
114 : histogram_name_(histogram_name) {}
115 // Logs the new value if it is different from the last logged value, or if
116 // this is the first call.
117 void MaybeLog(int value);
118
119 private:
120 int last_value_ = 0;
121 int first_time_ = true;
122 const std::string histogram_name_;
123 };
124
kwibergc13ded52016-06-17 06:00:45 -0700125 int Add10MsDataInternal(const AudioFrame& audio_frame, InputData* input_data)
danilchap56359be2017-09-07 07:53:45 -0700126 RTC_EXCLUSIVE_LOCKS_REQUIRED(acm_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700127 int Encode(const InputData& input_data)
danilchap56359be2017-09-07 07:53:45 -0700128 RTC_EXCLUSIVE_LOCKS_REQUIRED(acm_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700129
danilchap56359be2017-09-07 07:53:45 -0700130 int InitializeReceiverSafe() RTC_EXCLUSIVE_LOCKS_REQUIRED(acm_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700131
132 bool HaveValidEncoder(const char* caller_name) const
danilchap56359be2017-09-07 07:53:45 -0700133 RTC_EXCLUSIVE_LOCKS_REQUIRED(acm_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700134
135 // Preprocessing of input audio, including resampling and down-mixing if
136 // required, before pushing audio into encoder's buffer.
137 //
138 // in_frame: input audio-frame
139 // ptr_out: pointer to output audio_frame. If no preprocessing is required
140 // |ptr_out| will be pointing to |in_frame|, otherwise pointing to
141 // |preprocess_frame_|.
142 //
143 // Return value:
144 // -1: if encountering an error.
145 // 0: otherwise.
146 int PreprocessToAddData(const AudioFrame& in_frame,
147 const AudioFrame** ptr_out)
danilchap56359be2017-09-07 07:53:45 -0700148 RTC_EXCLUSIVE_LOCKS_REQUIRED(acm_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700149
150 // Change required states after starting to receive the codec corresponding
151 // to |index|.
152 int UpdateUponReceivingCodec(int index);
153
154 rtc::CriticalSection acm_crit_sect_;
danilchap56359be2017-09-07 07:53:45 -0700155 rtc::Buffer encode_buffer_ RTC_GUARDED_BY(acm_crit_sect_);
danilchap56359be2017-09-07 07:53:45 -0700156 uint32_t expected_codec_ts_ RTC_GUARDED_BY(acm_crit_sect_);
157 uint32_t expected_in_ts_ RTC_GUARDED_BY(acm_crit_sect_);
158 acm2::ACMResampler resampler_ RTC_GUARDED_BY(acm_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700159 acm2::AcmReceiver receiver_; // AcmReceiver has it's own internal lock.
danilchap56359be2017-09-07 07:53:45 -0700160 ChangeLogger bitrate_logger_ RTC_GUARDED_BY(acm_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700161
Karl Wiberg49c33ce2018-11-12 14:21:58 +0100162 // Current encoder stack, provided by a call to RegisterEncoder.
danilchap56359be2017-09-07 07:53:45 -0700163 std::unique_ptr<AudioEncoder> encoder_stack_ RTC_GUARDED_BY(acm_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700164
kwibergc13ded52016-06-17 06:00:45 -0700165 // This is to keep track of CN instances where we can send DTMFs.
danilchap56359be2017-09-07 07:53:45 -0700166 uint8_t previous_pltype_ RTC_GUARDED_BY(acm_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700167
danilchap56359be2017-09-07 07:53:45 -0700168 bool receiver_initialized_ RTC_GUARDED_BY(acm_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700169
danilchap56359be2017-09-07 07:53:45 -0700170 AudioFrame preprocess_frame_ RTC_GUARDED_BY(acm_crit_sect_);
171 bool first_10ms_data_ RTC_GUARDED_BY(acm_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700172
danilchap56359be2017-09-07 07:53:45 -0700173 bool first_frame_ RTC_GUARDED_BY(acm_crit_sect_);
174 uint32_t last_timestamp_ RTC_GUARDED_BY(acm_crit_sect_);
175 uint32_t last_rtp_timestamp_ RTC_GUARDED_BY(acm_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700176
177 rtc::CriticalSection callback_crit_sect_;
178 AudioPacketizationCallback* packetization_callback_
danilchap56359be2017-09-07 07:53:45 -0700179 RTC_GUARDED_BY(callback_crit_sect_);
180 ACMVADCallback* vad_callback_ RTC_GUARDED_BY(callback_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700181
182 int codec_histogram_bins_log_[static_cast<size_t>(
183 AudioEncoder::CodecType::kMaxLoggedAudioCodecTypes)];
184 int number_of_consecutive_empty_packets_;
185};
186
187// Adds a codec usage sample to the histogram.
188void UpdateCodecTypeHistogram(size_t codec_type) {
189 RTC_HISTOGRAM_ENUMERATION(
190 "WebRTC.Audio.Encoder.CodecType", static_cast<int>(codec_type),
191 static_cast<int>(
192 webrtc::AudioEncoder::CodecType::kMaxLoggedAudioCodecTypes));
193}
194
kwibergc13ded52016-06-17 06:00:45 -0700195// Stereo-to-mono can be used as in-place.
196int DownMix(const AudioFrame& frame,
197 size_t length_out_buff,
198 int16_t* out_buff) {
yujo36b1a5f2017-06-12 12:45:32 -0700199 RTC_DCHECK_EQ(frame.num_channels_, 2);
200 RTC_DCHECK_GE(length_out_buff, frame.samples_per_channel_);
201
202 if (!frame.muted()) {
203 const int16_t* frame_data = frame.data();
204 for (size_t n = 0; n < frame.samples_per_channel_; ++n) {
Yves Gerey665174f2018-06-19 15:03:05 +0200205 out_buff[n] =
206 static_cast<int16_t>((static_cast<int32_t>(frame_data[2 * n]) +
207 static_cast<int32_t>(frame_data[2 * n + 1])) >>
208 1);
yujo36b1a5f2017-06-12 12:45:32 -0700209 }
210 } else {
Jonathan Yu36344a02017-07-30 01:55:34 -0700211 std::fill(out_buff, out_buff + frame.samples_per_channel_, 0);
kwibergc13ded52016-06-17 06:00:45 -0700212 }
kwibergc13ded52016-06-17 06:00:45 -0700213 return 0;
214}
215
216// Mono-to-stereo can be used as in-place.
217int UpMix(const AudioFrame& frame, size_t length_out_buff, int16_t* out_buff) {
yujo36b1a5f2017-06-12 12:45:32 -0700218 RTC_DCHECK_EQ(frame.num_channels_, 1);
219 RTC_DCHECK_GE(length_out_buff, 2 * frame.samples_per_channel_);
220
221 if (!frame.muted()) {
222 const int16_t* frame_data = frame.data();
223 for (size_t n = frame.samples_per_channel_; n != 0; --n) {
224 size_t i = n - 1;
225 int16_t sample = frame_data[i];
226 out_buff[2 * i + 1] = sample;
227 out_buff[2 * i] = sample;
228 }
229 } else {
Jonathan Yu36344a02017-07-30 01:55:34 -0700230 std::fill(out_buff, out_buff + frame.samples_per_channel_ * 2, 0);
kwibergc13ded52016-06-17 06:00:45 -0700231 }
232 return 0;
233}
234
kwibergc13ded52016-06-17 06:00:45 -0700235void AudioCodingModuleImpl::ChangeLogger::MaybeLog(int value) {
236 if (value != last_value_ || first_time_) {
237 first_time_ = false;
238 last_value_ = value;
239 RTC_HISTOGRAM_COUNTS_SPARSE_100(histogram_name_, value);
240 }
241}
242
243AudioCodingModuleImpl::AudioCodingModuleImpl(
244 const AudioCodingModule::Config& config)
solenbergc7b4a452017-09-28 07:37:11 -0700245 : expected_codec_ts_(0xD87F3F9F),
kwibergc13ded52016-06-17 06:00:45 -0700246 expected_in_ts_(0xD87F3F9F),
247 receiver_(config),
248 bitrate_logger_("WebRTC.Audio.TargetBitrateInKbps"),
kwibergc13ded52016-06-17 06:00:45 -0700249 encoder_stack_(nullptr),
250 previous_pltype_(255),
251 receiver_initialized_(false),
252 first_10ms_data_(false),
253 first_frame_(true),
254 packetization_callback_(NULL),
255 vad_callback_(NULL),
256 codec_histogram_bins_log_(),
257 number_of_consecutive_empty_packets_(0) {
258 if (InitializeReceiverSafe() < 0) {
Mirko Bonadei675513b2017-11-09 11:09:25 +0100259 RTC_LOG(LS_ERROR) << "Cannot initialize receiver";
kwibergc13ded52016-06-17 06:00:45 -0700260 }
Mirko Bonadei675513b2017-11-09 11:09:25 +0100261 RTC_LOG(LS_INFO) << "Created";
kwibergc13ded52016-06-17 06:00:45 -0700262}
263
264AudioCodingModuleImpl::~AudioCodingModuleImpl() = default;
265
266int32_t AudioCodingModuleImpl::Encode(const InputData& input_data) {
267 AudioEncoder::EncodedInfo encoded_info;
268 uint8_t previous_pltype;
269
270 // Check if there is an encoder before.
271 if (!HaveValidEncoder("Process"))
272 return -1;
273
Yves Gerey665174f2018-06-19 15:03:05 +0200274 if (!first_frame_) {
deadbeeffcada902016-08-24 12:45:13 -0700275 RTC_DCHECK(IsNewerTimestamp(input_data.input_timestamp, last_timestamp_))
ossu63fb95a2016-07-06 09:34:22 -0700276 << "Time should not move backwards";
277 }
278
kwibergc13ded52016-06-17 06:00:45 -0700279 // Scale the timestamp to the codec's RTP timestamp rate.
280 uint32_t rtp_timestamp =
Karl Wiberg053c3712019-05-16 15:24:17 +0200281 first_frame_
282 ? input_data.input_timestamp
283 : last_rtp_timestamp_ +
284 rtc::dchecked_cast<uint32_t>(rtc::CheckedDivExact(
285 int64_t{input_data.input_timestamp - last_timestamp_} *
286 encoder_stack_->RtpTimestampRateHz(),
287 int64_t{encoder_stack_->SampleRateHz()}));
kwibergc13ded52016-06-17 06:00:45 -0700288 last_timestamp_ = input_data.input_timestamp;
289 last_rtp_timestamp_ = rtp_timestamp;
290 first_frame_ = false;
291
292 // Clear the buffer before reuse - encoded data will get appended.
293 encode_buffer_.Clear();
294 encoded_info = encoder_stack_->Encode(
Yves Gerey665174f2018-06-19 15:03:05 +0200295 rtp_timestamp,
296 rtc::ArrayView<const int16_t>(
297 input_data.audio,
298 input_data.audio_channel * input_data.length_per_channel),
kwibergc13ded52016-06-17 06:00:45 -0700299 &encode_buffer_);
300
301 bitrate_logger_.MaybeLog(encoder_stack_->GetTargetBitrate() / 1000);
302 if (encode_buffer_.size() == 0 && !encoded_info.send_even_if_empty) {
303 // Not enough data.
304 return 0;
305 }
306 previous_pltype = previous_pltype_; // Read it while we have the critsect.
307
308 // Log codec type to histogram once every 500 packets.
309 if (encoded_info.encoded_bytes == 0) {
310 ++number_of_consecutive_empty_packets_;
311 } else {
312 size_t codec_type = static_cast<size_t>(encoded_info.encoder_type);
313 codec_histogram_bins_log_[codec_type] +=
314 number_of_consecutive_empty_packets_ + 1;
315 number_of_consecutive_empty_packets_ = 0;
316 if (codec_histogram_bins_log_[codec_type] >= 500) {
317 codec_histogram_bins_log_[codec_type] -= 500;
318 UpdateCodecTypeHistogram(codec_type);
319 }
320 }
321
Niels Möller87e2d782019-03-07 10:18:23 +0100322 AudioFrameType frame_type;
kwibergc13ded52016-06-17 06:00:45 -0700323 if (encode_buffer_.size() == 0 && encoded_info.send_even_if_empty) {
Niels Möllerc936cb62019-03-19 14:10:16 +0100324 frame_type = AudioFrameType::kEmptyFrame;
kwibergc13ded52016-06-17 06:00:45 -0700325 encoded_info.payload_type = previous_pltype;
326 } else {
kwibergaf476c72016-11-28 15:21:39 -0800327 RTC_DCHECK_GT(encode_buffer_.size(), 0);
Niels Möllerc936cb62019-03-19 14:10:16 +0100328 frame_type = encoded_info.speech ? AudioFrameType::kAudioFrameSpeech
329 : AudioFrameType::kAudioFrameCN;
kwibergc13ded52016-06-17 06:00:45 -0700330 }
331
332 {
333 rtc::CritScope lock(&callback_crit_sect_);
334 if (packetization_callback_) {
335 packetization_callback_->SendData(
336 frame_type, encoded_info.payload_type, encoded_info.encoded_timestamp,
Niels Möllerc35b6e62019-04-25 16:31:18 +0200337 encode_buffer_.data(), encode_buffer_.size());
kwibergc13ded52016-06-17 06:00:45 -0700338 }
339
340 if (vad_callback_) {
341 // Callback with VAD decision.
342 vad_callback_->InFrameType(frame_type);
343 }
344 }
345 previous_pltype_ = encoded_info.payload_type;
346 return static_cast<int32_t>(encode_buffer_.size());
347}
348
349/////////////////////////////////////////
350// Sender
351//
352
kwibergc13ded52016-06-17 06:00:45 -0700353void AudioCodingModuleImpl::ModifyEncoder(
kwiberg24c7c122016-09-28 11:57:10 -0700354 rtc::FunctionView<void(std::unique_ptr<AudioEncoder>*)> modifier) {
kwibergc13ded52016-06-17 06:00:45 -0700355 rtc::CritScope lock(&acm_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700356 modifier(&encoder_stack_);
357}
358
kwibergc13ded52016-06-17 06:00:45 -0700359// Register a transport callback which will be called to deliver
360// the encoded buffers.
361int AudioCodingModuleImpl::RegisterTransportCallback(
362 AudioPacketizationCallback* transport) {
363 rtc::CritScope lock(&callback_crit_sect_);
364 packetization_callback_ = transport;
365 return 0;
366}
367
368// Add 10MS of raw (PCM) audio data to the encoder.
369int AudioCodingModuleImpl::Add10MsData(const AudioFrame& audio_frame) {
370 InputData input_data;
371 rtc::CritScope lock(&acm_crit_sect_);
372 int r = Add10MsDataInternal(audio_frame, &input_data);
373 return r < 0 ? r : Encode(input_data);
374}
375
376int AudioCodingModuleImpl::Add10MsDataInternal(const AudioFrame& audio_frame,
377 InputData* input_data) {
378 if (audio_frame.samples_per_channel_ == 0) {
379 assert(false);
Mirko Bonadei675513b2017-11-09 11:09:25 +0100380 RTC_LOG(LS_ERROR) << "Cannot Add 10 ms audio, payload length is zero";
kwibergc13ded52016-06-17 06:00:45 -0700381 return -1;
382 }
383
henrika33541572019-09-10 14:27:40 +0200384 if (audio_frame.sample_rate_hz_ > 192000) {
kwibergc13ded52016-06-17 06:00:45 -0700385 assert(false);
Mirko Bonadei675513b2017-11-09 11:09:25 +0100386 RTC_LOG(LS_ERROR) << "Cannot Add 10 ms audio, input frequency not valid";
kwibergc13ded52016-06-17 06:00:45 -0700387 return -1;
388 }
389
390 // If the length and frequency matches. We currently just support raw PCM.
391 if (static_cast<size_t>(audio_frame.sample_rate_hz_ / 100) !=
392 audio_frame.samples_per_channel_) {
Mirko Bonadei675513b2017-11-09 11:09:25 +0100393 RTC_LOG(LS_ERROR)
Alex Loiko300ec8c2017-05-30 17:23:28 +0200394 << "Cannot Add 10 ms audio, input frequency and length doesn't match";
kwibergc13ded52016-06-17 06:00:45 -0700395 return -1;
396 }
397
Alex Loiko65438812019-02-22 10:13:44 +0100398 if (audio_frame.num_channels_ != 1 && audio_frame.num_channels_ != 2 &&
399 audio_frame.num_channels_ != 4 && audio_frame.num_channels_ != 6 &&
400 audio_frame.num_channels_ != 8) {
Mirko Bonadei675513b2017-11-09 11:09:25 +0100401 RTC_LOG(LS_ERROR) << "Cannot Add 10 ms audio, invalid number of channels.";
kwibergc13ded52016-06-17 06:00:45 -0700402 return -1;
403 }
404
405 // Do we have a codec registered?
406 if (!HaveValidEncoder("Add10MsData")) {
407 return -1;
408 }
409
410 const AudioFrame* ptr_frame;
411 // Perform a resampling, also down-mix if it is required and can be
412 // performed before resampling (a down mix prior to resampling will take
413 // place if both primary and secondary encoders are mono and input is in
414 // stereo).
415 if (PreprocessToAddData(audio_frame, &ptr_frame) < 0) {
416 return -1;
417 }
418
419 // Check whether we need an up-mix or down-mix?
420 const size_t current_num_channels = encoder_stack_->NumChannels();
421 const bool same_num_channels =
422 ptr_frame->num_channels_ == current_num_channels;
423
424 if (!same_num_channels) {
425 if (ptr_frame->num_channels_ == 1) {
426 if (UpMix(*ptr_frame, WEBRTC_10MS_PCM_AUDIO, input_data->buffer) < 0)
427 return -1;
428 } else {
429 if (DownMix(*ptr_frame, WEBRTC_10MS_PCM_AUDIO, input_data->buffer) < 0)
430 return -1;
431 }
432 }
433
434 // When adding data to encoders this pointer is pointing to an audio buffer
435 // with correct number of channels.
yujo36b1a5f2017-06-12 12:45:32 -0700436 const int16_t* ptr_audio = ptr_frame->data();
kwibergc13ded52016-06-17 06:00:45 -0700437
438 // For pushing data to primary, point the |ptr_audio| to correct buffer.
439 if (!same_num_channels)
440 ptr_audio = input_data->buffer;
441
yujo36b1a5f2017-06-12 12:45:32 -0700442 // TODO(yujo): Skip encode of muted frames.
kwibergc13ded52016-06-17 06:00:45 -0700443 input_data->input_timestamp = ptr_frame->timestamp_;
444 input_data->audio = ptr_audio;
445 input_data->length_per_channel = ptr_frame->samples_per_channel_;
446 input_data->audio_channel = current_num_channels;
447
448 return 0;
449}
450
451// Perform a resampling and down-mix if required. We down-mix only if
452// encoder is mono and input is stereo. In case of dual-streaming, both
453// encoders has to be mono for down-mix to take place.
454// |*ptr_out| will point to the pre-processed audio-frame. If no pre-processing
455// is required, |*ptr_out| points to |in_frame|.
yujo36b1a5f2017-06-12 12:45:32 -0700456// TODO(yujo): Make this more efficient for muted frames.
kwibergc13ded52016-06-17 06:00:45 -0700457int AudioCodingModuleImpl::PreprocessToAddData(const AudioFrame& in_frame,
458 const AudioFrame** ptr_out) {
459 const bool resample =
460 in_frame.sample_rate_hz_ != encoder_stack_->SampleRateHz();
461
462 // This variable is true if primary codec and secondary codec (if exists)
463 // are both mono and input is stereo.
464 // TODO(henrik.lundin): This condition should probably be
465 // in_frame.num_channels_ > encoder_stack_->NumChannels()
466 const bool down_mix =
467 in_frame.num_channels_ == 2 && encoder_stack_->NumChannels() == 1;
468
469 if (!first_10ms_data_) {
470 expected_in_ts_ = in_frame.timestamp_;
471 expected_codec_ts_ = in_frame.timestamp_;
472 first_10ms_data_ = true;
473 } else if (in_frame.timestamp_ != expected_in_ts_) {
Mirko Bonadei675513b2017-11-09 11:09:25 +0100474 RTC_LOG(LS_WARNING) << "Unexpected input timestamp: " << in_frame.timestamp_
475 << ", expected: " << expected_in_ts_;
kwibergc13ded52016-06-17 06:00:45 -0700476 expected_codec_ts_ +=
477 (in_frame.timestamp_ - expected_in_ts_) *
478 static_cast<uint32_t>(
479 static_cast<double>(encoder_stack_->SampleRateHz()) /
480 static_cast<double>(in_frame.sample_rate_hz_));
481 expected_in_ts_ = in_frame.timestamp_;
482 }
483
kwibergc13ded52016-06-17 06:00:45 -0700484 if (!down_mix && !resample) {
485 // No pre-processing is required.
ossu63fb95a2016-07-06 09:34:22 -0700486 if (expected_in_ts_ == expected_codec_ts_) {
487 // If we've never resampled, we can use the input frame as-is
488 *ptr_out = &in_frame;
489 } else {
490 // Otherwise we'll need to alter the timestamp. Since in_frame is const,
491 // we'll have to make a copy of it.
492 preprocess_frame_.CopyFrom(in_frame);
493 preprocess_frame_.timestamp_ = expected_codec_ts_;
494 *ptr_out = &preprocess_frame_;
495 }
496
kwibergc13ded52016-06-17 06:00:45 -0700497 expected_in_ts_ += static_cast<uint32_t>(in_frame.samples_per_channel_);
498 expected_codec_ts_ += static_cast<uint32_t>(in_frame.samples_per_channel_);
kwibergc13ded52016-06-17 06:00:45 -0700499 return 0;
500 }
501
502 *ptr_out = &preprocess_frame_;
503 preprocess_frame_.num_channels_ = in_frame.num_channels_;
504 int16_t audio[WEBRTC_10MS_PCM_AUDIO];
yujo36b1a5f2017-06-12 12:45:32 -0700505 const int16_t* src_ptr_audio = in_frame.data();
kwibergc13ded52016-06-17 06:00:45 -0700506 if (down_mix) {
507 // If a resampling is required the output of a down-mix is written into a
508 // local buffer, otherwise, it will be written to the output frame.
Yves Gerey665174f2018-06-19 15:03:05 +0200509 int16_t* dest_ptr_audio =
510 resample ? audio : preprocess_frame_.mutable_data();
kwibergc13ded52016-06-17 06:00:45 -0700511 if (DownMix(in_frame, WEBRTC_10MS_PCM_AUDIO, dest_ptr_audio) < 0)
512 return -1;
513 preprocess_frame_.num_channels_ = 1;
514 // Set the input of the resampler is the down-mixed signal.
515 src_ptr_audio = audio;
516 }
517
518 preprocess_frame_.timestamp_ = expected_codec_ts_;
519 preprocess_frame_.samples_per_channel_ = in_frame.samples_per_channel_;
520 preprocess_frame_.sample_rate_hz_ = in_frame.sample_rate_hz_;
521 // If it is required, we have to do a resampling.
522 if (resample) {
523 // The result of the resampler is written to output frame.
yujo36b1a5f2017-06-12 12:45:32 -0700524 int16_t* dest_ptr_audio = preprocess_frame_.mutable_data();
kwibergc13ded52016-06-17 06:00:45 -0700525
526 int samples_per_channel = resampler_.Resample10Msec(
527 src_ptr_audio, in_frame.sample_rate_hz_, encoder_stack_->SampleRateHz(),
528 preprocess_frame_.num_channels_, AudioFrame::kMaxDataSizeSamples,
529 dest_ptr_audio);
530
531 if (samples_per_channel < 0) {
Mirko Bonadei675513b2017-11-09 11:09:25 +0100532 RTC_LOG(LS_ERROR) << "Cannot add 10 ms audio, resampling failed";
kwibergc13ded52016-06-17 06:00:45 -0700533 return -1;
534 }
535 preprocess_frame_.samples_per_channel_ =
536 static_cast<size_t>(samples_per_channel);
537 preprocess_frame_.sample_rate_hz_ = encoder_stack_->SampleRateHz();
538 }
539
540 expected_codec_ts_ +=
541 static_cast<uint32_t>(preprocess_frame_.samples_per_channel_);
542 expected_in_ts_ += static_cast<uint32_t>(in_frame.samples_per_channel_);
543
544 return 0;
545}
546
547/////////////////////////////////////////
kwibergc13ded52016-06-17 06:00:45 -0700548// (FEC) Forward Error Correction (codec internal)
549//
550
kwibergc13ded52016-06-17 06:00:45 -0700551int AudioCodingModuleImpl::SetPacketLossRate(int loss_rate) {
552 rtc::CritScope lock(&acm_crit_sect_);
553 if (HaveValidEncoder("SetPacketLossRate")) {
minyue4b9a2cb2016-11-30 06:49:59 -0800554 encoder_stack_->OnReceivedUplinkPacketLossFraction(loss_rate / 100.0);
kwibergc13ded52016-06-17 06:00:45 -0700555 }
556 return 0;
557}
558
559/////////////////////////////////////////
kwibergc13ded52016-06-17 06:00:45 -0700560// Receiver
561//
562
563int AudioCodingModuleImpl::InitializeReceiver() {
564 rtc::CritScope lock(&acm_crit_sect_);
565 return InitializeReceiverSafe();
566}
567
568// Initialize receiver, resets codec database etc.
569int AudioCodingModuleImpl::InitializeReceiverSafe() {
570 // If the receiver is already initialized then we want to destroy any
571 // existing decoders. After a call to this function, we should have a clean
572 // start-up.
kwiberg6b19b562016-09-20 04:02:25 -0700573 if (receiver_initialized_)
574 receiver_.RemoveAllCodecs();
kwibergc13ded52016-06-17 06:00:45 -0700575 receiver_.FlushBuffers();
576
kwibergc13ded52016-06-17 06:00:45 -0700577 receiver_initialized_ = true;
578 return 0;
579}
580
kwiberg1c07c702017-03-27 07:15:49 -0700581void AudioCodingModuleImpl::SetReceiveCodecs(
582 const std::map<int, SdpAudioFormat>& codecs) {
583 rtc::CritScope lock(&acm_crit_sect_);
584 receiver_.SetCodecs(codecs);
585}
586
kwibergc13ded52016-06-17 06:00:45 -0700587// Incoming packet from network parsed and ready for decode.
588int AudioCodingModuleImpl::IncomingPacket(const uint8_t* incoming_payload,
589 const size_t payload_length,
Niels Möllerafb5dbb2019-02-15 15:21:47 +0100590 const RTPHeader& rtp_header) {
henrik.lundinb8c55b12017-05-10 07:38:01 -0700591 RTC_DCHECK_EQ(payload_length == 0, incoming_payload == nullptr);
kwibergc13ded52016-06-17 06:00:45 -0700592 return receiver_.InsertPacket(
593 rtp_header,
594 rtc::ArrayView<const uint8_t>(incoming_payload, payload_length));
595}
596
kwibergc13ded52016-06-17 06:00:45 -0700597// Get 10 milliseconds of raw audio data to play out.
598// Automatic resample to the requested frequency.
599int AudioCodingModuleImpl::PlayoutData10Ms(int desired_freq_hz,
600 AudioFrame* audio_frame,
601 bool* muted) {
602 // GetAudio always returns 10 ms, at the requested sample rate.
603 if (receiver_.GetAudio(desired_freq_hz, audio_frame, muted) != 0) {
Mirko Bonadei675513b2017-11-09 11:09:25 +0100604 RTC_LOG(LS_ERROR) << "PlayoutData failed, RecOut Failed";
kwibergc13ded52016-06-17 06:00:45 -0700605 return -1;
606 }
kwibergc13ded52016-06-17 06:00:45 -0700607 return 0;
608}
609
kwibergc13ded52016-06-17 06:00:45 -0700610/////////////////////////////////////////
611// Statistics
612//
613
614// TODO(turajs) change the return value to void. Also change the corresponding
615// NetEq function.
616int AudioCodingModuleImpl::GetNetworkStatistics(NetworkStatistics* statistics) {
617 receiver_.GetNetworkStatistics(statistics);
618 return 0;
619}
620
621int AudioCodingModuleImpl::RegisterVADCallback(ACMVADCallback* vad_callback) {
Mirko Bonadei675513b2017-11-09 11:09:25 +0100622 RTC_LOG(LS_VERBOSE) << "RegisterVADCallback()";
kwibergc13ded52016-06-17 06:00:45 -0700623 rtc::CritScope lock(&callback_crit_sect_);
624 vad_callback_ = vad_callback;
625 return 0;
626}
627
kwibergc13ded52016-06-17 06:00:45 -0700628bool AudioCodingModuleImpl::HaveValidEncoder(const char* caller_name) const {
629 if (!encoder_stack_) {
Mirko Bonadei675513b2017-11-09 11:09:25 +0100630 RTC_LOG(LS_ERROR) << caller_name << " failed: No send codec is registered.";
kwibergc13ded52016-06-17 06:00:45 -0700631 return false;
632 }
633 return true;
634}
635
ivoce1198e02017-09-08 08:13:19 -0700636ANAStats AudioCodingModuleImpl::GetANAStats() const {
637 rtc::CritScope lock(&acm_crit_sect_);
638 if (encoder_stack_)
639 return encoder_stack_->GetANAStats();
640 // If no encoder is set, return default stats.
641 return ANAStats();
642}
643
kwibergc13ded52016-06-17 06:00:45 -0700644} // namespace
645
Karl Wiberg5817d3d2018-04-06 10:06:42 +0200646AudioCodingModule::Config::Config(
647 rtc::scoped_refptr<AudioDecoderFactory> decoder_factory)
648 : neteq_config(),
649 clock(Clock::GetRealTimeClock()),
650 decoder_factory(decoder_factory) {
kwiberg36a43882016-08-29 05:33:32 -0700651 // Post-decode VAD is disabled by default in NetEq, however, Audio
652 // Conference Mixer relies on VAD decisions and fails without them.
653 neteq_config.enable_post_decode_vad = true;
654}
655
656AudioCodingModule::Config::Config(const Config&) = default;
657AudioCodingModule::Config::~Config() = default;
658
Henrik Lundin64dad832015-05-11 12:44:23 +0200659AudioCodingModule* AudioCodingModule::Create(const Config& config) {
kwibergc13ded52016-06-17 06:00:45 -0700660 return new AudioCodingModuleImpl(config);
turaj@webrtc.org7959e162013-09-12 18:30:26 +0000661}
662
turaj@webrtc.org7959e162013-09-12 18:30:26 +0000663} // namespace webrtc