blob: e28be186f56b4c475674fdfb8d39dce4e2982440 [file] [log] [blame]
turaj@webrtc.org7959e162013-09-12 18:30:26 +00001/*
2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020011#include "modules/audio_coding/include/audio_coding_module.h"
turaj@webrtc.org7959e162013-09-12 18:30:26 +000012
Yves Gerey988cc082018-10-23 12:03:01 +020013#include <assert.h>
Jonathan Yu36344a02017-07-30 01:55:34 -070014#include <algorithm>
Yves Gerey988cc082018-10-23 12:03:01 +020015#include <cstdint>
Jonathan Yu36344a02017-07-30 01:55:34 -070016
Niels Möller2edab4c2018-10-22 09:48:08 +020017#include "absl/strings/match.h"
Yves Gerey988cc082018-10-23 12:03:01 +020018#include "api/array_view.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020019#include "modules/audio_coding/acm2/acm_receiver.h"
Per Åhgren4dd56a32019-11-19 21:00:59 +010020#include "modules/audio_coding/acm2/acm_remixing.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020021#include "modules/audio_coding/acm2/acm_resampler.h"
Fredrik Solenbergbbf21a32018-04-12 22:44:09 +020022#include "modules/include/module_common_types.h"
Yves Gerey988cc082018-10-23 12:03:01 +020023#include "modules/include/module_common_types_public.h"
24#include "rtc_base/buffer.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020025#include "rtc_base/checks.h"
Steve Anton10542f22019-01-11 09:11:00 -080026#include "rtc_base/critical_section.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020027#include "rtc_base/logging.h"
Karl Wiberge40468b2017-11-22 10:42:26 +010028#include "rtc_base/numerics/safe_conversions.h"
Yves Gerey988cc082018-10-23 12:03:01 +020029#include "rtc_base/thread_annotations.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020030#include "system_wrappers/include/metrics.h"
turaj@webrtc.org7959e162013-09-12 18:30:26 +000031
32namespace webrtc {
33
kwibergc13ded52016-06-17 06:00:45 -070034namespace {
35
Per Åhgren4f2e9402019-10-04 11:06:15 +020036// Initial size for the buffer in InputBuffer. This matches 6 channels of 10 ms
37// 48 kHz data.
38constexpr size_t kInitialInputDataBufferSize = 6 * 480;
39
kwibergc13ded52016-06-17 06:00:45 -070040class AudioCodingModuleImpl final : public AudioCodingModule {
41 public:
42 explicit AudioCodingModuleImpl(const AudioCodingModule::Config& config);
43 ~AudioCodingModuleImpl() override;
44
45 /////////////////////////////////////////
46 // Sender
47 //
48
kwiberg24c7c122016-09-28 11:57:10 -070049 void ModifyEncoder(rtc::FunctionView<void(std::unique_ptr<AudioEncoder>*)>
50 modifier) override;
kwibergc13ded52016-06-17 06:00:45 -070051
kwibergc13ded52016-06-17 06:00:45 -070052 // Register a transport callback which will be
53 // called to deliver the encoded buffers.
54 int RegisterTransportCallback(AudioPacketizationCallback* transport) override;
55
56 // Add 10 ms of raw (PCM) audio data to the encoder.
57 int Add10MsData(const AudioFrame& audio_frame) override;
58
59 /////////////////////////////////////////
kwibergc13ded52016-06-17 06:00:45 -070060 // (FEC) Forward Error Correction (codec internal)
61 //
62
kwibergc13ded52016-06-17 06:00:45 -070063 // Set target packet loss rate
64 int SetPacketLossRate(int loss_rate) override;
65
66 /////////////////////////////////////////
67 // (VAD) Voice Activity Detection
68 // and
69 // (CNG) Comfort Noise Generation
70 //
71
kwibergc13ded52016-06-17 06:00:45 -070072 int RegisterVADCallback(ACMVADCallback* vad_callback) override;
73
74 /////////////////////////////////////////
75 // Receiver
76 //
77
78 // Initialize receiver, resets codec database etc.
79 int InitializeReceiver() override;
80
kwiberg1c07c702017-03-27 07:15:49 -070081 void SetReceiveCodecs(const std::map<int, SdpAudioFormat>& codecs) override;
82
kwibergc13ded52016-06-17 06:00:45 -070083 // Incoming packet from network parsed and ready for decode.
84 int IncomingPacket(const uint8_t* incoming_payload,
85 const size_t payload_length,
Niels Möllerafb5dbb2019-02-15 15:21:47 +010086 const RTPHeader& rtp_info) override;
kwibergc13ded52016-06-17 06:00:45 -070087
kwibergc13ded52016-06-17 06:00:45 -070088 // Get 10 milliseconds of raw audio data to play out, and
89 // automatic resample to the requested frequency if > 0.
90 int PlayoutData10Ms(int desired_freq_hz,
91 AudioFrame* audio_frame,
92 bool* muted) override;
kwibergc13ded52016-06-17 06:00:45 -070093
94 /////////////////////////////////////////
95 // Statistics
96 //
97
98 int GetNetworkStatistics(NetworkStatistics* statistics) override;
99
ivoce1198e02017-09-08 08:13:19 -0700100 ANAStats GetANAStats() const override;
101
kwibergc13ded52016-06-17 06:00:45 -0700102 private:
103 struct InputData {
Per Åhgren4f2e9402019-10-04 11:06:15 +0200104 InputData() : buffer(kInitialInputDataBufferSize) {}
kwibergc13ded52016-06-17 06:00:45 -0700105 uint32_t input_timestamp;
106 const int16_t* audio;
107 size_t length_per_channel;
108 size_t audio_channel;
109 // If a re-mix is required (up or down), this buffer will store a re-mixed
110 // version of the input.
Per Åhgren4f2e9402019-10-04 11:06:15 +0200111 std::vector<int16_t> buffer;
kwibergc13ded52016-06-17 06:00:45 -0700112 };
113
Per Åhgren4f2e9402019-10-04 11:06:15 +0200114 InputData input_data_ RTC_GUARDED_BY(acm_crit_sect_);
115
kwibergc13ded52016-06-17 06:00:45 -0700116 // This member class writes values to the named UMA histogram, but only if
117 // the value has changed since the last time (and always for the first call).
118 class ChangeLogger {
119 public:
120 explicit ChangeLogger(const std::string& histogram_name)
121 : histogram_name_(histogram_name) {}
122 // Logs the new value if it is different from the last logged value, or if
123 // this is the first call.
124 void MaybeLog(int value);
125
126 private:
127 int last_value_ = 0;
128 int first_time_ = true;
129 const std::string histogram_name_;
130 };
131
kwibergc13ded52016-06-17 06:00:45 -0700132 int Add10MsDataInternal(const AudioFrame& audio_frame, InputData* input_data)
danilchap56359be2017-09-07 07:53:45 -0700133 RTC_EXCLUSIVE_LOCKS_REQUIRED(acm_crit_sect_);
Minyue Lidea73ee2020-02-18 15:45:41 +0100134
135 // TODO(bugs.webrtc.org/10739): change |absolute_capture_timestamp_ms| to
136 // int64_t when it always receives a valid value.
137 int Encode(const InputData& input_data,
138 absl::optional<int64_t> absolute_capture_timestamp_ms)
danilchap56359be2017-09-07 07:53:45 -0700139 RTC_EXCLUSIVE_LOCKS_REQUIRED(acm_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700140
danilchap56359be2017-09-07 07:53:45 -0700141 int InitializeReceiverSafe() RTC_EXCLUSIVE_LOCKS_REQUIRED(acm_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700142
143 bool HaveValidEncoder(const char* caller_name) const
danilchap56359be2017-09-07 07:53:45 -0700144 RTC_EXCLUSIVE_LOCKS_REQUIRED(acm_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700145
146 // Preprocessing of input audio, including resampling and down-mixing if
147 // required, before pushing audio into encoder's buffer.
148 //
149 // in_frame: input audio-frame
150 // ptr_out: pointer to output audio_frame. If no preprocessing is required
151 // |ptr_out| will be pointing to |in_frame|, otherwise pointing to
152 // |preprocess_frame_|.
153 //
154 // Return value:
155 // -1: if encountering an error.
156 // 0: otherwise.
157 int PreprocessToAddData(const AudioFrame& in_frame,
158 const AudioFrame** ptr_out)
danilchap56359be2017-09-07 07:53:45 -0700159 RTC_EXCLUSIVE_LOCKS_REQUIRED(acm_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700160
161 // Change required states after starting to receive the codec corresponding
162 // to |index|.
163 int UpdateUponReceivingCodec(int index);
164
165 rtc::CriticalSection acm_crit_sect_;
danilchap56359be2017-09-07 07:53:45 -0700166 rtc::Buffer encode_buffer_ RTC_GUARDED_BY(acm_crit_sect_);
danilchap56359be2017-09-07 07:53:45 -0700167 uint32_t expected_codec_ts_ RTC_GUARDED_BY(acm_crit_sect_);
168 uint32_t expected_in_ts_ RTC_GUARDED_BY(acm_crit_sect_);
169 acm2::ACMResampler resampler_ RTC_GUARDED_BY(acm_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700170 acm2::AcmReceiver receiver_; // AcmReceiver has it's own internal lock.
danilchap56359be2017-09-07 07:53:45 -0700171 ChangeLogger bitrate_logger_ RTC_GUARDED_BY(acm_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700172
Karl Wiberg49c33ce2018-11-12 14:21:58 +0100173 // Current encoder stack, provided by a call to RegisterEncoder.
danilchap56359be2017-09-07 07:53:45 -0700174 std::unique_ptr<AudioEncoder> encoder_stack_ RTC_GUARDED_BY(acm_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700175
kwibergc13ded52016-06-17 06:00:45 -0700176 // This is to keep track of CN instances where we can send DTMFs.
danilchap56359be2017-09-07 07:53:45 -0700177 uint8_t previous_pltype_ RTC_GUARDED_BY(acm_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700178
danilchap56359be2017-09-07 07:53:45 -0700179 bool receiver_initialized_ RTC_GUARDED_BY(acm_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700180
danilchap56359be2017-09-07 07:53:45 -0700181 AudioFrame preprocess_frame_ RTC_GUARDED_BY(acm_crit_sect_);
182 bool first_10ms_data_ RTC_GUARDED_BY(acm_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700183
danilchap56359be2017-09-07 07:53:45 -0700184 bool first_frame_ RTC_GUARDED_BY(acm_crit_sect_);
185 uint32_t last_timestamp_ RTC_GUARDED_BY(acm_crit_sect_);
186 uint32_t last_rtp_timestamp_ RTC_GUARDED_BY(acm_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700187
188 rtc::CriticalSection callback_crit_sect_;
189 AudioPacketizationCallback* packetization_callback_
danilchap56359be2017-09-07 07:53:45 -0700190 RTC_GUARDED_BY(callback_crit_sect_);
191 ACMVADCallback* vad_callback_ RTC_GUARDED_BY(callback_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700192
193 int codec_histogram_bins_log_[static_cast<size_t>(
194 AudioEncoder::CodecType::kMaxLoggedAudioCodecTypes)];
195 int number_of_consecutive_empty_packets_;
196};
197
198// Adds a codec usage sample to the histogram.
199void UpdateCodecTypeHistogram(size_t codec_type) {
200 RTC_HISTOGRAM_ENUMERATION(
201 "WebRTC.Audio.Encoder.CodecType", static_cast<int>(codec_type),
202 static_cast<int>(
203 webrtc::AudioEncoder::CodecType::kMaxLoggedAudioCodecTypes));
204}
205
kwibergc13ded52016-06-17 06:00:45 -0700206void AudioCodingModuleImpl::ChangeLogger::MaybeLog(int value) {
207 if (value != last_value_ || first_time_) {
208 first_time_ = false;
209 last_value_ = value;
210 RTC_HISTOGRAM_COUNTS_SPARSE_100(histogram_name_, value);
211 }
212}
213
214AudioCodingModuleImpl::AudioCodingModuleImpl(
215 const AudioCodingModule::Config& config)
solenbergc7b4a452017-09-28 07:37:11 -0700216 : expected_codec_ts_(0xD87F3F9F),
kwibergc13ded52016-06-17 06:00:45 -0700217 expected_in_ts_(0xD87F3F9F),
218 receiver_(config),
219 bitrate_logger_("WebRTC.Audio.TargetBitrateInKbps"),
kwibergc13ded52016-06-17 06:00:45 -0700220 encoder_stack_(nullptr),
221 previous_pltype_(255),
222 receiver_initialized_(false),
223 first_10ms_data_(false),
224 first_frame_(true),
225 packetization_callback_(NULL),
226 vad_callback_(NULL),
227 codec_histogram_bins_log_(),
228 number_of_consecutive_empty_packets_(0) {
229 if (InitializeReceiverSafe() < 0) {
Mirko Bonadei675513b2017-11-09 11:09:25 +0100230 RTC_LOG(LS_ERROR) << "Cannot initialize receiver";
kwibergc13ded52016-06-17 06:00:45 -0700231 }
Mirko Bonadei675513b2017-11-09 11:09:25 +0100232 RTC_LOG(LS_INFO) << "Created";
kwibergc13ded52016-06-17 06:00:45 -0700233}
234
235AudioCodingModuleImpl::~AudioCodingModuleImpl() = default;
236
Minyue Lidea73ee2020-02-18 15:45:41 +0100237int32_t AudioCodingModuleImpl::Encode(
238 const InputData& input_data,
239 absl::optional<int64_t> absolute_capture_timestamp_ms) {
240 // TODO(bugs.webrtc.org/10739): add dcheck that
241 // |audio_frame.absolute_capture_timestamp_ms()| always has a value.
kwibergc13ded52016-06-17 06:00:45 -0700242 AudioEncoder::EncodedInfo encoded_info;
243 uint8_t previous_pltype;
244
245 // Check if there is an encoder before.
246 if (!HaveValidEncoder("Process"))
247 return -1;
248
Yves Gerey665174f2018-06-19 15:03:05 +0200249 if (!first_frame_) {
deadbeeffcada902016-08-24 12:45:13 -0700250 RTC_DCHECK(IsNewerTimestamp(input_data.input_timestamp, last_timestamp_))
ossu63fb95a2016-07-06 09:34:22 -0700251 << "Time should not move backwards";
252 }
253
kwibergc13ded52016-06-17 06:00:45 -0700254 // Scale the timestamp to the codec's RTP timestamp rate.
255 uint32_t rtp_timestamp =
Karl Wiberg053c3712019-05-16 15:24:17 +0200256 first_frame_
257 ? input_data.input_timestamp
258 : last_rtp_timestamp_ +
259 rtc::dchecked_cast<uint32_t>(rtc::CheckedDivExact(
260 int64_t{input_data.input_timestamp - last_timestamp_} *
261 encoder_stack_->RtpTimestampRateHz(),
262 int64_t{encoder_stack_->SampleRateHz()}));
Minyue Liff0e4db2020-01-23 13:45:50 +0100263
kwibergc13ded52016-06-17 06:00:45 -0700264 last_timestamp_ = input_data.input_timestamp;
265 last_rtp_timestamp_ = rtp_timestamp;
266 first_frame_ = false;
267
268 // Clear the buffer before reuse - encoded data will get appended.
269 encode_buffer_.Clear();
270 encoded_info = encoder_stack_->Encode(
Yves Gerey665174f2018-06-19 15:03:05 +0200271 rtp_timestamp,
272 rtc::ArrayView<const int16_t>(
273 input_data.audio,
274 input_data.audio_channel * input_data.length_per_channel),
kwibergc13ded52016-06-17 06:00:45 -0700275 &encode_buffer_);
276
277 bitrate_logger_.MaybeLog(encoder_stack_->GetTargetBitrate() / 1000);
278 if (encode_buffer_.size() == 0 && !encoded_info.send_even_if_empty) {
279 // Not enough data.
280 return 0;
281 }
282 previous_pltype = previous_pltype_; // Read it while we have the critsect.
283
284 // Log codec type to histogram once every 500 packets.
285 if (encoded_info.encoded_bytes == 0) {
286 ++number_of_consecutive_empty_packets_;
287 } else {
288 size_t codec_type = static_cast<size_t>(encoded_info.encoder_type);
289 codec_histogram_bins_log_[codec_type] +=
290 number_of_consecutive_empty_packets_ + 1;
291 number_of_consecutive_empty_packets_ = 0;
292 if (codec_histogram_bins_log_[codec_type] >= 500) {
293 codec_histogram_bins_log_[codec_type] -= 500;
294 UpdateCodecTypeHistogram(codec_type);
295 }
296 }
297
Niels Möller87e2d782019-03-07 10:18:23 +0100298 AudioFrameType frame_type;
kwibergc13ded52016-06-17 06:00:45 -0700299 if (encode_buffer_.size() == 0 && encoded_info.send_even_if_empty) {
Niels Möllerc936cb62019-03-19 14:10:16 +0100300 frame_type = AudioFrameType::kEmptyFrame;
kwibergc13ded52016-06-17 06:00:45 -0700301 encoded_info.payload_type = previous_pltype;
302 } else {
kwibergaf476c72016-11-28 15:21:39 -0800303 RTC_DCHECK_GT(encode_buffer_.size(), 0);
Niels Möllerc936cb62019-03-19 14:10:16 +0100304 frame_type = encoded_info.speech ? AudioFrameType::kAudioFrameSpeech
305 : AudioFrameType::kAudioFrameCN;
kwibergc13ded52016-06-17 06:00:45 -0700306 }
307
308 {
309 rtc::CritScope lock(&callback_crit_sect_);
310 if (packetization_callback_) {
311 packetization_callback_->SendData(
312 frame_type, encoded_info.payload_type, encoded_info.encoded_timestamp,
Minyue Liff0e4db2020-01-23 13:45:50 +0100313 encode_buffer_.data(), encode_buffer_.size(),
Minyue Lidea73ee2020-02-18 15:45:41 +0100314 absolute_capture_timestamp_ms.value_or(-1));
kwibergc13ded52016-06-17 06:00:45 -0700315 }
316
317 if (vad_callback_) {
318 // Callback with VAD decision.
319 vad_callback_->InFrameType(frame_type);
320 }
321 }
322 previous_pltype_ = encoded_info.payload_type;
323 return static_cast<int32_t>(encode_buffer_.size());
324}
325
326/////////////////////////////////////////
327// Sender
328//
329
kwibergc13ded52016-06-17 06:00:45 -0700330void AudioCodingModuleImpl::ModifyEncoder(
kwiberg24c7c122016-09-28 11:57:10 -0700331 rtc::FunctionView<void(std::unique_ptr<AudioEncoder>*)> modifier) {
kwibergc13ded52016-06-17 06:00:45 -0700332 rtc::CritScope lock(&acm_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700333 modifier(&encoder_stack_);
334}
335
kwibergc13ded52016-06-17 06:00:45 -0700336// Register a transport callback which will be called to deliver
337// the encoded buffers.
338int AudioCodingModuleImpl::RegisterTransportCallback(
339 AudioPacketizationCallback* transport) {
340 rtc::CritScope lock(&callback_crit_sect_);
341 packetization_callback_ = transport;
342 return 0;
343}
344
345// Add 10MS of raw (PCM) audio data to the encoder.
346int AudioCodingModuleImpl::Add10MsData(const AudioFrame& audio_frame) {
kwibergc13ded52016-06-17 06:00:45 -0700347 rtc::CritScope lock(&acm_crit_sect_);
Per Åhgren4f2e9402019-10-04 11:06:15 +0200348 int r = Add10MsDataInternal(audio_frame, &input_data_);
Minyue Lidea73ee2020-02-18 15:45:41 +0100349 // TODO(bugs.webrtc.org/10739): add dcheck that
350 // |audio_frame.absolute_capture_timestamp_ms()| always has a value.
351 return r < 0
352 ? r
353 : Encode(input_data_, audio_frame.absolute_capture_timestamp_ms());
kwibergc13ded52016-06-17 06:00:45 -0700354}
355
356int AudioCodingModuleImpl::Add10MsDataInternal(const AudioFrame& audio_frame,
357 InputData* input_data) {
358 if (audio_frame.samples_per_channel_ == 0) {
359 assert(false);
Mirko Bonadei675513b2017-11-09 11:09:25 +0100360 RTC_LOG(LS_ERROR) << "Cannot Add 10 ms audio, payload length is zero";
kwibergc13ded52016-06-17 06:00:45 -0700361 return -1;
362 }
363
henrika33541572019-09-10 14:27:40 +0200364 if (audio_frame.sample_rate_hz_ > 192000) {
kwibergc13ded52016-06-17 06:00:45 -0700365 assert(false);
Mirko Bonadei675513b2017-11-09 11:09:25 +0100366 RTC_LOG(LS_ERROR) << "Cannot Add 10 ms audio, input frequency not valid";
kwibergc13ded52016-06-17 06:00:45 -0700367 return -1;
368 }
369
370 // If the length and frequency matches. We currently just support raw PCM.
371 if (static_cast<size_t>(audio_frame.sample_rate_hz_ / 100) !=
372 audio_frame.samples_per_channel_) {
Mirko Bonadei675513b2017-11-09 11:09:25 +0100373 RTC_LOG(LS_ERROR)
Alex Loiko300ec8c2017-05-30 17:23:28 +0200374 << "Cannot Add 10 ms audio, input frequency and length doesn't match";
kwibergc13ded52016-06-17 06:00:45 -0700375 return -1;
376 }
377
Alex Loiko65438812019-02-22 10:13:44 +0100378 if (audio_frame.num_channels_ != 1 && audio_frame.num_channels_ != 2 &&
379 audio_frame.num_channels_ != 4 && audio_frame.num_channels_ != 6 &&
380 audio_frame.num_channels_ != 8) {
Mirko Bonadei675513b2017-11-09 11:09:25 +0100381 RTC_LOG(LS_ERROR) << "Cannot Add 10 ms audio, invalid number of channels.";
kwibergc13ded52016-06-17 06:00:45 -0700382 return -1;
383 }
384
385 // Do we have a codec registered?
386 if (!HaveValidEncoder("Add10MsData")) {
387 return -1;
388 }
389
390 const AudioFrame* ptr_frame;
391 // Perform a resampling, also down-mix if it is required and can be
392 // performed before resampling (a down mix prior to resampling will take
393 // place if both primary and secondary encoders are mono and input is in
394 // stereo).
395 if (PreprocessToAddData(audio_frame, &ptr_frame) < 0) {
396 return -1;
397 }
398
399 // Check whether we need an up-mix or down-mix?
400 const size_t current_num_channels = encoder_stack_->NumChannels();
401 const bool same_num_channels =
402 ptr_frame->num_channels_ == current_num_channels;
403
yujo36b1a5f2017-06-12 12:45:32 -0700404 // TODO(yujo): Skip encode of muted frames.
kwibergc13ded52016-06-17 06:00:45 -0700405 input_data->input_timestamp = ptr_frame->timestamp_;
kwibergc13ded52016-06-17 06:00:45 -0700406 input_data->length_per_channel = ptr_frame->samples_per_channel_;
407 input_data->audio_channel = current_num_channels;
408
Per Åhgren4f2e9402019-10-04 11:06:15 +0200409 if (!same_num_channels) {
410 // Remixes the input frame to the output data and in the process resize the
411 // output data if needed.
Per Åhgren4dd56a32019-11-19 21:00:59 +0100412 ReMixFrame(*ptr_frame, current_num_channels, &input_data->buffer);
Per Åhgren4f2e9402019-10-04 11:06:15 +0200413
414 // For pushing data to primary, point the |ptr_audio| to correct buffer.
415 input_data->audio = input_data->buffer.data();
416 RTC_DCHECK_GE(input_data->buffer.size(),
417 input_data->length_per_channel * input_data->audio_channel);
418 } else {
419 // When adding data to encoders this pointer is pointing to an audio buffer
420 // with correct number of channels.
421 input_data->audio = ptr_frame->data();
422 }
423
kwibergc13ded52016-06-17 06:00:45 -0700424 return 0;
425}
426
427// Perform a resampling and down-mix if required. We down-mix only if
428// encoder is mono and input is stereo. In case of dual-streaming, both
429// encoders has to be mono for down-mix to take place.
430// |*ptr_out| will point to the pre-processed audio-frame. If no pre-processing
431// is required, |*ptr_out| points to |in_frame|.
yujo36b1a5f2017-06-12 12:45:32 -0700432// TODO(yujo): Make this more efficient for muted frames.
kwibergc13ded52016-06-17 06:00:45 -0700433int AudioCodingModuleImpl::PreprocessToAddData(const AudioFrame& in_frame,
434 const AudioFrame** ptr_out) {
435 const bool resample =
436 in_frame.sample_rate_hz_ != encoder_stack_->SampleRateHz();
437
438 // This variable is true if primary codec and secondary codec (if exists)
439 // are both mono and input is stereo.
440 // TODO(henrik.lundin): This condition should probably be
441 // in_frame.num_channels_ > encoder_stack_->NumChannels()
442 const bool down_mix =
443 in_frame.num_channels_ == 2 && encoder_stack_->NumChannels() == 1;
444
445 if (!first_10ms_data_) {
446 expected_in_ts_ = in_frame.timestamp_;
447 expected_codec_ts_ = in_frame.timestamp_;
448 first_10ms_data_ = true;
449 } else if (in_frame.timestamp_ != expected_in_ts_) {
Mirko Bonadei675513b2017-11-09 11:09:25 +0100450 RTC_LOG(LS_WARNING) << "Unexpected input timestamp: " << in_frame.timestamp_
451 << ", expected: " << expected_in_ts_;
kwibergc13ded52016-06-17 06:00:45 -0700452 expected_codec_ts_ +=
453 (in_frame.timestamp_ - expected_in_ts_) *
454 static_cast<uint32_t>(
455 static_cast<double>(encoder_stack_->SampleRateHz()) /
456 static_cast<double>(in_frame.sample_rate_hz_));
457 expected_in_ts_ = in_frame.timestamp_;
458 }
459
kwibergc13ded52016-06-17 06:00:45 -0700460 if (!down_mix && !resample) {
461 // No pre-processing is required.
ossu63fb95a2016-07-06 09:34:22 -0700462 if (expected_in_ts_ == expected_codec_ts_) {
463 // If we've never resampled, we can use the input frame as-is
464 *ptr_out = &in_frame;
465 } else {
466 // Otherwise we'll need to alter the timestamp. Since in_frame is const,
467 // we'll have to make a copy of it.
468 preprocess_frame_.CopyFrom(in_frame);
469 preprocess_frame_.timestamp_ = expected_codec_ts_;
470 *ptr_out = &preprocess_frame_;
471 }
472
kwibergc13ded52016-06-17 06:00:45 -0700473 expected_in_ts_ += static_cast<uint32_t>(in_frame.samples_per_channel_);
474 expected_codec_ts_ += static_cast<uint32_t>(in_frame.samples_per_channel_);
kwibergc13ded52016-06-17 06:00:45 -0700475 return 0;
476 }
477
478 *ptr_out = &preprocess_frame_;
479 preprocess_frame_.num_channels_ = in_frame.num_channels_;
Per Åhgren4dd56a32019-11-19 21:00:59 +0100480 preprocess_frame_.samples_per_channel_ = in_frame.samples_per_channel_;
481 std::array<int16_t, WEBRTC_10MS_PCM_AUDIO> audio;
yujo36b1a5f2017-06-12 12:45:32 -0700482 const int16_t* src_ptr_audio = in_frame.data();
kwibergc13ded52016-06-17 06:00:45 -0700483 if (down_mix) {
484 // If a resampling is required the output of a down-mix is written into a
485 // local buffer, otherwise, it will be written to the output frame.
Yves Gerey665174f2018-06-19 15:03:05 +0200486 int16_t* dest_ptr_audio =
Per Åhgren4dd56a32019-11-19 21:00:59 +0100487 resample ? audio.data() : preprocess_frame_.mutable_data();
488 RTC_DCHECK_GE(audio.size(), in_frame.samples_per_channel_);
489 DownMixFrame(in_frame,
490 rtc::ArrayView<int16_t>(
491 dest_ptr_audio, preprocess_frame_.samples_per_channel_));
kwibergc13ded52016-06-17 06:00:45 -0700492 preprocess_frame_.num_channels_ = 1;
493 // Set the input of the resampler is the down-mixed signal.
Per Åhgren4dd56a32019-11-19 21:00:59 +0100494 src_ptr_audio = audio.data();
kwibergc13ded52016-06-17 06:00:45 -0700495 }
496
497 preprocess_frame_.timestamp_ = expected_codec_ts_;
kwibergc13ded52016-06-17 06:00:45 -0700498 preprocess_frame_.sample_rate_hz_ = in_frame.sample_rate_hz_;
499 // If it is required, we have to do a resampling.
500 if (resample) {
501 // The result of the resampler is written to output frame.
yujo36b1a5f2017-06-12 12:45:32 -0700502 int16_t* dest_ptr_audio = preprocess_frame_.mutable_data();
kwibergc13ded52016-06-17 06:00:45 -0700503
504 int samples_per_channel = resampler_.Resample10Msec(
505 src_ptr_audio, in_frame.sample_rate_hz_, encoder_stack_->SampleRateHz(),
506 preprocess_frame_.num_channels_, AudioFrame::kMaxDataSizeSamples,
507 dest_ptr_audio);
508
509 if (samples_per_channel < 0) {
Mirko Bonadei675513b2017-11-09 11:09:25 +0100510 RTC_LOG(LS_ERROR) << "Cannot add 10 ms audio, resampling failed";
kwibergc13ded52016-06-17 06:00:45 -0700511 return -1;
512 }
513 preprocess_frame_.samples_per_channel_ =
514 static_cast<size_t>(samples_per_channel);
515 preprocess_frame_.sample_rate_hz_ = encoder_stack_->SampleRateHz();
516 }
517
518 expected_codec_ts_ +=
519 static_cast<uint32_t>(preprocess_frame_.samples_per_channel_);
520 expected_in_ts_ += static_cast<uint32_t>(in_frame.samples_per_channel_);
521
522 return 0;
523}
524
525/////////////////////////////////////////
kwibergc13ded52016-06-17 06:00:45 -0700526// (FEC) Forward Error Correction (codec internal)
527//
528
kwibergc13ded52016-06-17 06:00:45 -0700529int AudioCodingModuleImpl::SetPacketLossRate(int loss_rate) {
530 rtc::CritScope lock(&acm_crit_sect_);
531 if (HaveValidEncoder("SetPacketLossRate")) {
minyue4b9a2cb2016-11-30 06:49:59 -0800532 encoder_stack_->OnReceivedUplinkPacketLossFraction(loss_rate / 100.0);
kwibergc13ded52016-06-17 06:00:45 -0700533 }
534 return 0;
535}
536
537/////////////////////////////////////////
kwibergc13ded52016-06-17 06:00:45 -0700538// Receiver
539//
540
541int AudioCodingModuleImpl::InitializeReceiver() {
542 rtc::CritScope lock(&acm_crit_sect_);
543 return InitializeReceiverSafe();
544}
545
546// Initialize receiver, resets codec database etc.
547int AudioCodingModuleImpl::InitializeReceiverSafe() {
548 // If the receiver is already initialized then we want to destroy any
549 // existing decoders. After a call to this function, we should have a clean
550 // start-up.
kwiberg6b19b562016-09-20 04:02:25 -0700551 if (receiver_initialized_)
552 receiver_.RemoveAllCodecs();
kwibergc13ded52016-06-17 06:00:45 -0700553 receiver_.FlushBuffers();
554
kwibergc13ded52016-06-17 06:00:45 -0700555 receiver_initialized_ = true;
556 return 0;
557}
558
kwiberg1c07c702017-03-27 07:15:49 -0700559void AudioCodingModuleImpl::SetReceiveCodecs(
560 const std::map<int, SdpAudioFormat>& codecs) {
561 rtc::CritScope lock(&acm_crit_sect_);
562 receiver_.SetCodecs(codecs);
563}
564
kwibergc13ded52016-06-17 06:00:45 -0700565// Incoming packet from network parsed and ready for decode.
566int AudioCodingModuleImpl::IncomingPacket(const uint8_t* incoming_payload,
567 const size_t payload_length,
Niels Möllerafb5dbb2019-02-15 15:21:47 +0100568 const RTPHeader& rtp_header) {
henrik.lundinb8c55b12017-05-10 07:38:01 -0700569 RTC_DCHECK_EQ(payload_length == 0, incoming_payload == nullptr);
kwibergc13ded52016-06-17 06:00:45 -0700570 return receiver_.InsertPacket(
571 rtp_header,
572 rtc::ArrayView<const uint8_t>(incoming_payload, payload_length));
573}
574
kwibergc13ded52016-06-17 06:00:45 -0700575// Get 10 milliseconds of raw audio data to play out.
576// Automatic resample to the requested frequency.
577int AudioCodingModuleImpl::PlayoutData10Ms(int desired_freq_hz,
578 AudioFrame* audio_frame,
579 bool* muted) {
580 // GetAudio always returns 10 ms, at the requested sample rate.
581 if (receiver_.GetAudio(desired_freq_hz, audio_frame, muted) != 0) {
Mirko Bonadei675513b2017-11-09 11:09:25 +0100582 RTC_LOG(LS_ERROR) << "PlayoutData failed, RecOut Failed";
kwibergc13ded52016-06-17 06:00:45 -0700583 return -1;
584 }
kwibergc13ded52016-06-17 06:00:45 -0700585 return 0;
586}
587
kwibergc13ded52016-06-17 06:00:45 -0700588/////////////////////////////////////////
589// Statistics
590//
591
592// TODO(turajs) change the return value to void. Also change the corresponding
593// NetEq function.
594int AudioCodingModuleImpl::GetNetworkStatistics(NetworkStatistics* statistics) {
595 receiver_.GetNetworkStatistics(statistics);
596 return 0;
597}
598
599int AudioCodingModuleImpl::RegisterVADCallback(ACMVADCallback* vad_callback) {
Mirko Bonadei675513b2017-11-09 11:09:25 +0100600 RTC_LOG(LS_VERBOSE) << "RegisterVADCallback()";
kwibergc13ded52016-06-17 06:00:45 -0700601 rtc::CritScope lock(&callback_crit_sect_);
602 vad_callback_ = vad_callback;
603 return 0;
604}
605
kwibergc13ded52016-06-17 06:00:45 -0700606bool AudioCodingModuleImpl::HaveValidEncoder(const char* caller_name) const {
607 if (!encoder_stack_) {
Mirko Bonadei675513b2017-11-09 11:09:25 +0100608 RTC_LOG(LS_ERROR) << caller_name << " failed: No send codec is registered.";
kwibergc13ded52016-06-17 06:00:45 -0700609 return false;
610 }
611 return true;
612}
613
ivoce1198e02017-09-08 08:13:19 -0700614ANAStats AudioCodingModuleImpl::GetANAStats() const {
615 rtc::CritScope lock(&acm_crit_sect_);
616 if (encoder_stack_)
617 return encoder_stack_->GetANAStats();
618 // If no encoder is set, return default stats.
619 return ANAStats();
620}
621
kwibergc13ded52016-06-17 06:00:45 -0700622} // namespace
623
Karl Wiberg5817d3d2018-04-06 10:06:42 +0200624AudioCodingModule::Config::Config(
625 rtc::scoped_refptr<AudioDecoderFactory> decoder_factory)
626 : neteq_config(),
627 clock(Clock::GetRealTimeClock()),
628 decoder_factory(decoder_factory) {
kwiberg36a43882016-08-29 05:33:32 -0700629 // Post-decode VAD is disabled by default in NetEq, however, Audio
630 // Conference Mixer relies on VAD decisions and fails without them.
631 neteq_config.enable_post_decode_vad = true;
632}
633
634AudioCodingModule::Config::Config(const Config&) = default;
635AudioCodingModule::Config::~Config() = default;
636
Henrik Lundin64dad832015-05-11 12:44:23 +0200637AudioCodingModule* AudioCodingModule::Create(const Config& config) {
kwibergc13ded52016-06-17 06:00:45 -0700638 return new AudioCodingModuleImpl(config);
turaj@webrtc.org7959e162013-09-12 18:30:26 +0000639}
640
turaj@webrtc.org7959e162013-09-12 18:30:26 +0000641} // namespace webrtc