blob: 1547b3719836ef522fcd6200c38ba2a49115f094 [file] [log] [blame]
turaj@webrtc.org7959e162013-09-12 18:30:26 +00001/*
2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020011#include "modules/audio_coding/include/audio_coding_module.h"
turaj@webrtc.org7959e162013-09-12 18:30:26 +000012
Yves Gerey988cc082018-10-23 12:03:01 +020013#include <assert.h>
Jonathan Yu36344a02017-07-30 01:55:34 -070014#include <algorithm>
Yves Gerey988cc082018-10-23 12:03:01 +020015#include <cstdint>
Jonathan Yu36344a02017-07-30 01:55:34 -070016
Niels Möller2edab4c2018-10-22 09:48:08 +020017#include "absl/strings/match.h"
Yves Gerey988cc082018-10-23 12:03:01 +020018#include "api/array_view.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020019#include "modules/audio_coding/acm2/acm_receiver.h"
20#include "modules/audio_coding/acm2/acm_resampler.h"
Fredrik Solenbergbbf21a32018-04-12 22:44:09 +020021#include "modules/include/module_common_types.h"
Yves Gerey988cc082018-10-23 12:03:01 +020022#include "modules/include/module_common_types_public.h"
23#include "rtc_base/buffer.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020024#include "rtc_base/checks.h"
Steve Anton10542f22019-01-11 09:11:00 -080025#include "rtc_base/critical_section.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020026#include "rtc_base/logging.h"
Karl Wiberge40468b2017-11-22 10:42:26 +010027#include "rtc_base/numerics/safe_conversions.h"
Yves Gerey988cc082018-10-23 12:03:01 +020028#include "rtc_base/thread_annotations.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020029#include "system_wrappers/include/metrics.h"
turaj@webrtc.org7959e162013-09-12 18:30:26 +000030
31namespace webrtc {
32
kwibergc13ded52016-06-17 06:00:45 -070033namespace {
34
kwibergc13ded52016-06-17 06:00:45 -070035class AudioCodingModuleImpl final : public AudioCodingModule {
36 public:
37 explicit AudioCodingModuleImpl(const AudioCodingModule::Config& config);
38 ~AudioCodingModuleImpl() override;
39
40 /////////////////////////////////////////
41 // Sender
42 //
43
kwiberg24c7c122016-09-28 11:57:10 -070044 void ModifyEncoder(rtc::FunctionView<void(std::unique_ptr<AudioEncoder>*)>
45 modifier) override;
kwibergc13ded52016-06-17 06:00:45 -070046
kwibergc13ded52016-06-17 06:00:45 -070047 // Sets the bitrate to the specified value in bits/sec. In case the codec does
48 // not support the requested value it will choose an appropriate value
49 // instead.
50 void SetBitRate(int bitrate_bps) override;
51
52 // Register a transport callback which will be
53 // called to deliver the encoded buffers.
54 int RegisterTransportCallback(AudioPacketizationCallback* transport) override;
55
56 // Add 10 ms of raw (PCM) audio data to the encoder.
57 int Add10MsData(const AudioFrame& audio_frame) override;
58
59 /////////////////////////////////////////
kwibergc13ded52016-06-17 06:00:45 -070060 // (FEC) Forward Error Correction (codec internal)
61 //
62
kwibergc13ded52016-06-17 06:00:45 -070063 // Set target packet loss rate
64 int SetPacketLossRate(int loss_rate) override;
65
66 /////////////////////////////////////////
67 // (VAD) Voice Activity Detection
68 // and
69 // (CNG) Comfort Noise Generation
70 //
71
kwibergc13ded52016-06-17 06:00:45 -070072 int RegisterVADCallback(ACMVADCallback* vad_callback) override;
73
74 /////////////////////////////////////////
75 // Receiver
76 //
77
78 // Initialize receiver, resets codec database etc.
79 int InitializeReceiver() override;
80
81 // Get current receive frequency.
82 int ReceiveFrequency() const override;
83
84 // Get current playout frequency.
85 int PlayoutFrequency() const override;
86
kwiberg1c07c702017-03-27 07:15:49 -070087 void SetReceiveCodecs(const std::map<int, SdpAudioFormat>& codecs) override;
88
kwibergc13ded52016-06-17 06:00:45 -070089 // Get current received codec.
Fredrik Solenbergf693bfa2018-12-11 12:22:10 +010090 absl::optional<std::pair<int, SdpAudioFormat>> ReceiveCodec() const override;
ossue280cde2016-10-12 11:04:10 -070091
kwibergc13ded52016-06-17 06:00:45 -070092 // Incoming packet from network parsed and ready for decode.
93 int IncomingPacket(const uint8_t* incoming_payload,
94 const size_t payload_length,
Niels Möllerafb5dbb2019-02-15 15:21:47 +010095 const RTPHeader& rtp_info) override;
kwibergc13ded52016-06-17 06:00:45 -070096
kwibergc13ded52016-06-17 06:00:45 -070097 // Minimum playout delay.
98 int SetMinimumPlayoutDelay(int time_ms) override;
99
100 // Maximum playout delay.
101 int SetMaximumPlayoutDelay(int time_ms) override;
102
Ruslan Burakov3b50f9f2019-02-06 09:45:56 +0100103 bool SetBaseMinimumPlayoutDelayMs(int delay_ms) override;
104
105 int GetBaseMinimumPlayoutDelayMs() const override;
106
Danil Chapovalovb6021232018-06-19 13:26:36 +0200107 absl::optional<uint32_t> PlayoutTimestamp() override;
kwibergc13ded52016-06-17 06:00:45 -0700108
henrik.lundinb3f1c5d2016-08-22 15:39:53 -0700109 int FilteredCurrentDelayMs() const override;
110
Henrik Lundinabbff892017-11-29 09:14:04 +0100111 int TargetDelayMs() const override;
112
kwibergc13ded52016-06-17 06:00:45 -0700113 // Get 10 milliseconds of raw audio data to play out, and
114 // automatic resample to the requested frequency if > 0.
115 int PlayoutData10Ms(int desired_freq_hz,
116 AudioFrame* audio_frame,
117 bool* muted) override;
kwibergc13ded52016-06-17 06:00:45 -0700118
119 /////////////////////////////////////////
120 // Statistics
121 //
122
123 int GetNetworkStatistics(NetworkStatistics* statistics) override;
124
kwibergc13ded52016-06-17 06:00:45 -0700125 // If current send codec is Opus, informs it about the maximum playback rate
126 // the receiver will render.
127 int SetOpusMaxPlaybackRate(int frequency_hz) override;
128
129 int EnableOpusDtx() override;
130
131 int DisableOpusDtx() override;
132
kwibergc13ded52016-06-17 06:00:45 -0700133 int EnableNack(size_t max_nack_list_size) override;
134
135 void DisableNack() override;
136
137 std::vector<uint16_t> GetNackList(int64_t round_trip_time_ms) const override;
138
139 void GetDecodingCallStatistics(AudioDecodingCallStats* stats) const override;
140
ivoce1198e02017-09-08 08:13:19 -0700141 ANAStats GetANAStats() const override;
142
kwibergc13ded52016-06-17 06:00:45 -0700143 private:
144 struct InputData {
145 uint32_t input_timestamp;
146 const int16_t* audio;
147 size_t length_per_channel;
148 size_t audio_channel;
149 // If a re-mix is required (up or down), this buffer will store a re-mixed
150 // version of the input.
151 int16_t buffer[WEBRTC_10MS_PCM_AUDIO];
152 };
153
154 // This member class writes values to the named UMA histogram, but only if
155 // the value has changed since the last time (and always for the first call).
156 class ChangeLogger {
157 public:
158 explicit ChangeLogger(const std::string& histogram_name)
159 : histogram_name_(histogram_name) {}
160 // Logs the new value if it is different from the last logged value, or if
161 // this is the first call.
162 void MaybeLog(int value);
163
164 private:
165 int last_value_ = 0;
166 int first_time_ = true;
167 const std::string histogram_name_;
168 };
169
kwibergc13ded52016-06-17 06:00:45 -0700170 int Add10MsDataInternal(const AudioFrame& audio_frame, InputData* input_data)
danilchap56359be2017-09-07 07:53:45 -0700171 RTC_EXCLUSIVE_LOCKS_REQUIRED(acm_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700172 int Encode(const InputData& input_data)
danilchap56359be2017-09-07 07:53:45 -0700173 RTC_EXCLUSIVE_LOCKS_REQUIRED(acm_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700174
danilchap56359be2017-09-07 07:53:45 -0700175 int InitializeReceiverSafe() RTC_EXCLUSIVE_LOCKS_REQUIRED(acm_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700176
177 bool HaveValidEncoder(const char* caller_name) const
danilchap56359be2017-09-07 07:53:45 -0700178 RTC_EXCLUSIVE_LOCKS_REQUIRED(acm_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700179
180 // Preprocessing of input audio, including resampling and down-mixing if
181 // required, before pushing audio into encoder's buffer.
182 //
183 // in_frame: input audio-frame
184 // ptr_out: pointer to output audio_frame. If no preprocessing is required
185 // |ptr_out| will be pointing to |in_frame|, otherwise pointing to
186 // |preprocess_frame_|.
187 //
188 // Return value:
189 // -1: if encountering an error.
190 // 0: otherwise.
191 int PreprocessToAddData(const AudioFrame& in_frame,
192 const AudioFrame** ptr_out)
danilchap56359be2017-09-07 07:53:45 -0700193 RTC_EXCLUSIVE_LOCKS_REQUIRED(acm_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700194
195 // Change required states after starting to receive the codec corresponding
196 // to |index|.
197 int UpdateUponReceivingCodec(int index);
198
199 rtc::CriticalSection acm_crit_sect_;
danilchap56359be2017-09-07 07:53:45 -0700200 rtc::Buffer encode_buffer_ RTC_GUARDED_BY(acm_crit_sect_);
danilchap56359be2017-09-07 07:53:45 -0700201 uint32_t expected_codec_ts_ RTC_GUARDED_BY(acm_crit_sect_);
202 uint32_t expected_in_ts_ RTC_GUARDED_BY(acm_crit_sect_);
203 acm2::ACMResampler resampler_ RTC_GUARDED_BY(acm_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700204 acm2::AcmReceiver receiver_; // AcmReceiver has it's own internal lock.
danilchap56359be2017-09-07 07:53:45 -0700205 ChangeLogger bitrate_logger_ RTC_GUARDED_BY(acm_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700206
Karl Wiberg49c33ce2018-11-12 14:21:58 +0100207 // Current encoder stack, provided by a call to RegisterEncoder.
danilchap56359be2017-09-07 07:53:45 -0700208 std::unique_ptr<AudioEncoder> encoder_stack_ RTC_GUARDED_BY(acm_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700209
danilchap56359be2017-09-07 07:53:45 -0700210 std::unique_ptr<AudioDecoder> isac_decoder_16k_
211 RTC_GUARDED_BY(acm_crit_sect_);
212 std::unique_ptr<AudioDecoder> isac_decoder_32k_
213 RTC_GUARDED_BY(acm_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700214
215 // This is to keep track of CN instances where we can send DTMFs.
danilchap56359be2017-09-07 07:53:45 -0700216 uint8_t previous_pltype_ RTC_GUARDED_BY(acm_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700217
danilchap56359be2017-09-07 07:53:45 -0700218 bool receiver_initialized_ RTC_GUARDED_BY(acm_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700219
danilchap56359be2017-09-07 07:53:45 -0700220 AudioFrame preprocess_frame_ RTC_GUARDED_BY(acm_crit_sect_);
221 bool first_10ms_data_ RTC_GUARDED_BY(acm_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700222
danilchap56359be2017-09-07 07:53:45 -0700223 bool first_frame_ RTC_GUARDED_BY(acm_crit_sect_);
224 uint32_t last_timestamp_ RTC_GUARDED_BY(acm_crit_sect_);
225 uint32_t last_rtp_timestamp_ RTC_GUARDED_BY(acm_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700226
227 rtc::CriticalSection callback_crit_sect_;
228 AudioPacketizationCallback* packetization_callback_
danilchap56359be2017-09-07 07:53:45 -0700229 RTC_GUARDED_BY(callback_crit_sect_);
230 ACMVADCallback* vad_callback_ RTC_GUARDED_BY(callback_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700231
232 int codec_histogram_bins_log_[static_cast<size_t>(
233 AudioEncoder::CodecType::kMaxLoggedAudioCodecTypes)];
234 int number_of_consecutive_empty_packets_;
235};
236
237// Adds a codec usage sample to the histogram.
238void UpdateCodecTypeHistogram(size_t codec_type) {
239 RTC_HISTOGRAM_ENUMERATION(
240 "WebRTC.Audio.Encoder.CodecType", static_cast<int>(codec_type),
241 static_cast<int>(
242 webrtc::AudioEncoder::CodecType::kMaxLoggedAudioCodecTypes));
243}
244
kwibergc13ded52016-06-17 06:00:45 -0700245// Stereo-to-mono can be used as in-place.
246int DownMix(const AudioFrame& frame,
247 size_t length_out_buff,
248 int16_t* out_buff) {
yujo36b1a5f2017-06-12 12:45:32 -0700249 RTC_DCHECK_EQ(frame.num_channels_, 2);
250 RTC_DCHECK_GE(length_out_buff, frame.samples_per_channel_);
251
252 if (!frame.muted()) {
253 const int16_t* frame_data = frame.data();
254 for (size_t n = 0; n < frame.samples_per_channel_; ++n) {
Yves Gerey665174f2018-06-19 15:03:05 +0200255 out_buff[n] =
256 static_cast<int16_t>((static_cast<int32_t>(frame_data[2 * n]) +
257 static_cast<int32_t>(frame_data[2 * n + 1])) >>
258 1);
yujo36b1a5f2017-06-12 12:45:32 -0700259 }
260 } else {
Jonathan Yu36344a02017-07-30 01:55:34 -0700261 std::fill(out_buff, out_buff + frame.samples_per_channel_, 0);
kwibergc13ded52016-06-17 06:00:45 -0700262 }
kwibergc13ded52016-06-17 06:00:45 -0700263 return 0;
264}
265
266// Mono-to-stereo can be used as in-place.
267int UpMix(const AudioFrame& frame, size_t length_out_buff, int16_t* out_buff) {
yujo36b1a5f2017-06-12 12:45:32 -0700268 RTC_DCHECK_EQ(frame.num_channels_, 1);
269 RTC_DCHECK_GE(length_out_buff, 2 * frame.samples_per_channel_);
270
271 if (!frame.muted()) {
272 const int16_t* frame_data = frame.data();
273 for (size_t n = frame.samples_per_channel_; n != 0; --n) {
274 size_t i = n - 1;
275 int16_t sample = frame_data[i];
276 out_buff[2 * i + 1] = sample;
277 out_buff[2 * i] = sample;
278 }
279 } else {
Jonathan Yu36344a02017-07-30 01:55:34 -0700280 std::fill(out_buff, out_buff + frame.samples_per_channel_ * 2, 0);
kwibergc13ded52016-06-17 06:00:45 -0700281 }
282 return 0;
283}
284
285void ConvertEncodedInfoToFragmentationHeader(
286 const AudioEncoder::EncodedInfo& info,
287 RTPFragmentationHeader* frag) {
288 if (info.redundant.empty()) {
289 frag->fragmentationVectorSize = 0;
290 return;
291 }
292
293 frag->VerifyAndAllocateFragmentationHeader(
294 static_cast<uint16_t>(info.redundant.size()));
295 frag->fragmentationVectorSize = static_cast<uint16_t>(info.redundant.size());
296 size_t offset = 0;
297 for (size_t i = 0; i < info.redundant.size(); ++i) {
298 frag->fragmentationOffset[i] = offset;
299 offset += info.redundant[i].encoded_bytes;
300 frag->fragmentationLength[i] = info.redundant[i].encoded_bytes;
kwibergd3edd772017-03-01 18:52:48 -0800301 frag->fragmentationTimeDiff[i] = rtc::dchecked_cast<uint16_t>(
kwibergc13ded52016-06-17 06:00:45 -0700302 info.encoded_timestamp - info.redundant[i].encoded_timestamp);
303 frag->fragmentationPlType[i] = info.redundant[i].payload_type;
304 }
305}
306
kwibergc13ded52016-06-17 06:00:45 -0700307void AudioCodingModuleImpl::ChangeLogger::MaybeLog(int value) {
308 if (value != last_value_ || first_time_) {
309 first_time_ = false;
310 last_value_ = value;
311 RTC_HISTOGRAM_COUNTS_SPARSE_100(histogram_name_, value);
312 }
313}
314
315AudioCodingModuleImpl::AudioCodingModuleImpl(
316 const AudioCodingModule::Config& config)
solenbergc7b4a452017-09-28 07:37:11 -0700317 : expected_codec_ts_(0xD87F3F9F),
kwibergc13ded52016-06-17 06:00:45 -0700318 expected_in_ts_(0xD87F3F9F),
319 receiver_(config),
320 bitrate_logger_("WebRTC.Audio.TargetBitrateInKbps"),
kwibergc13ded52016-06-17 06:00:45 -0700321 encoder_stack_(nullptr),
322 previous_pltype_(255),
323 receiver_initialized_(false),
324 first_10ms_data_(false),
325 first_frame_(true),
326 packetization_callback_(NULL),
327 vad_callback_(NULL),
328 codec_histogram_bins_log_(),
329 number_of_consecutive_empty_packets_(0) {
330 if (InitializeReceiverSafe() < 0) {
Mirko Bonadei675513b2017-11-09 11:09:25 +0100331 RTC_LOG(LS_ERROR) << "Cannot initialize receiver";
kwibergc13ded52016-06-17 06:00:45 -0700332 }
Mirko Bonadei675513b2017-11-09 11:09:25 +0100333 RTC_LOG(LS_INFO) << "Created";
kwibergc13ded52016-06-17 06:00:45 -0700334}
335
336AudioCodingModuleImpl::~AudioCodingModuleImpl() = default;
337
338int32_t AudioCodingModuleImpl::Encode(const InputData& input_data) {
339 AudioEncoder::EncodedInfo encoded_info;
340 uint8_t previous_pltype;
341
342 // Check if there is an encoder before.
343 if (!HaveValidEncoder("Process"))
344 return -1;
345
Yves Gerey665174f2018-06-19 15:03:05 +0200346 if (!first_frame_) {
deadbeeffcada902016-08-24 12:45:13 -0700347 RTC_DCHECK(IsNewerTimestamp(input_data.input_timestamp, last_timestamp_))
ossu63fb95a2016-07-06 09:34:22 -0700348 << "Time should not move backwards";
349 }
350
kwibergc13ded52016-06-17 06:00:45 -0700351 // Scale the timestamp to the codec's RTP timestamp rate.
352 uint32_t rtp_timestamp =
353 first_frame_ ? input_data.input_timestamp
354 : last_rtp_timestamp_ +
355 rtc::CheckedDivExact(
356 input_data.input_timestamp - last_timestamp_,
357 static_cast<uint32_t>(rtc::CheckedDivExact(
358 encoder_stack_->SampleRateHz(),
359 encoder_stack_->RtpTimestampRateHz())));
360 last_timestamp_ = input_data.input_timestamp;
361 last_rtp_timestamp_ = rtp_timestamp;
362 first_frame_ = false;
363
364 // Clear the buffer before reuse - encoded data will get appended.
365 encode_buffer_.Clear();
366 encoded_info = encoder_stack_->Encode(
Yves Gerey665174f2018-06-19 15:03:05 +0200367 rtp_timestamp,
368 rtc::ArrayView<const int16_t>(
369 input_data.audio,
370 input_data.audio_channel * input_data.length_per_channel),
kwibergc13ded52016-06-17 06:00:45 -0700371 &encode_buffer_);
372
373 bitrate_logger_.MaybeLog(encoder_stack_->GetTargetBitrate() / 1000);
374 if (encode_buffer_.size() == 0 && !encoded_info.send_even_if_empty) {
375 // Not enough data.
376 return 0;
377 }
378 previous_pltype = previous_pltype_; // Read it while we have the critsect.
379
380 // Log codec type to histogram once every 500 packets.
381 if (encoded_info.encoded_bytes == 0) {
382 ++number_of_consecutive_empty_packets_;
383 } else {
384 size_t codec_type = static_cast<size_t>(encoded_info.encoder_type);
385 codec_histogram_bins_log_[codec_type] +=
386 number_of_consecutive_empty_packets_ + 1;
387 number_of_consecutive_empty_packets_ = 0;
388 if (codec_histogram_bins_log_[codec_type] >= 500) {
389 codec_histogram_bins_log_[codec_type] -= 500;
390 UpdateCodecTypeHistogram(codec_type);
391 }
392 }
393
394 RTPFragmentationHeader my_fragmentation;
395 ConvertEncodedInfoToFragmentationHeader(encoded_info, &my_fragmentation);
396 FrameType frame_type;
397 if (encode_buffer_.size() == 0 && encoded_info.send_even_if_empty) {
398 frame_type = kEmptyFrame;
399 encoded_info.payload_type = previous_pltype;
400 } else {
kwibergaf476c72016-11-28 15:21:39 -0800401 RTC_DCHECK_GT(encode_buffer_.size(), 0);
kwibergc13ded52016-06-17 06:00:45 -0700402 frame_type = encoded_info.speech ? kAudioFrameSpeech : kAudioFrameCN;
403 }
404
405 {
406 rtc::CritScope lock(&callback_crit_sect_);
407 if (packetization_callback_) {
408 packetization_callback_->SendData(
409 frame_type, encoded_info.payload_type, encoded_info.encoded_timestamp,
410 encode_buffer_.data(), encode_buffer_.size(),
411 my_fragmentation.fragmentationVectorSize > 0 ? &my_fragmentation
412 : nullptr);
413 }
414
415 if (vad_callback_) {
416 // Callback with VAD decision.
417 vad_callback_->InFrameType(frame_type);
418 }
419 }
420 previous_pltype_ = encoded_info.payload_type;
421 return static_cast<int32_t>(encode_buffer_.size());
422}
423
424/////////////////////////////////////////
425// Sender
426//
427
kwibergc13ded52016-06-17 06:00:45 -0700428void AudioCodingModuleImpl::ModifyEncoder(
kwiberg24c7c122016-09-28 11:57:10 -0700429 rtc::FunctionView<void(std::unique_ptr<AudioEncoder>*)> modifier) {
kwibergc13ded52016-06-17 06:00:45 -0700430 rtc::CritScope lock(&acm_crit_sect_);
kwibergc13ded52016-06-17 06:00:45 -0700431 modifier(&encoder_stack_);
432}
433
kwibergc13ded52016-06-17 06:00:45 -0700434void AudioCodingModuleImpl::SetBitRate(int bitrate_bps) {
435 rtc::CritScope lock(&acm_crit_sect_);
436 if (encoder_stack_) {
Danil Chapovalovb6021232018-06-19 13:26:36 +0200437 encoder_stack_->OnReceivedUplinkBandwidth(bitrate_bps, absl::nullopt);
kwibergc13ded52016-06-17 06:00:45 -0700438 }
439}
440
441// Register a transport callback which will be called to deliver
442// the encoded buffers.
443int AudioCodingModuleImpl::RegisterTransportCallback(
444 AudioPacketizationCallback* transport) {
445 rtc::CritScope lock(&callback_crit_sect_);
446 packetization_callback_ = transport;
447 return 0;
448}
449
450// Add 10MS of raw (PCM) audio data to the encoder.
451int AudioCodingModuleImpl::Add10MsData(const AudioFrame& audio_frame) {
452 InputData input_data;
453 rtc::CritScope lock(&acm_crit_sect_);
454 int r = Add10MsDataInternal(audio_frame, &input_data);
455 return r < 0 ? r : Encode(input_data);
456}
457
458int AudioCodingModuleImpl::Add10MsDataInternal(const AudioFrame& audio_frame,
459 InputData* input_data) {
460 if (audio_frame.samples_per_channel_ == 0) {
461 assert(false);
Mirko Bonadei675513b2017-11-09 11:09:25 +0100462 RTC_LOG(LS_ERROR) << "Cannot Add 10 ms audio, payload length is zero";
kwibergc13ded52016-06-17 06:00:45 -0700463 return -1;
464 }
465
466 if (audio_frame.sample_rate_hz_ > 48000) {
467 assert(false);
Mirko Bonadei675513b2017-11-09 11:09:25 +0100468 RTC_LOG(LS_ERROR) << "Cannot Add 10 ms audio, input frequency not valid";
kwibergc13ded52016-06-17 06:00:45 -0700469 return -1;
470 }
471
472 // If the length and frequency matches. We currently just support raw PCM.
473 if (static_cast<size_t>(audio_frame.sample_rate_hz_ / 100) !=
474 audio_frame.samples_per_channel_) {
Mirko Bonadei675513b2017-11-09 11:09:25 +0100475 RTC_LOG(LS_ERROR)
Alex Loiko300ec8c2017-05-30 17:23:28 +0200476 << "Cannot Add 10 ms audio, input frequency and length doesn't match";
kwibergc13ded52016-06-17 06:00:45 -0700477 return -1;
478 }
479
Alex Loiko65438812019-02-22 10:13:44 +0100480 if (audio_frame.num_channels_ != 1 && audio_frame.num_channels_ != 2 &&
481 audio_frame.num_channels_ != 4 && audio_frame.num_channels_ != 6 &&
482 audio_frame.num_channels_ != 8) {
Mirko Bonadei675513b2017-11-09 11:09:25 +0100483 RTC_LOG(LS_ERROR) << "Cannot Add 10 ms audio, invalid number of channels.";
kwibergc13ded52016-06-17 06:00:45 -0700484 return -1;
485 }
486
487 // Do we have a codec registered?
488 if (!HaveValidEncoder("Add10MsData")) {
489 return -1;
490 }
491
492 const AudioFrame* ptr_frame;
493 // Perform a resampling, also down-mix if it is required and can be
494 // performed before resampling (a down mix prior to resampling will take
495 // place if both primary and secondary encoders are mono and input is in
496 // stereo).
497 if (PreprocessToAddData(audio_frame, &ptr_frame) < 0) {
498 return -1;
499 }
500
501 // Check whether we need an up-mix or down-mix?
502 const size_t current_num_channels = encoder_stack_->NumChannels();
503 const bool same_num_channels =
504 ptr_frame->num_channels_ == current_num_channels;
505
506 if (!same_num_channels) {
507 if (ptr_frame->num_channels_ == 1) {
508 if (UpMix(*ptr_frame, WEBRTC_10MS_PCM_AUDIO, input_data->buffer) < 0)
509 return -1;
510 } else {
511 if (DownMix(*ptr_frame, WEBRTC_10MS_PCM_AUDIO, input_data->buffer) < 0)
512 return -1;
513 }
514 }
515
516 // When adding data to encoders this pointer is pointing to an audio buffer
517 // with correct number of channels.
yujo36b1a5f2017-06-12 12:45:32 -0700518 const int16_t* ptr_audio = ptr_frame->data();
kwibergc13ded52016-06-17 06:00:45 -0700519
520 // For pushing data to primary, point the |ptr_audio| to correct buffer.
521 if (!same_num_channels)
522 ptr_audio = input_data->buffer;
523
yujo36b1a5f2017-06-12 12:45:32 -0700524 // TODO(yujo): Skip encode of muted frames.
kwibergc13ded52016-06-17 06:00:45 -0700525 input_data->input_timestamp = ptr_frame->timestamp_;
526 input_data->audio = ptr_audio;
527 input_data->length_per_channel = ptr_frame->samples_per_channel_;
528 input_data->audio_channel = current_num_channels;
529
530 return 0;
531}
532
533// Perform a resampling and down-mix if required. We down-mix only if
534// encoder is mono and input is stereo. In case of dual-streaming, both
535// encoders has to be mono for down-mix to take place.
536// |*ptr_out| will point to the pre-processed audio-frame. If no pre-processing
537// is required, |*ptr_out| points to |in_frame|.
yujo36b1a5f2017-06-12 12:45:32 -0700538// TODO(yujo): Make this more efficient for muted frames.
kwibergc13ded52016-06-17 06:00:45 -0700539int AudioCodingModuleImpl::PreprocessToAddData(const AudioFrame& in_frame,
540 const AudioFrame** ptr_out) {
541 const bool resample =
542 in_frame.sample_rate_hz_ != encoder_stack_->SampleRateHz();
543
544 // This variable is true if primary codec and secondary codec (if exists)
545 // are both mono and input is stereo.
546 // TODO(henrik.lundin): This condition should probably be
547 // in_frame.num_channels_ > encoder_stack_->NumChannels()
548 const bool down_mix =
549 in_frame.num_channels_ == 2 && encoder_stack_->NumChannels() == 1;
550
551 if (!first_10ms_data_) {
552 expected_in_ts_ = in_frame.timestamp_;
553 expected_codec_ts_ = in_frame.timestamp_;
554 first_10ms_data_ = true;
555 } else if (in_frame.timestamp_ != expected_in_ts_) {
Mirko Bonadei675513b2017-11-09 11:09:25 +0100556 RTC_LOG(LS_WARNING) << "Unexpected input timestamp: " << in_frame.timestamp_
557 << ", expected: " << expected_in_ts_;
kwibergc13ded52016-06-17 06:00:45 -0700558 expected_codec_ts_ +=
559 (in_frame.timestamp_ - expected_in_ts_) *
560 static_cast<uint32_t>(
561 static_cast<double>(encoder_stack_->SampleRateHz()) /
562 static_cast<double>(in_frame.sample_rate_hz_));
563 expected_in_ts_ = in_frame.timestamp_;
564 }
565
kwibergc13ded52016-06-17 06:00:45 -0700566 if (!down_mix && !resample) {
567 // No pre-processing is required.
ossu63fb95a2016-07-06 09:34:22 -0700568 if (expected_in_ts_ == expected_codec_ts_) {
569 // If we've never resampled, we can use the input frame as-is
570 *ptr_out = &in_frame;
571 } else {
572 // Otherwise we'll need to alter the timestamp. Since in_frame is const,
573 // we'll have to make a copy of it.
574 preprocess_frame_.CopyFrom(in_frame);
575 preprocess_frame_.timestamp_ = expected_codec_ts_;
576 *ptr_out = &preprocess_frame_;
577 }
578
kwibergc13ded52016-06-17 06:00:45 -0700579 expected_in_ts_ += static_cast<uint32_t>(in_frame.samples_per_channel_);
580 expected_codec_ts_ += static_cast<uint32_t>(in_frame.samples_per_channel_);
kwibergc13ded52016-06-17 06:00:45 -0700581 return 0;
582 }
583
584 *ptr_out = &preprocess_frame_;
585 preprocess_frame_.num_channels_ = in_frame.num_channels_;
586 int16_t audio[WEBRTC_10MS_PCM_AUDIO];
yujo36b1a5f2017-06-12 12:45:32 -0700587 const int16_t* src_ptr_audio = in_frame.data();
kwibergc13ded52016-06-17 06:00:45 -0700588 if (down_mix) {
589 // If a resampling is required the output of a down-mix is written into a
590 // local buffer, otherwise, it will be written to the output frame.
Yves Gerey665174f2018-06-19 15:03:05 +0200591 int16_t* dest_ptr_audio =
592 resample ? audio : preprocess_frame_.mutable_data();
kwibergc13ded52016-06-17 06:00:45 -0700593 if (DownMix(in_frame, WEBRTC_10MS_PCM_AUDIO, dest_ptr_audio) < 0)
594 return -1;
595 preprocess_frame_.num_channels_ = 1;
596 // Set the input of the resampler is the down-mixed signal.
597 src_ptr_audio = audio;
598 }
599
600 preprocess_frame_.timestamp_ = expected_codec_ts_;
601 preprocess_frame_.samples_per_channel_ = in_frame.samples_per_channel_;
602 preprocess_frame_.sample_rate_hz_ = in_frame.sample_rate_hz_;
603 // If it is required, we have to do a resampling.
604 if (resample) {
605 // The result of the resampler is written to output frame.
yujo36b1a5f2017-06-12 12:45:32 -0700606 int16_t* dest_ptr_audio = preprocess_frame_.mutable_data();
kwibergc13ded52016-06-17 06:00:45 -0700607
608 int samples_per_channel = resampler_.Resample10Msec(
609 src_ptr_audio, in_frame.sample_rate_hz_, encoder_stack_->SampleRateHz(),
610 preprocess_frame_.num_channels_, AudioFrame::kMaxDataSizeSamples,
611 dest_ptr_audio);
612
613 if (samples_per_channel < 0) {
Mirko Bonadei675513b2017-11-09 11:09:25 +0100614 RTC_LOG(LS_ERROR) << "Cannot add 10 ms audio, resampling failed";
kwibergc13ded52016-06-17 06:00:45 -0700615 return -1;
616 }
617 preprocess_frame_.samples_per_channel_ =
618 static_cast<size_t>(samples_per_channel);
619 preprocess_frame_.sample_rate_hz_ = encoder_stack_->SampleRateHz();
620 }
621
622 expected_codec_ts_ +=
623 static_cast<uint32_t>(preprocess_frame_.samples_per_channel_);
624 expected_in_ts_ += static_cast<uint32_t>(in_frame.samples_per_channel_);
625
626 return 0;
627}
628
629/////////////////////////////////////////
kwibergc13ded52016-06-17 06:00:45 -0700630// (FEC) Forward Error Correction (codec internal)
631//
632
kwibergc13ded52016-06-17 06:00:45 -0700633int AudioCodingModuleImpl::SetPacketLossRate(int loss_rate) {
634 rtc::CritScope lock(&acm_crit_sect_);
635 if (HaveValidEncoder("SetPacketLossRate")) {
minyue4b9a2cb2016-11-30 06:49:59 -0800636 encoder_stack_->OnReceivedUplinkPacketLossFraction(loss_rate / 100.0);
kwibergc13ded52016-06-17 06:00:45 -0700637 }
638 return 0;
639}
640
641/////////////////////////////////////////
kwibergc13ded52016-06-17 06:00:45 -0700642// Receiver
643//
644
645int AudioCodingModuleImpl::InitializeReceiver() {
646 rtc::CritScope lock(&acm_crit_sect_);
647 return InitializeReceiverSafe();
648}
649
650// Initialize receiver, resets codec database etc.
651int AudioCodingModuleImpl::InitializeReceiverSafe() {
652 // If the receiver is already initialized then we want to destroy any
653 // existing decoders. After a call to this function, we should have a clean
654 // start-up.
kwiberg6b19b562016-09-20 04:02:25 -0700655 if (receiver_initialized_)
656 receiver_.RemoveAllCodecs();
kwibergc13ded52016-06-17 06:00:45 -0700657 receiver_.FlushBuffers();
658
kwibergc13ded52016-06-17 06:00:45 -0700659 receiver_initialized_ = true;
660 return 0;
661}
662
663// Get current receive frequency.
664int AudioCodingModuleImpl::ReceiveFrequency() const {
665 const auto last_packet_sample_rate = receiver_.last_packet_sample_rate_hz();
666 return last_packet_sample_rate ? *last_packet_sample_rate
667 : receiver_.last_output_sample_rate_hz();
668}
669
670// Get current playout frequency.
671int AudioCodingModuleImpl::PlayoutFrequency() const {
kwibergc13ded52016-06-17 06:00:45 -0700672 return receiver_.last_output_sample_rate_hz();
673}
674
kwiberg1c07c702017-03-27 07:15:49 -0700675void AudioCodingModuleImpl::SetReceiveCodecs(
676 const std::map<int, SdpAudioFormat>& codecs) {
677 rtc::CritScope lock(&acm_crit_sect_);
678 receiver_.SetCodecs(codecs);
679}
680
Fredrik Solenbergf693bfa2018-12-11 12:22:10 +0100681absl::optional<std::pair<int, SdpAudioFormat>>
682 AudioCodingModuleImpl::ReceiveCodec() const {
kwiberg5adaf732016-10-04 09:33:27 -0700683 rtc::CritScope lock(&acm_crit_sect_);
Fredrik Solenbergf693bfa2018-12-11 12:22:10 +0100684 return receiver_.LastDecoder();
ossue280cde2016-10-12 11:04:10 -0700685}
686
kwibergc13ded52016-06-17 06:00:45 -0700687// Incoming packet from network parsed and ready for decode.
688int AudioCodingModuleImpl::IncomingPacket(const uint8_t* incoming_payload,
689 const size_t payload_length,
Niels Möllerafb5dbb2019-02-15 15:21:47 +0100690 const RTPHeader& rtp_header) {
henrik.lundinb8c55b12017-05-10 07:38:01 -0700691 RTC_DCHECK_EQ(payload_length == 0, incoming_payload == nullptr);
kwibergc13ded52016-06-17 06:00:45 -0700692 return receiver_.InsertPacket(
693 rtp_header,
694 rtc::ArrayView<const uint8_t>(incoming_payload, payload_length));
695}
696
697// Minimum playout delay (Used for lip-sync).
698int AudioCodingModuleImpl::SetMinimumPlayoutDelay(int time_ms) {
699 if ((time_ms < 0) || (time_ms > 10000)) {
Mirko Bonadei675513b2017-11-09 11:09:25 +0100700 RTC_LOG(LS_ERROR) << "Delay must be in the range of 0-10000 milliseconds.";
kwibergc13ded52016-06-17 06:00:45 -0700701 return -1;
702 }
703 return receiver_.SetMinimumDelay(time_ms);
704}
705
706int AudioCodingModuleImpl::SetMaximumPlayoutDelay(int time_ms) {
707 if ((time_ms < 0) || (time_ms > 10000)) {
Mirko Bonadei675513b2017-11-09 11:09:25 +0100708 RTC_LOG(LS_ERROR) << "Delay must be in the range of 0-10000 milliseconds.";
kwibergc13ded52016-06-17 06:00:45 -0700709 return -1;
710 }
711 return receiver_.SetMaximumDelay(time_ms);
712}
713
Ruslan Burakov3b50f9f2019-02-06 09:45:56 +0100714bool AudioCodingModuleImpl::SetBaseMinimumPlayoutDelayMs(int delay_ms) {
715 // All necessary validation happens on NetEq level.
716 return receiver_.SetBaseMinimumDelayMs(delay_ms);
717}
718
719int AudioCodingModuleImpl::GetBaseMinimumPlayoutDelayMs() const {
720 return receiver_.GetBaseMinimumDelayMs();
721}
722
kwibergc13ded52016-06-17 06:00:45 -0700723// Get 10 milliseconds of raw audio data to play out.
724// Automatic resample to the requested frequency.
725int AudioCodingModuleImpl::PlayoutData10Ms(int desired_freq_hz,
726 AudioFrame* audio_frame,
727 bool* muted) {
728 // GetAudio always returns 10 ms, at the requested sample rate.
729 if (receiver_.GetAudio(desired_freq_hz, audio_frame, muted) != 0) {
Mirko Bonadei675513b2017-11-09 11:09:25 +0100730 RTC_LOG(LS_ERROR) << "PlayoutData failed, RecOut Failed";
kwibergc13ded52016-06-17 06:00:45 -0700731 return -1;
732 }
kwibergc13ded52016-06-17 06:00:45 -0700733 return 0;
734}
735
kwibergc13ded52016-06-17 06:00:45 -0700736/////////////////////////////////////////
737// Statistics
738//
739
740// TODO(turajs) change the return value to void. Also change the corresponding
741// NetEq function.
742int AudioCodingModuleImpl::GetNetworkStatistics(NetworkStatistics* statistics) {
743 receiver_.GetNetworkStatistics(statistics);
744 return 0;
745}
746
747int AudioCodingModuleImpl::RegisterVADCallback(ACMVADCallback* vad_callback) {
Mirko Bonadei675513b2017-11-09 11:09:25 +0100748 RTC_LOG(LS_VERBOSE) << "RegisterVADCallback()";
kwibergc13ded52016-06-17 06:00:45 -0700749 rtc::CritScope lock(&callback_crit_sect_);
750 vad_callback_ = vad_callback;
751 return 0;
752}
753
kwibergc13ded52016-06-17 06:00:45 -0700754// Informs Opus encoder of the maximum playback rate the receiver will render.
755int AudioCodingModuleImpl::SetOpusMaxPlaybackRate(int frequency_hz) {
756 rtc::CritScope lock(&acm_crit_sect_);
757 if (!HaveValidEncoder("SetOpusMaxPlaybackRate")) {
758 return -1;
759 }
760 encoder_stack_->SetMaxPlaybackRate(frequency_hz);
761 return 0;
762}
763
764int AudioCodingModuleImpl::EnableOpusDtx() {
765 rtc::CritScope lock(&acm_crit_sect_);
766 if (!HaveValidEncoder("EnableOpusDtx")) {
767 return -1;
768 }
769 return encoder_stack_->SetDtx(true) ? 0 : -1;
770}
771
772int AudioCodingModuleImpl::DisableOpusDtx() {
773 rtc::CritScope lock(&acm_crit_sect_);
774 if (!HaveValidEncoder("DisableOpusDtx")) {
775 return -1;
776 }
777 return encoder_stack_->SetDtx(false) ? 0 : -1;
778}
779
Danil Chapovalovb6021232018-06-19 13:26:36 +0200780absl::optional<uint32_t> AudioCodingModuleImpl::PlayoutTimestamp() {
kwibergc13ded52016-06-17 06:00:45 -0700781 return receiver_.GetPlayoutTimestamp();
782}
783
henrik.lundinb3f1c5d2016-08-22 15:39:53 -0700784int AudioCodingModuleImpl::FilteredCurrentDelayMs() const {
785 return receiver_.FilteredCurrentDelayMs();
786}
787
Henrik Lundinabbff892017-11-29 09:14:04 +0100788int AudioCodingModuleImpl::TargetDelayMs() const {
789 return receiver_.TargetDelayMs();
790}
791
kwibergc13ded52016-06-17 06:00:45 -0700792bool AudioCodingModuleImpl::HaveValidEncoder(const char* caller_name) const {
793 if (!encoder_stack_) {
Mirko Bonadei675513b2017-11-09 11:09:25 +0100794 RTC_LOG(LS_ERROR) << caller_name << " failed: No send codec is registered.";
kwibergc13ded52016-06-17 06:00:45 -0700795 return false;
796 }
797 return true;
798}
799
kwibergc13ded52016-06-17 06:00:45 -0700800int AudioCodingModuleImpl::EnableNack(size_t max_nack_list_size) {
801 return receiver_.EnableNack(max_nack_list_size);
802}
803
804void AudioCodingModuleImpl::DisableNack() {
805 receiver_.DisableNack();
806}
807
808std::vector<uint16_t> AudioCodingModuleImpl::GetNackList(
809 int64_t round_trip_time_ms) const {
810 return receiver_.GetNackList(round_trip_time_ms);
811}
812
kwibergc13ded52016-06-17 06:00:45 -0700813void AudioCodingModuleImpl::GetDecodingCallStatistics(
Yves Gerey665174f2018-06-19 15:03:05 +0200814 AudioDecodingCallStats* call_stats) const {
kwibergc13ded52016-06-17 06:00:45 -0700815 receiver_.GetDecodingCallStatistics(call_stats);
816}
817
ivoce1198e02017-09-08 08:13:19 -0700818ANAStats AudioCodingModuleImpl::GetANAStats() const {
819 rtc::CritScope lock(&acm_crit_sect_);
820 if (encoder_stack_)
821 return encoder_stack_->GetANAStats();
822 // If no encoder is set, return default stats.
823 return ANAStats();
824}
825
kwibergc13ded52016-06-17 06:00:45 -0700826} // namespace
827
Karl Wiberg5817d3d2018-04-06 10:06:42 +0200828AudioCodingModule::Config::Config(
829 rtc::scoped_refptr<AudioDecoderFactory> decoder_factory)
830 : neteq_config(),
831 clock(Clock::GetRealTimeClock()),
832 decoder_factory(decoder_factory) {
kwiberg36a43882016-08-29 05:33:32 -0700833 // Post-decode VAD is disabled by default in NetEq, however, Audio
834 // Conference Mixer relies on VAD decisions and fails without them.
835 neteq_config.enable_post_decode_vad = true;
836}
837
838AudioCodingModule::Config::Config(const Config&) = default;
839AudioCodingModule::Config::~Config() = default;
840
Henrik Lundin64dad832015-05-11 12:44:23 +0200841AudioCodingModule* AudioCodingModule::Create(const Config& config) {
kwibergc13ded52016-06-17 06:00:45 -0700842 return new AudioCodingModuleImpl(config);
turaj@webrtc.org7959e162013-09-12 18:30:26 +0000843}
844
turaj@webrtc.org7959e162013-09-12 18:30:26 +0000845} // namespace webrtc