blob: 0d47f21a0651e2addca3138a68d0ccf23d6bab1a [file] [log] [blame]
henrik.lundin@webrtc.orgd94659d2013-01-29 12:09:21 +00001/*
2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020011#ifndef MODULES_AUDIO_CODING_NETEQ_INCLUDE_NETEQ_H_
12#define MODULES_AUDIO_CODING_NETEQ_INCLUDE_NETEQ_H_
henrik.lundin@webrtc.orgd94659d2013-01-29 12:09:21 +000013
pbos@webrtc.org12dc1a32013-08-05 16:22:53 +000014#include <string.h> // Provide access to size_t.
15
Henrik Lundin905495c2015-05-25 16:58:41 +020016#include <string>
henrik.lundin114c1b32017-04-26 07:47:32 -070017#include <vector>
henrik.lundin@webrtc.orgd94659d2013-01-29 12:09:21 +000018
Karl Wiberg31fbb542017-10-16 12:42:38 +020019#include "api/audio_codecs/audio_decoder.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020020#include "api/optional.h"
Patrik Höglund3e113432017-12-15 14:40:10 +010021#include "api/rtp_headers.h"
Mirko Bonadei71207422017-09-15 13:58:09 +020022#include "common_types.h" // NOLINT(build/include)
Karl Wiberg31fbb542017-10-16 12:42:38 +020023#include "modules/audio_coding/neteq/neteq_decoder_enum.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020024#include "rtc_base/constructormagic.h"
25#include "rtc_base/scoped_ref_ptr.h"
Mirko Bonadei71207422017-09-15 13:58:09 +020026#include "typedefs.h" // NOLINT(build/include)
henrik.lundin@webrtc.orgd94659d2013-01-29 12:09:21 +000027
28namespace webrtc {
29
30// Forward declarations.
henrik.lundin6d8e0112016-03-04 10:34:21 -080031class AudioFrame;
ossue3525782016-05-25 07:37:43 -070032class AudioDecoderFactory;
henrik.lundin@webrtc.orgd94659d2013-01-29 12:09:21 +000033
henrik.lundin@webrtc.orgd94659d2013-01-29 12:09:21 +000034struct NetEqNetworkStatistics {
35 uint16_t current_buffer_size_ms; // Current jitter buffer size in ms.
36 uint16_t preferred_buffer_size_ms; // Target buffer size in ms.
37 uint16_t jitter_peaks_found; // 1 if adding extra delay due to peaky
38 // jitter; 0 otherwise.
39 uint16_t packet_loss_rate; // Loss rate (network + late) in Q14.
henrik.lundin@webrtc.orgd94659d2013-01-29 12:09:21 +000040 uint16_t expand_rate; // Fraction (of original stream) of synthesized
minyue@webrtc.org7d721ee2015-02-18 10:01:53 +000041 // audio inserted through expansion (in Q14).
42 uint16_t speech_expand_rate; // Fraction (of original stream) of synthesized
43 // speech inserted through expansion (in Q14).
henrik.lundin@webrtc.orgd94659d2013-01-29 12:09:21 +000044 uint16_t preemptive_rate; // Fraction of data inserted through pre-emptive
45 // expansion (in Q14).
46 uint16_t accelerate_rate; // Fraction of data removed through acceleration
47 // (in Q14).
minyue-webrtc0c3ca752017-08-23 15:59:38 +020048 uint16_t secondary_decoded_rate; // Fraction of data coming from FEC/RED
minyue@webrtc.org2c1bcf22015-02-17 10:17:09 +000049 // decoding (in Q14).
minyue-webrtc0c3ca752017-08-23 15:59:38 +020050 uint16_t secondary_discarded_rate; // Fraction of discarded FEC/RED data (in
51 // Q14).
henrik.lundin@webrtc.orgd94659d2013-01-29 12:09:21 +000052 int32_t clockdrift_ppm; // Average clock-drift in parts-per-million
53 // (positive or negative).
Peter Kastingdce40cf2015-08-24 14:52:23 -070054 size_t added_zero_samples; // Number of zero samples added in "off" mode.
Henrik Lundin1bb8cf82015-08-25 13:08:04 +020055 // Statistics for packet waiting times, i.e., the time between a packet
56 // arrives until it is decoded.
57 int mean_waiting_time_ms;
58 int median_waiting_time_ms;
59 int min_waiting_time_ms;
60 int max_waiting_time_ms;
henrik.lundin@webrtc.orgd94659d2013-01-29 12:09:21 +000061};
62
Steve Anton2dbc69f2017-08-24 17:15:13 -070063// NetEq statistics that persist over the lifetime of the class.
64// These metrics are never reset.
65struct NetEqLifetimeStatistics {
Gustaf Ullberg9a2e9062017-09-18 09:28:20 +020066 // Stats below correspond to similarly-named fields in the WebRTC stats spec.
67 // https://w3c.github.io/webrtc-stats/#dom-rtcmediastreamtrackstats
Steve Anton2dbc69f2017-08-24 17:15:13 -070068 uint64_t total_samples_received = 0;
Steve Anton2dbc69f2017-08-24 17:15:13 -070069 uint64_t concealed_samples = 0;
Gustaf Ullberg9a2e9062017-09-18 09:28:20 +020070 uint64_t concealment_events = 0;
Gustaf Ullbergb0a02072017-10-02 12:00:34 +020071 uint64_t jitter_buffer_delay_ms = 0;
Steve Anton2dbc69f2017-08-24 17:15:13 -070072};
73
henrik.lundin@webrtc.orgd94659d2013-01-29 12:09:21 +000074enum NetEqPlayoutMode {
75 kPlayoutOn,
76 kPlayoutOff,
77 kPlayoutFax,
78 kPlayoutStreaming
79};
80
81// This is the interface class for NetEq.
82class NetEq {
83 public:
henrik.lundin@webrtc.orgea257842014-08-07 12:27:37 +000084 enum BackgroundNoiseMode {
85 kBgnOn, // Default behavior with eternal noise.
86 kBgnFade, // Noise fades to zero after some time.
87 kBgnOff // Background noise is always zero.
88 };
89
henrik.lundin@webrtc.org35ead382014-04-14 18:49:17 +000090 struct Config {
91 Config()
92 : sample_rate_hz(16000),
henrik.lundin9bc26672015-11-02 03:25:57 -080093 enable_post_decode_vad(false),
henrik.lundin@webrtc.org116ed1d2014-04-28 08:20:04 +000094 max_packets_in_buffer(50),
95 // |max_delay_ms| has the same effect as calling SetMaximumDelay().
henrik.lundin@webrtc.orgea257842014-08-07 12:27:37 +000096 max_delay_ms(2000),
henrik.lundin@webrtc.org7cbc4f92014-10-07 06:37:39 +000097 background_noise_mode(kBgnOff),
Henrik Lundincf808d22015-05-27 14:33:29 +020098 playout_mode(kPlayoutOn),
99 enable_fast_accelerate(false) {}
henrik.lundin@webrtc.org35ead382014-04-14 18:49:17 +0000100
Henrik Lundin905495c2015-05-25 16:58:41 +0200101 std::string ToString() const;
102
Henrik Lundin83b5c052015-05-08 10:33:57 +0200103 int sample_rate_hz; // Initial value. Will change with input data.
henrik.lundin9bc26672015-11-02 03:25:57 -0800104 bool enable_post_decode_vad;
Peter Kastingdce40cf2015-08-24 14:52:23 -0700105 size_t max_packets_in_buffer;
henrik.lundin@webrtc.org116ed1d2014-04-28 08:20:04 +0000106 int max_delay_ms;
henrik.lundin@webrtc.orgea257842014-08-07 12:27:37 +0000107 BackgroundNoiseMode background_noise_mode;
henrik.lundin@webrtc.org7cbc4f92014-10-07 06:37:39 +0000108 NetEqPlayoutMode playout_mode;
Henrik Lundincf808d22015-05-27 14:33:29 +0200109 bool enable_fast_accelerate;
henrik.lundin7a926812016-05-12 13:51:28 -0700110 bool enable_muted_state = false;
henrik.lundin@webrtc.org35ead382014-04-14 18:49:17 +0000111 };
112
henrik.lundin@webrtc.orgd94659d2013-01-29 12:09:21 +0000113 enum ReturnCodes {
114 kOK = 0,
115 kFail = -1,
116 kNotImplemented = -2
117 };
118
henrik.lundin@webrtc.org35ead382014-04-14 18:49:17 +0000119 // Creates a new NetEq object, with parameters set in |config|. The |config|
120 // object will only have to be valid for the duration of the call to this
121 // method.
ossue3525782016-05-25 07:37:43 -0700122 static NetEq* Create(
123 const NetEq::Config& config,
124 const rtc::scoped_refptr<AudioDecoderFactory>& decoder_factory);
henrik.lundin@webrtc.orgd94659d2013-01-29 12:09:21 +0000125
126 virtual ~NetEq() {}
127
128 // Inserts a new packet into NetEq. The |receive_timestamp| is an indication
129 // of the time when the packet was received, and should be measured with
130 // the same tick rate as the RTP timestamp of the current payload.
131 // Returns 0 on success, -1 on failure.
Henrik Lundin70c09bd2017-04-24 15:56:56 +0200132 virtual int InsertPacket(const RTPHeader& rtp_header,
kwibergee2bac22015-11-11 10:34:00 -0800133 rtc::ArrayView<const uint8_t> payload,
henrik.lundin@webrtc.orgd94659d2013-01-29 12:09:21 +0000134 uint32_t receive_timestamp) = 0;
135
henrik.lundinb8c55b12017-05-10 07:38:01 -0700136 // Lets NetEq know that a packet arrived with an empty payload. This typically
137 // happens when empty packets are used for probing the network channel, and
138 // these packets use RTP sequence numbers from the same series as the actual
139 // audio packets.
140 virtual void InsertEmptyPacket(const RTPHeader& rtp_header) = 0;
141
henrik.lundin@webrtc.orgd94659d2013-01-29 12:09:21 +0000142 // Instructs NetEq to deliver 10 ms of audio data. The data is written to
henrik.lundin7dc68892016-04-06 01:03:02 -0700143 // |audio_frame|. All data in |audio_frame| is wiped; |data_|, |speech_type_|,
144 // |num_channels_|, |sample_rate_hz_|, |samples_per_channel_|, and
henrik.lundin55480f52016-03-08 02:37:57 -0800145 // |vad_activity_| are updated upon success. If an error is returned, some
henrik.lundin5fac3f02016-08-24 11:18:49 -0700146 // fields may not have been updated, or may contain inconsistent values.
henrik.lundin7a926812016-05-12 13:51:28 -0700147 // If muted state is enabled (through Config::enable_muted_state), |muted|
148 // may be set to true after a prolonged expand period. When this happens, the
149 // |data_| in |audio_frame| is not written, but should be interpreted as being
150 // all zeros.
henrik.lundin@webrtc.orgd94659d2013-01-29 12:09:21 +0000151 // Returns kOK on success, or kFail in case of an error.
henrik.lundin7a926812016-05-12 13:51:28 -0700152 virtual int GetAudio(AudioFrame* audio_frame, bool* muted) = 0;
henrik.lundin@webrtc.orgd94659d2013-01-29 12:09:21 +0000153
kwiberg1c07c702017-03-27 07:15:49 -0700154 // Replaces the current set of decoders with the given one.
155 virtual void SetCodecs(const std::map<int, SdpAudioFormat>& codecs) = 0;
156
henrik.lundin4cf61dd2015-12-09 06:20:58 -0800157 // Associates |rtp_payload_type| with |codec| and |codec_name|, and stores the
158 // information in the codec database. Returns 0 on success, -1 on failure.
159 // The name is only used to provide information back to the caller about the
160 // decoders. Hence, the name is arbitrary, and may be empty.
kwibergee1879c2015-10-29 06:20:28 -0700161 virtual int RegisterPayloadType(NetEqDecoder codec,
henrik.lundin4cf61dd2015-12-09 06:20:58 -0800162 const std::string& codec_name,
henrik.lundin@webrtc.orgd94659d2013-01-29 12:09:21 +0000163 uint8_t rtp_payload_type) = 0;
164
165 // Provides an externally created decoder object |decoder| to insert in the
166 // decoder database. The decoder implements a decoder of type |codec| and
kwiberg342f7402016-06-16 03:18:00 -0700167 // associates it with |rtp_payload_type| and |codec_name|. Returns kOK on
168 // success, kFail on failure. The name is only used to provide information
169 // back to the caller about the decoders. Hence, the name is arbitrary, and
170 // may be empty.
henrik.lundin@webrtc.orgd94659d2013-01-29 12:09:21 +0000171 virtual int RegisterExternalDecoder(AudioDecoder* decoder,
kwibergee1879c2015-10-29 06:20:28 -0700172 NetEqDecoder codec,
henrik.lundin4cf61dd2015-12-09 06:20:58 -0800173 const std::string& codec_name,
kwiberg342f7402016-06-16 03:18:00 -0700174 uint8_t rtp_payload_type) = 0;
henrik.lundin@webrtc.orgd94659d2013-01-29 12:09:21 +0000175
kwiberg5adaf732016-10-04 09:33:27 -0700176 // Associates |rtp_payload_type| with the given codec, which NetEq will
177 // instantiate when it needs it. Returns true iff successful.
178 virtual bool RegisterPayloadType(int rtp_payload_type,
179 const SdpAudioFormat& audio_format) = 0;
180
henrik.lundin@webrtc.orgd94659d2013-01-29 12:09:21 +0000181 // Removes |rtp_payload_type| from the codec database. Returns 0 on success,
Henrik Lundinc417d9e2017-06-14 12:29:03 +0200182 // -1 on failure. Removing a payload type that is not registered is ok and
183 // will not result in an error.
henrik.lundin@webrtc.orgd94659d2013-01-29 12:09:21 +0000184 virtual int RemovePayloadType(uint8_t rtp_payload_type) = 0;
185
kwiberg6b19b562016-09-20 04:02:25 -0700186 // Removes all payload types from the codec database.
187 virtual void RemoveAllPayloadTypes() = 0;
188
turaj@webrtc.orgf1efc572013-08-16 23:44:24 +0000189 // Sets a minimum delay in millisecond for packet buffer. The minimum is
190 // maintained unless a higher latency is dictated by channel condition.
191 // Returns true if the minimum is successfully applied, otherwise false is
192 // returned.
193 virtual bool SetMinimumDelay(int delay_ms) = 0;
194
195 // Sets a maximum delay in milliseconds for packet buffer. The latency will
196 // not exceed the given value, even required delay (given the channel
henrik.lundin@webrtc.org116ed1d2014-04-28 08:20:04 +0000197 // conditions) is higher. Calling this method has the same effect as setting
198 // the |max_delay_ms| value in the NetEq::Config struct.
turaj@webrtc.orgf1efc572013-08-16 23:44:24 +0000199 virtual bool SetMaximumDelay(int delay_ms) = 0;
200
201 // The smallest latency required. This is computed bases on inter-arrival
202 // time and internal NetEq logic. Note that in computing this latency none of
203 // the user defined limits (applied by calling setMinimumDelay() and/or
204 // SetMaximumDelay()) are applied.
205 virtual int LeastRequiredDelayMs() const = 0;
henrik.lundin@webrtc.orgd94659d2013-01-29 12:09:21 +0000206
207 // Not implemented.
208 virtual int SetTargetDelay() = 0;
209
henrik.lundin114c1b32017-04-26 07:47:32 -0700210 // Returns the current target delay in ms. This includes any extra delay
211 // requested through SetMinimumDelay.
Henrik Lundinabbff892017-11-29 09:14:04 +0100212 virtual int TargetDelayMs() const = 0;
henrik.lundin@webrtc.orgd94659d2013-01-29 12:09:21 +0000213
henrik.lundin9c3efd02015-08-27 13:12:22 -0700214 // Returns the current total delay (packet buffer and sync buffer) in ms.
215 virtual int CurrentDelayMs() const = 0;
henrik.lundin@webrtc.orgd94659d2013-01-29 12:09:21 +0000216
henrik.lundinb3f1c5d2016-08-22 15:39:53 -0700217 // Returns the current total delay (packet buffer and sync buffer) in ms,
218 // with smoothing applied to even out short-time fluctuations due to jitter.
219 // The packet buffer part of the delay is not updated during DTX/CNG periods.
220 virtual int FilteredCurrentDelayMs() const = 0;
221
henrik.lundin@webrtc.orgd94659d2013-01-29 12:09:21 +0000222 // Sets the playout mode to |mode|.
henrik.lundin@webrtc.org7cbc4f92014-10-07 06:37:39 +0000223 // Deprecated. Set the mode in the Config struct passed to the constructor.
224 // TODO(henrik.lundin) Delete.
henrik.lundin@webrtc.orgd94659d2013-01-29 12:09:21 +0000225 virtual void SetPlayoutMode(NetEqPlayoutMode mode) = 0;
226
227 // Returns the current playout mode.
henrik.lundin@webrtc.org7cbc4f92014-10-07 06:37:39 +0000228 // Deprecated.
229 // TODO(henrik.lundin) Delete.
henrik.lundin@webrtc.orgd94659d2013-01-29 12:09:21 +0000230 virtual NetEqPlayoutMode PlayoutMode() const = 0;
231
232 // Writes the current network statistics to |stats|. The statistics are reset
233 // after the call.
234 virtual int NetworkStatistics(NetEqNetworkStatistics* stats) = 0;
235
Steve Anton2dbc69f2017-08-24 17:15:13 -0700236 // Returns a copy of this class's lifetime statistics. These statistics are
237 // never reset.
238 virtual NetEqLifetimeStatistics GetLifetimeStatistics() const = 0;
239
henrik.lundin@webrtc.orgd94659d2013-01-29 12:09:21 +0000240 // Writes the current RTCP statistics to |stats|. The statistics are reset
241 // and a new report period is started with the call.
242 virtual void GetRtcpStatistics(RtcpStatistics* stats) = 0;
243
244 // Same as RtcpStatistics(), but does not reset anything.
245 virtual void GetRtcpStatisticsNoReset(RtcpStatistics* stats) = 0;
246
247 // Enables post-decode VAD. When enabled, GetAudio() will return
248 // kOutputVADPassive when the signal contains no speech.
249 virtual void EnableVad() = 0;
250
251 // Disables post-decode VAD.
252 virtual void DisableVad() = 0;
253
henrik.lundin9a410dd2016-04-06 01:39:22 -0700254 // Returns the RTP timestamp for the last sample delivered by GetAudio().
255 // The return value will be empty if no valid timestamp is available.
henrik.lundin15c51e32016-04-06 08:38:56 -0700256 virtual rtc::Optional<uint32_t> GetPlayoutTimestamp() const = 0;
henrik.lundin@webrtc.orgd94659d2013-01-29 12:09:21 +0000257
henrik.lundind89814b2015-11-23 06:49:25 -0800258 // Returns the sample rate in Hz of the audio produced in the last GetAudio
259 // call. If GetAudio has not been called yet, the configured sample rate
260 // (Config::sample_rate_hz) is returned.
261 virtual int last_output_sample_rate_hz() const = 0;
262
kwiberg6f0f6162016-09-20 03:07:46 -0700263 // Returns info about the decoder for the given payload type, or an empty
264 // value if we have no decoder for that payload type.
265 virtual rtc::Optional<CodecInst> GetDecoder(int payload_type) const = 0;
266
ossuf1b08da2016-09-23 02:19:43 -0700267 // Returns the decoder format for the given payload type. Returns empty if no
268 // such payload type was registered.
269 virtual rtc::Optional<SdpAudioFormat> GetDecoderFormat(
270 int payload_type) const = 0;
kwibergc4ccd4d2016-09-21 10:55:15 -0700271
henrik.lundin@webrtc.orgd94659d2013-01-29 12:09:21 +0000272 // Not implemented.
273 virtual int SetTargetNumberOfChannels() = 0;
274
275 // Not implemented.
276 virtual int SetTargetSampleRate() = 0;
277
henrik.lundin@webrtc.orgd94659d2013-01-29 12:09:21 +0000278 // Flushes both the packet buffer and the sync buffer.
279 virtual void FlushBuffers() = 0;
280
turaj@webrtc.org7df97062013-08-02 18:07:13 +0000281 // Current usage of packet-buffer and it's limits.
282 virtual void PacketBufferStatistics(int* current_num_packets,
henrik.lundin@webrtc.org116ed1d2014-04-28 08:20:04 +0000283 int* max_num_packets) const = 0;
turaj@webrtc.org7df97062013-08-02 18:07:13 +0000284
henrik.lundin48ed9302015-10-29 05:36:24 -0700285 // Enables NACK and sets the maximum size of the NACK list, which should be
286 // positive and no larger than Nack::kNackListSizeLimit. If NACK is already
287 // enabled then the maximum NACK list size is modified accordingly.
288 virtual void EnableNack(size_t max_nack_list_size) = 0;
289
290 virtual void DisableNack() = 0;
291
292 // Returns a list of RTP sequence numbers corresponding to packets to be
293 // retransmitted, given an estimate of the round-trip time in milliseconds.
294 virtual std::vector<uint16_t> GetNackList(
295 int64_t round_trip_time_ms) const = 0;
minyue@webrtc.orgd7301772013-08-29 00:58:14 +0000296
henrik.lundin114c1b32017-04-26 07:47:32 -0700297 // Returns a vector containing the timestamps of the packets that were decoded
298 // in the last GetAudio call. If no packets were decoded in the last call, the
299 // vector is empty.
300 // Mainly intended for testing.
301 virtual std::vector<uint32_t> LastDecodedTimestamps() const = 0;
302
303 // Returns the length of the audio yet to play in the sync buffer.
304 // Mainly intended for testing.
305 virtual int SyncBufferSizeMs() const = 0;
306
henrik.lundin@webrtc.orgd94659d2013-01-29 12:09:21 +0000307 protected:
308 NetEq() {}
309
310 private:
henrikg3c089d72015-09-16 05:37:44 -0700311 RTC_DISALLOW_COPY_AND_ASSIGN(NetEq);
henrik.lundin@webrtc.orgd94659d2013-01-29 12:09:21 +0000312};
313
314} // namespace webrtc
Mirko Bonadei92ea95e2017-09-15 06:47:31 +0200315#endif // MODULES_AUDIO_CODING_NETEQ_INCLUDE_NETEQ_H_