blob: 9091afd5cee7ec8c034f3760bcf2a3dc05322e39 [file] [log] [blame]
Fredrik Solenberg23fba1f2015-04-29 15:24:01 +02001/*
2 * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020011#ifndef CALL_AUDIO_RECEIVE_STREAM_H_
12#define CALL_AUDIO_RECEIVE_STREAM_H_
Fredrik Solenberg23fba1f2015-04-29 15:24:01 +020013
Fredrik Solenberg04f49312015-06-08 13:04:56 +020014#include <map>
kwibergfffa42b2016-02-23 10:46:32 -080015#include <memory>
Fredrik Solenberg23fba1f2015-04-29 15:24:01 +020016#include <string>
17#include <vector>
18
Danil Chapovalovb9b146c2018-06-15 12:28:07 +020019#include "absl/types/optional.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020020#include "api/audio_codecs/audio_decoder_factory.h"
21#include "api/call/transport.h"
Steve Anton10542f22019-01-11 09:11:00 -080022#include "api/crypto/crypto_options.h"
Niels Möller7d76a312018-10-26 12:57:07 +020023#include "api/media_transport_interface.h"
Steve Anton10542f22019-01-11 09:11:00 -080024#include "api/rtp_parameters.h"
25#include "api/rtp_receiver_interface.h"
Mirko Bonadeid9708072019-01-25 20:26:48 +010026#include "api/scoped_refptr.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020027#include "call/rtp_config.h"
Fredrik Solenberg23fba1f2015-04-29 15:24:01 +020028
29namespace webrtc {
Tommif888bb52015-12-12 01:37:01 +010030class AudioSinkInterface;
Benjamin Wright84583f62018-10-04 14:22:34 -070031class FrameDecryptorInterface;
Fredrik Solenberg04f49312015-06-08 13:04:56 +020032
pbos1ba8d392016-05-01 20:18:34 -070033class AudioReceiveStream {
Fredrik Solenberg23fba1f2015-04-29 15:24:01 +020034 public:
Fredrik Solenberg4f4ec0a2015-10-22 10:49:27 +020035 struct Stats {
Paulina Hensman11b34f42018-04-09 14:24:52 +020036 Stats();
37 ~Stats();
Fredrik Solenberg4f4ec0a2015-10-22 10:49:27 +020038 uint32_t remote_ssrc = 0;
39 int64_t bytes_rcvd = 0;
40 uint32_t packets_rcvd = 0;
41 uint32_t packets_lost = 0;
42 float fraction_lost = 0.0f;
43 std::string codec_name;
Danil Chapovalovb9b146c2018-06-15 12:28:07 +020044 absl::optional<int> codec_payload_type;
Fredrik Solenberg4f4ec0a2015-10-22 10:49:27 +020045 uint32_t ext_seqnum = 0;
46 uint32_t jitter_ms = 0;
47 uint32_t jitter_buffer_ms = 0;
48 uint32_t jitter_buffer_preferred_ms = 0;
49 uint32_t delay_estimate_ms = 0;
50 int32_t audio_level = -1;
Gustaf Ullberg9a2e9062017-09-18 09:28:20 +020051 // Stats below correspond to similarly-named fields in the WebRTC stats
52 // spec. https://w3c.github.io/webrtc-stats/#dom-rtcmediastreamtrackstats
zsteine76bd3a2017-07-14 12:17:49 -070053 double total_output_energy = 0.0;
Steve Anton2dbc69f2017-08-24 17:15:13 -070054 uint64_t total_samples_received = 0;
zsteine76bd3a2017-07-14 12:17:49 -070055 double total_output_duration = 0.0;
Steve Anton2dbc69f2017-08-24 17:15:13 -070056 uint64_t concealed_samples = 0;
Gustaf Ullberg9a2e9062017-09-18 09:28:20 +020057 uint64_t concealment_events = 0;
Gustaf Ullbergb0a02072017-10-02 12:00:34 +020058 double jitter_buffer_delay_seconds = 0.0;
Chen Xing0acffb52019-01-15 15:46:29 +010059 uint64_t jitter_buffer_emitted_count = 0;
Gustaf Ullberg9a2e9062017-09-18 09:28:20 +020060 // Stats below DO NOT correspond directly to anything in the WebRTC stats
Fredrik Solenberg4f4ec0a2015-10-22 10:49:27 +020061 float expand_rate = 0.0f;
62 float speech_expand_rate = 0.0f;
63 float secondary_decoded_rate = 0.0f;
minyue-webrtc0e320ec2017-08-28 13:51:27 +020064 float secondary_discarded_rate = 0.0f;
Fredrik Solenberg4f4ec0a2015-10-22 10:49:27 +020065 float accelerate_rate = 0.0f;
66 float preemptive_expand_rate = 0.0f;
Jakob Ivarsson352ce5c2018-11-27 12:52:16 +010067 uint64_t delayed_packet_outage_samples = 0;
Fredrik Solenberg4f4ec0a2015-10-22 10:49:27 +020068 int32_t decoding_calls_to_silence_generator = 0;
69 int32_t decoding_calls_to_neteq = 0;
70 int32_t decoding_normal = 0;
71 int32_t decoding_plc = 0;
72 int32_t decoding_cng = 0;
73 int32_t decoding_plc_cng = 0;
henrik.lundin63489782016-09-20 01:47:12 -070074 int32_t decoding_muted_output = 0;
Fredrik Solenberg4f4ec0a2015-10-22 10:49:27 +020075 int64_t capture_start_ntp_time_ms = 0;
Henrik Boström01738c62019-04-15 17:32:00 +020076 // The timestamp at which the last packet was received, i.e. the time of the
77 // local clock when it was received - not the RTP timestamp of that packet.
78 // https://w3c.github.io/webrtc-stats/#dom-rtcinboundrtpstreamstats-lastpacketreceivedtimestamp
79 absl::optional<int64_t> last_packet_received_timestamp_ms;
Ruslan Burakov8af88962018-11-22 17:21:10 +010080 uint64_t jitter_buffer_flushes = 0;
Jakob Ivarsson232b3fd2019-03-06 09:18:40 +010081 double relative_packet_arrival_delay_seconds = 0.0;
Henrik Lundin44125fa2019-04-29 17:00:46 +020082 int32_t interruption_count = 0;
83 int32_t total_interruption_duration_ms = 0;
Fredrik Solenberg4f4ec0a2015-10-22 10:49:27 +020084 };
Fredrik Solenberg04f49312015-06-08 13:04:56 +020085
Fredrik Solenberg23fba1f2015-04-29 15:24:01 +020086 struct Config {
Paulina Hensman11b34f42018-04-09 14:24:52 +020087 Config();
88 ~Config();
89
Fredrik Solenberg23fba1f2015-04-29 15:24:01 +020090 std::string ToString() const;
91
92 // Receive-stream specific RTP settings.
93 struct Rtp {
Paulina Hensman11b34f42018-04-09 14:24:52 +020094 Rtp();
95 ~Rtp();
96
Fredrik Solenberg23fba1f2015-04-29 15:24:01 +020097 std::string ToString() const;
98
99 // Synchronization source (stream identifier) to be received.
Fredrik Solenberg04f49312015-06-08 13:04:56 +0200100 uint32_t remote_ssrc = 0;
101
102 // Sender SSRC used for sending RTCP (such as receiver reports).
103 uint32_t local_ssrc = 0;
Fredrik Solenberg23fba1f2015-04-29 15:24:01 +0200104
Stefan Holmer3842c5c2016-01-12 13:55:00 +0100105 // Enable feedback for send side bandwidth estimation.
106 // See
107 // https://tools.ietf.org/html/draft-holmer-rmcat-transport-wide-cc-extensions
108 // for details.
109 bool transport_cc = false;
110
solenberg8189b022016-06-14 12:13:00 -0700111 // See NackConfig for description.
112 NackConfig nack;
113
Fredrik Solenberg23fba1f2015-04-29 15:24:01 +0200114 // RTP header extensions used for the received stream.
115 std::vector<RtpExtension> extensions;
116 } rtp;
Fredrik Solenberg04f49312015-06-08 13:04:56 +0200117
solenbergcf18b342015-10-01 08:13:42 -0700118 Transport* rtcp_send_transport = nullptr;
119
Niels Möller7d76a312018-10-26 12:57:07 +0200120 MediaTransportInterface* media_transport = nullptr;
121
Fredrik Solenberg8f5787a2018-01-11 13:52:30 +0100122 // NetEq settings.
Jakob Ivarsson647d5e62019-03-15 10:37:31 +0100123 size_t jitter_buffer_max_packets = 200;
Fredrik Solenberg8f5787a2018-01-11 13:52:30 +0100124 bool jitter_buffer_fast_accelerate = false;
Jakob Ivarsson10403ae2018-11-27 15:45:20 +0100125 int jitter_buffer_min_delay_ms = 0;
Jakob Ivarsson53eae872019-01-10 15:58:36 +0100126 bool jitter_buffer_enable_rtx_handling = false;
Fredrik Solenberg8f5787a2018-01-11 13:52:30 +0100127
pbos8fc7fa72015-07-15 08:02:58 -0700128 // Identifier for an A/V synchronization group. Empty string to disable.
129 // TODO(pbos): Synchronize streams in a sync group, not just one video
130 // stream to one audio stream. Tracked by issue webrtc:4762.
131 std::string sync_group;
132
kwibergd32bf752017-01-19 07:03:59 -0800133 // Decoder specifications for every payload type that we can receive.
134 std::map<int, SdpAudioFormat> decoder_map;
ossu29b1a8d2016-06-13 07:34:51 -0700135
136 rtc::scoped_refptr<AudioDecoderFactory> decoder_factory;
Karl Wiberg08126342018-03-20 19:18:55 +0100137
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200138 absl::optional<AudioCodecPairId> codec_pair_id;
Benjamin Wright84583f62018-10-04 14:22:34 -0700139
Benjamin Wrightbfb444c2018-10-15 10:20:24 -0700140 // Per PeerConnection crypto options.
141 webrtc::CryptoOptions crypto_options;
142
Benjamin Wright84583f62018-10-04 14:22:34 -0700143 // An optional custom frame decryptor that allows the entire frame to be
144 // decrypted in whatever way the caller choses. This is not required by
145 // default.
146 rtc::scoped_refptr<webrtc::FrameDecryptorInterface> frame_decryptor;
Fredrik Solenberg23fba1f2015-04-29 15:24:01 +0200147 };
148
Fredrik Solenberg3b903d02018-01-10 15:17:10 +0100149 // Reconfigure the stream according to the Configuration.
150 virtual void Reconfigure(const Config& config) = 0;
151
pbos1ba8d392016-05-01 20:18:34 -0700152 // Starts stream activity.
153 // When a stream is active, it can receive, process and deliver packets.
154 virtual void Start() = 0;
155 // Stops stream activity.
156 // When a stream is stopped, it can't receive, process or deliver packets.
157 virtual void Stop() = 0;
158
Fredrik Solenberg04f49312015-06-08 13:04:56 +0200159 virtual Stats GetStats() const = 0;
Tommif888bb52015-12-12 01:37:01 +0100160
161 // Sets an audio sink that receives unmixed audio from the receive stream.
Fredrik Solenberg8f5787a2018-01-11 13:52:30 +0100162 // Ownership of the sink is managed by the caller.
deadbeef884f5852016-01-15 09:20:04 -0800163 // Only one sink can be set and passing a null sink clears an existing one.
Tommif888bb52015-12-12 01:37:01 +0100164 // NOTE: Audio must still somehow be pulled through AudioTransport for audio
165 // to stream through this sink. In practice, this happens if mixed audio
166 // is being pulled+rendered and/or if audio is being pulled for the purposes
167 // of feeding to the AEC.
Fredrik Solenberg8f5787a2018-01-11 13:52:30 +0100168 virtual void SetSink(AudioSinkInterface* sink) = 0;
pbos1ba8d392016-05-01 20:18:34 -0700169
solenberg217fb662016-06-17 08:30:54 -0700170 // Sets playback gain of the stream, applied when mixing, and thus after it
171 // is potentially forwarded to any attached AudioSinkInterface implementation.
172 virtual void SetGain(float gain) = 0;
173
Ruslan Burakov3b50f9f2019-02-06 09:45:56 +0100174 // Sets a base minimum for the playout delay. Base minimum delay sets lower
175 // bound on minimum delay value determining lower bound on playout delay.
176 //
177 // Returns true if value was successfully set, false overwise.
178 virtual bool SetBaseMinimumPlayoutDelayMs(int delay_ms) = 0;
179
180 // Returns current value of base minimum delay in milliseconds.
181 virtual int GetBaseMinimumPlayoutDelayMs() const = 0;
182
hbos8d609f62017-04-10 07:39:05 -0700183 virtual std::vector<RtpSource> GetSources() const = 0;
184
pbos1ba8d392016-05-01 20:18:34 -0700185 protected:
186 virtual ~AudioReceiveStream() {}
Fredrik Solenberg23fba1f2015-04-29 15:24:01 +0200187};
Fredrik Solenberg23fba1f2015-04-29 15:24:01 +0200188} // namespace webrtc
189
Mirko Bonadei92ea95e2017-09-15 06:47:31 +0200190#endif // CALL_AUDIO_RECEIVE_STREAM_H_