blob: 113b284d5a4556a6cfb213bdaeffc6d3afc429a5 [file] [log] [blame]
henrike@webrtc.org28e20752013-07-10 00:45:36 +00001/*
kjellanderb24317b2016-02-10 07:54:43 -08002 * Copyright 2012 The WebRTC project authors. All Rights Reserved.
henrike@webrtc.org28e20752013-07-10 00:45:36 +00003 *
kjellanderb24317b2016-02-10 07:54:43 -08004 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
henrike@webrtc.org28e20752013-07-10 00:45:36 +00009 */
10
11// This file contains interfaces for MediaStream, MediaTrack and MediaSource.
12// These interfaces are used for implementing MediaStream and MediaTrack as
13// defined in http://dev.w3.org/2011/webrtc/editor/webrtc.html#stream-api. These
14// interfaces must be used only with PeerConnection. PeerConnectionManager
15// interface provides the factory methods to create MediaStream and MediaTracks.
16
Steve Anton10542f22019-01-11 09:11:00 -080017#ifndef API_MEDIA_STREAM_INTERFACE_H_
18#define API_MEDIA_STREAM_INTERFACE_H_
henrike@webrtc.org28e20752013-07-10 00:45:36 +000019
pbos9baddf22017-01-02 06:44:41 -080020#include <stddef.h>
21
henrike@webrtc.org28e20752013-07-10 00:45:36 +000022#include <string>
23#include <vector>
24
Danil Chapovalov0bc58cf2018-06-21 13:32:56 +020025#include "absl/types/optional.h"
Piotr (Peter) Slatala95ca6e12018-11-13 07:57:07 -080026#include "api/audio_options.h"
Mirko Bonadeid9708072019-01-25 20:26:48 +010027#include "api/scoped_refptr.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020028#include "api/video/video_frame.h"
Niels Möllerc6ce9c52018-05-11 11:15:30 +020029#include "api/video/video_sink_interface.h"
Niels Möller0327c2d2018-05-21 14:09:31 +020030#include "api/video/video_source_interface.h"
Ivo Creusen56d46092017-11-24 17:29:59 +010031#include "modules/audio_processing/include/audio_processing_statistics.h"
Steve Anton10542f22019-01-11 09:11:00 -080032#include "rtc_base/ref_count.h"
Mirko Bonadei66e76792019-04-02 11:33:59 +020033#include "rtc_base/system/rtc_export.h"
henrike@webrtc.org28e20752013-07-10 00:45:36 +000034
henrike@webrtc.org28e20752013-07-10 00:45:36 +000035namespace webrtc {
36
37// Generic observer interface.
38class ObserverInterface {
39 public:
40 virtual void OnChanged() = 0;
41
42 protected:
43 virtual ~ObserverInterface() {}
44};
45
46class NotifierInterface {
47 public:
48 virtual void RegisterObserver(ObserverInterface* observer) = 0;
49 virtual void UnregisterObserver(ObserverInterface* observer) = 0;
50
51 virtual ~NotifierInterface() {}
52};
53
deadbeefb10f32f2017-02-08 01:38:21 -080054// Base class for sources. A MediaStreamTrack has an underlying source that
55// provides media. A source can be shared by multiple tracks.
Mirko Bonadei66e76792019-04-02 11:33:59 +020056class RTC_EXPORT MediaSourceInterface : public rtc::RefCountInterface,
57 public NotifierInterface {
henrike@webrtc.org28e20752013-07-10 00:45:36 +000058 public:
Yves Gerey665174f2018-06-19 15:03:05 +020059 enum SourceState { kInitializing, kLive, kEnded, kMuted };
henrike@webrtc.org28e20752013-07-10 00:45:36 +000060
61 virtual SourceState state() const = 0;
62
tommi6eca7e32015-12-15 04:27:11 -080063 virtual bool remote() const = 0;
64
henrike@webrtc.org28e20752013-07-10 00:45:36 +000065 protected:
Danil Chapovalov2a5ce2b2018-02-07 09:38:31 +010066 ~MediaSourceInterface() override = default;
henrike@webrtc.org28e20752013-07-10 00:45:36 +000067};
68
deadbeefb10f32f2017-02-08 01:38:21 -080069// C++ version of MediaStreamTrack.
70// See: https://www.w3.org/TR/mediacapture-streams/#mediastreamtrack
Mirko Bonadei66e76792019-04-02 11:33:59 +020071class RTC_EXPORT MediaStreamTrackInterface : public rtc::RefCountInterface,
72 public NotifierInterface {
henrike@webrtc.org28e20752013-07-10 00:45:36 +000073 public:
74 enum TrackState {
perkjc8f952d2016-03-23 00:33:56 -070075 kLive,
76 kEnded,
henrike@webrtc.org28e20752013-07-10 00:45:36 +000077 };
78
Niels Möller6dcd4dc2019-08-26 10:45:28 +020079 static const char* const kAudioKind;
80 static const char* const kVideoKind;
deadbeeffac06552015-11-25 11:26:01 -080081
nissefcc640f2016-04-01 01:10:42 -070082 // The kind() method must return kAudioKind only if the object is a
83 // subclass of AudioTrackInterface, and kVideoKind only if the
84 // object is a subclass of VideoTrackInterface. It is typically used
85 // to protect a static_cast<> to the corresponding subclass.
henrike@webrtc.org28e20752013-07-10 00:45:36 +000086 virtual std::string kind() const = 0;
deadbeefb10f32f2017-02-08 01:38:21 -080087
88 // Track identifier.
henrike@webrtc.org28e20752013-07-10 00:45:36 +000089 virtual std::string id() const = 0;
deadbeefb10f32f2017-02-08 01:38:21 -080090
91 // A disabled track will produce silence (if audio) or black frames (if
92 // video). Can be disabled and re-enabled.
henrike@webrtc.org28e20752013-07-10 00:45:36 +000093 virtual bool enabled() const = 0;
henrike@webrtc.org28e20752013-07-10 00:45:36 +000094 virtual bool set_enabled(bool enable) = 0;
fischman@webrtc.org32001ef2013-08-12 23:26:21 +000095
deadbeefb10f32f2017-02-08 01:38:21 -080096 // Live or ended. A track will never be live again after becoming ended.
97 virtual TrackState state() const = 0;
98
fischman@webrtc.org32001ef2013-08-12 23:26:21 +000099 protected:
Danil Chapovalov2a5ce2b2018-02-07 09:38:31 +0100100 ~MediaStreamTrackInterface() override = default;
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000101};
102
deadbeefb10f32f2017-02-08 01:38:21 -0800103// VideoTrackSourceInterface is a reference counted source used for
104// VideoTracks. The same source can be used by multiple VideoTracks.
perkj773be362017-07-31 23:22:01 -0700105// VideoTrackSourceInterface is designed to be invoked on the signaling thread
106// except for rtc::VideoSourceInterface<VideoFrame> methods that will be invoked
107// on the worker thread via a VideoTrack. A custom implementation of a source
108// can inherit AdaptedVideoTrackSource instead of directly implementing this
109// interface.
Yves Gerey665174f2018-06-19 15:03:05 +0200110class VideoTrackSourceInterface : public MediaSourceInterface,
111 public rtc::VideoSourceInterface<VideoFrame> {
perkja3ede6c2016-03-08 01:27:48 +0100112 public:
nissefcc640f2016-04-01 01:10:42 -0700113 struct Stats {
114 // Original size of captured frame, before video adaptation.
115 int input_width;
116 int input_height;
117 };
perkja3ede6c2016-03-08 01:27:48 +0100118
perkj0d3eef22016-03-09 02:39:17 +0100119 // Indicates that parameters suitable for screencasts should be automatically
120 // applied to RtpSenders.
121 // TODO(perkj): Remove these once all known applications have moved to
deadbeefb10f32f2017-02-08 01:38:21 -0800122 // explicitly setting suitable parameters for screencasts and don't need this
perkj0d3eef22016-03-09 02:39:17 +0100123 // implicit behavior.
124 virtual bool is_screencast() const = 0;
125
Perc0d31e92016-03-31 17:23:39 +0200126 // Indicates that the encoder should denoise video before encoding it.
127 // If it is not set, the default configuration is used which is different
128 // depending on video codec.
perkj0d3eef22016-03-09 02:39:17 +0100129 // TODO(perkj): Remove this once denoising is done by the source, and not by
130 // the encoder.
Danil Chapovalov0bc58cf2018-06-21 13:32:56 +0200131 virtual absl::optional<bool> needs_denoising() const = 0;
perkja3ede6c2016-03-08 01:27:48 +0100132
deadbeefb10f32f2017-02-08 01:38:21 -0800133 // Returns false if no stats are available, e.g, for a remote source, or a
134 // source which has not seen its first frame yet.
135 //
136 // Implementation should avoid blocking.
nissefcc640f2016-04-01 01:10:42 -0700137 virtual bool GetStats(Stats* stats) = 0;
138
perkja3ede6c2016-03-08 01:27:48 +0100139 protected:
Danil Chapovalov2a5ce2b2018-02-07 09:38:31 +0100140 ~VideoTrackSourceInterface() override = default;
perkja3ede6c2016-03-08 01:27:48 +0100141};
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000142
perkj773be362017-07-31 23:22:01 -0700143// VideoTrackInterface is designed to be invoked on the signaling thread except
144// for rtc::VideoSourceInterface<VideoFrame> methods that must be invoked
145// on the worker thread.
146// PeerConnectionFactory::CreateVideoTrack can be used for creating a VideoTrack
147// that ensures thread safety and that all methods are called on the right
148// thread.
Yves Gerey665174f2018-06-19 15:03:05 +0200149class VideoTrackInterface : public MediaStreamTrackInterface,
150 public rtc::VideoSourceInterface<VideoFrame> {
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000151 public:
pbos5214a0a2016-12-16 15:39:11 -0800152 // Video track content hint, used to override the source is_screencast
153 // property.
Harald Alvestrandc19ab072018-06-18 08:53:10 +0200154 // See https://crbug.com/653531 and https://w3c.github.io/mst-content-hint.
155 enum class ContentHint { kNone, kFluid, kDetailed, kText };
pbos5214a0a2016-12-16 15:39:11 -0800156
mbonadei539d1042017-07-10 02:40:49 -0700157 // Register a video sink for this track. Used to connect the track to the
158 // underlying video engine.
159 void AddOrUpdateSink(rtc::VideoSinkInterface<VideoFrame>* sink,
160 const rtc::VideoSinkWants& wants) override {}
161 void RemoveSink(rtc::VideoSinkInterface<VideoFrame>* sink) override {}
162
perkja3ede6c2016-03-08 01:27:48 +0100163 virtual VideoTrackSourceInterface* GetSource() const = 0;
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000164
Danil Chapovalov2a5ce2b2018-02-07 09:38:31 +0100165 virtual ContentHint content_hint() const;
pbos5214a0a2016-12-16 15:39:11 -0800166 virtual void set_content_hint(ContentHint hint) {}
167
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000168 protected:
Danil Chapovalov2a5ce2b2018-02-07 09:38:31 +0100169 ~VideoTrackInterface() override = default;
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000170};
171
tommi6eca7e32015-12-15 04:27:11 -0800172// Interface for receiving audio data from a AudioTrack.
173class AudioTrackSinkInterface {
174 public:
175 virtual void OnData(const void* audio_data,
176 int bits_per_sample,
177 int sample_rate,
Peter Kasting69558702016-01-12 16:26:35 -0800178 size_t number_of_channels,
tommi6eca7e32015-12-15 04:27:11 -0800179 size_t number_of_frames) = 0;
180
181 protected:
182 virtual ~AudioTrackSinkInterface() {}
183};
184
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000185// AudioSourceInterface is a reference counted source used for AudioTracks.
deadbeefb10f32f2017-02-08 01:38:21 -0800186// The same source can be used by multiple AudioTracks.
Mirko Bonadei66e76792019-04-02 11:33:59 +0200187class RTC_EXPORT AudioSourceInterface : public MediaSourceInterface {
wu@webrtc.orgb9a088b2014-02-13 23:18:49 +0000188 public:
189 class AudioObserver {
190 public:
191 virtual void OnSetVolume(double volume) = 0;
192
193 protected:
194 virtual ~AudioObserver() {}
195 };
196
deadbeefb10f32f2017-02-08 01:38:21 -0800197 // TODO(deadbeef): Makes all the interfaces pure virtual after they're
198 // implemented in chromium.
199
200 // Sets the volume of the source. |volume| is in the range of [0, 10].
Tommif888bb52015-12-12 01:37:01 +0100201 // TODO(tommi): This method should be on the track and ideally volume should
202 // be applied in the track in a way that does not affect clones of the track.
wu@webrtc.orgb9a088b2014-02-13 23:18:49 +0000203 virtual void SetVolume(double volume) {}
204
deadbeefb10f32f2017-02-08 01:38:21 -0800205 // Registers/unregisters observers to the audio source.
wu@webrtc.orgb9a088b2014-02-13 23:18:49 +0000206 virtual void RegisterAudioObserver(AudioObserver* observer) {}
207 virtual void UnregisterAudioObserver(AudioObserver* observer) {}
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000208
tommi6eca7e32015-12-15 04:27:11 -0800209 // TODO(tommi): Make pure virtual.
210 virtual void AddSink(AudioTrackSinkInterface* sink) {}
211 virtual void RemoveSink(AudioTrackSinkInterface* sink) {}
Piotr (Peter) Slatala95ca6e12018-11-13 07:57:07 -0800212
213 // Returns options for the AudioSource.
214 // (for some of the settings this approach is broken, e.g. setting
215 // audio network adaptation on the source is the wrong layer of abstraction).
216 virtual const cricket::AudioOptions options() const;
mallinath@webrtc.org67ee6b92014-02-03 16:57:16 +0000217};
218
henrike@webrtc.org40b3b682014-03-03 18:30:11 +0000219// Interface of the audio processor used by the audio track to collect
220// statistics.
buildbot@webrtc.orgd4e598d2014-07-29 17:36:52 +0000221class AudioProcessorInterface : public rtc::RefCountInterface {
henrike@webrtc.org40b3b682014-03-03 18:30:11 +0000222 public:
Ivo Creusenae026092017-11-20 13:07:16 +0100223 struct AudioProcessorStatistics {
224 bool typing_noise_detected = false;
Ivo Creusen56d46092017-11-24 17:29:59 +0100225 AudioProcessingStats apm_statistics;
Ivo Creusenae026092017-11-20 13:07:16 +0100226 };
henrike@webrtc.org40b3b682014-03-03 18:30:11 +0000227
Ivo Creusenae026092017-11-20 13:07:16 +0100228 // Get audio processor statistics. The |has_remote_tracks| argument should be
229 // set if there are active remote tracks (this would usually be true during
230 // a call). If there are no remote tracks some of the stats will not be set by
231 // the AudioProcessor, because they only make sense if there is at least one
232 // remote track.
Sam Zackrisson28127632018-11-01 11:37:15 +0100233 virtual AudioProcessorStatistics GetStats(bool has_remote_tracks) = 0;
Ivo Creusenae026092017-11-20 13:07:16 +0100234
henrike@webrtc.org40b3b682014-03-03 18:30:11 +0000235 protected:
Danil Chapovalov2a5ce2b2018-02-07 09:38:31 +0100236 ~AudioProcessorInterface() override = default;
henrike@webrtc.org40b3b682014-03-03 18:30:11 +0000237};
238
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000239class AudioTrackInterface : public MediaStreamTrackInterface {
240 public:
deadbeefb10f32f2017-02-08 01:38:21 -0800241 // TODO(deadbeef): Figure out if the following interface should be const or
242 // not.
Yves Gerey665174f2018-06-19 15:03:05 +0200243 virtual AudioSourceInterface* GetSource() const = 0;
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000244
henrike@webrtc.org40b3b682014-03-03 18:30:11 +0000245 // Add/Remove a sink that will receive the audio data from the track.
246 virtual void AddSink(AudioTrackSinkInterface* sink) = 0;
247 virtual void RemoveSink(AudioTrackSinkInterface* sink) = 0;
mallinath@webrtc.org67ee6b92014-02-03 16:57:16 +0000248
henrike@webrtc.org40b3b682014-03-03 18:30:11 +0000249 // Get the signal level from the audio track.
250 // Return true on success, otherwise false.
deadbeefb10f32f2017-02-08 01:38:21 -0800251 // TODO(deadbeef): Change the interface to int GetSignalLevel() and pure
252 // virtual after it's implemented in chromium.
Danil Chapovalov2a5ce2b2018-02-07 09:38:31 +0100253 virtual bool GetSignalLevel(int* level);
henrike@webrtc.org40b3b682014-03-03 18:30:11 +0000254
deadbeef8d60a942017-02-27 14:47:33 -0800255 // Get the audio processor used by the audio track. Return null if the track
henrike@webrtc.org40b3b682014-03-03 18:30:11 +0000256 // does not have any processor.
deadbeefb10f32f2017-02-08 01:38:21 -0800257 // TODO(deadbeef): Make the interface pure virtual.
Danil Chapovalov2a5ce2b2018-02-07 09:38:31 +0100258 virtual rtc::scoped_refptr<AudioProcessorInterface> GetAudioProcessor();
henrike@webrtc.org40b3b682014-03-03 18:30:11 +0000259
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000260 protected:
Danil Chapovalov2a5ce2b2018-02-07 09:38:31 +0100261 ~AudioTrackInterface() override = default;
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000262};
263
Yves Gerey665174f2018-06-19 15:03:05 +0200264typedef std::vector<rtc::scoped_refptr<AudioTrackInterface> > AudioTrackVector;
265typedef std::vector<rtc::scoped_refptr<VideoTrackInterface> > VideoTrackVector;
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000266
deadbeefb10f32f2017-02-08 01:38:21 -0800267// C++ version of https://www.w3.org/TR/mediacapture-streams/#mediastream.
268//
269// A major difference is that remote audio/video tracks (received by a
270// PeerConnection/RtpReceiver) are not synchronized simply by adding them to
271// the same stream; a session description with the correct "a=msid" attributes
272// must be pushed down.
273//
274// Thus, this interface acts as simply a container for tracks.
buildbot@webrtc.orgd4e598d2014-07-29 17:36:52 +0000275class MediaStreamInterface : public rtc::RefCountInterface,
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000276 public NotifierInterface {
277 public:
Seth Hampson13b8bad2018-03-13 16:05:28 -0700278 virtual std::string id() const = 0;
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000279
280 virtual AudioTrackVector GetAudioTracks() = 0;
281 virtual VideoTrackVector GetVideoTracks() = 0;
Yves Gerey665174f2018-06-19 15:03:05 +0200282 virtual rtc::scoped_refptr<AudioTrackInterface> FindAudioTrack(
283 const std::string& track_id) = 0;
284 virtual rtc::scoped_refptr<VideoTrackInterface> FindVideoTrack(
285 const std::string& track_id) = 0;
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000286
287 virtual bool AddTrack(AudioTrackInterface* track) = 0;
288 virtual bool AddTrack(VideoTrackInterface* track) = 0;
289 virtual bool RemoveTrack(AudioTrackInterface* track) = 0;
290 virtual bool RemoveTrack(VideoTrackInterface* track) = 0;
291
292 protected:
Danil Chapovalov2a5ce2b2018-02-07 09:38:31 +0100293 ~MediaStreamInterface() override = default;
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000294};
295
296} // namespace webrtc
297
Steve Anton10542f22019-01-11 09:11:00 -0800298#endif // API_MEDIA_STREAM_INTERFACE_H_