blob: 35e6b855dedc328dabaa8b760ae60f7231f5ca07 [file] [log] [blame]
eladalonf1841382017-06-12 01:16:46 -07001/*
2 * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Steve Anton10542f22019-01-11 09:11:00 -080011#ifndef MEDIA_ENGINE_WEBRTC_VIDEO_ENGINE_H_
12#define MEDIA_ENGINE_WEBRTC_VIDEO_ENGINE_H_
eladalonf1841382017-06-12 01:16:46 -070013
14#include <map>
15#include <memory>
16#include <set>
17#include <string>
18#include <vector>
19
Danil Chapovalov00c71832018-06-15 15:58:38 +020020#include "absl/types/optional.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020021#include "api/call/transport.h"
Jiawei Ouc2ebe212018-11-08 10:02:56 -080022#include "api/video/video_bitrate_allocator_factory.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020023#include "api/video/video_frame.h"
Niels Möllerc6ce9c52018-05-11 11:15:30 +020024#include "api/video/video_sink_interface.h"
Niels Möller0327c2d2018-05-21 14:09:31 +020025#include "api/video/video_source_interface.h"
26#include "api/video_codecs/sdp_video_format.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020027#include "call/call.h"
28#include "call/flexfec_receive_stream.h"
29#include "call/video_receive_stream.h"
30#include "call/video_send_stream.h"
Steve Anton10542f22019-01-11 09:11:00 -080031#include "media/base/media_engine.h"
Steve Anton10542f22019-01-11 09:11:00 -080032#include "rtc_base/async_invoker.h"
33#include "rtc_base/critical_section.h"
34#include "rtc_base/network_route.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020035#include "rtc_base/thread_annotations.h"
36#include "rtc_base/thread_checker.h"
eladalonf1841382017-06-12 01:16:46 -070037
38namespace webrtc {
Magnus Jedvertd4b0c052017-09-14 10:24:54 +020039class VideoDecoderFactory;
Magnus Jedvertd4b0c052017-09-14 10:24:54 +020040class VideoEncoderFactory;
eladalonf1841382017-06-12 01:16:46 -070041struct MediaConfig;
Yves Gerey665174f2018-06-19 15:03:05 +020042} // namespace webrtc
eladalonf1841382017-06-12 01:16:46 -070043
44namespace rtc {
45class Thread;
46} // namespace rtc
47
48namespace cricket {
49
eladalonf1841382017-06-12 01:16:46 -070050class WebRtcVideoChannel;
eladalonf1841382017-06-12 01:16:46 -070051
eladalonf1841382017-06-12 01:16:46 -070052class UnsignalledSsrcHandler {
53 public:
54 enum Action {
55 kDropPacket,
56 kDeliverPacket,
57 };
58 virtual Action OnUnsignalledSsrc(WebRtcVideoChannel* channel,
59 uint32_t ssrc) = 0;
60 virtual ~UnsignalledSsrcHandler() = default;
61};
62
63// TODO(pbos): Remove, use external handlers only.
64class DefaultUnsignalledSsrcHandler : public UnsignalledSsrcHandler {
65 public:
66 DefaultUnsignalledSsrcHandler();
Yves Gerey665174f2018-06-19 15:03:05 +020067 Action OnUnsignalledSsrc(WebRtcVideoChannel* channel, uint32_t ssrc) override;
eladalonf1841382017-06-12 01:16:46 -070068
69 rtc::VideoSinkInterface<webrtc::VideoFrame>* GetDefaultSink() const;
70 void SetDefaultSink(WebRtcVideoChannel* channel,
71 rtc::VideoSinkInterface<webrtc::VideoFrame>* sink);
72
73 virtual ~DefaultUnsignalledSsrcHandler() = default;
74
75 private:
76 rtc::VideoSinkInterface<webrtc::VideoFrame>* default_sink_;
77};
78
79// WebRtcVideoEngine is used for the new native WebRTC Video API (webrtc:1667).
Sebastian Jansson84848f22018-11-16 10:40:36 +010080class WebRtcVideoEngine : public VideoEngineInterface {
eladalonf1841382017-06-12 01:16:46 -070081 public:
Magnus Jedvertd4b0c052017-09-14 10:24:54 +020082 // These video codec factories represents all video codecs, i.e. both software
83 // and external hardware codecs.
84 WebRtcVideoEngine(
85 std::unique_ptr<webrtc::VideoEncoderFactory> video_encoder_factory,
Jiawei Ouc2ebe212018-11-08 10:02:56 -080086 std::unique_ptr<webrtc::VideoDecoderFactory> video_decoder_factory,
87 std::unique_ptr<webrtc::VideoBitrateAllocatorFactory>
88 video_bitrate_allocator_factory);
Magnus Jedvertd4b0c052017-09-14 10:24:54 +020089
Sebastian Jansson84848f22018-11-16 10:40:36 +010090 ~WebRtcVideoEngine() override;
eladalonf1841382017-06-12 01:16:46 -070091
Sebastian Jansson84848f22018-11-16 10:40:36 +010092 VideoMediaChannel* CreateMediaChannel(
Benjamin Wrightbfb444c2018-10-15 10:20:24 -070093 webrtc::Call* call,
94 const MediaConfig& config,
95 const VideoOptions& options,
Sebastian Jansson84848f22018-11-16 10:40:36 +010096 const webrtc::CryptoOptions& crypto_options) override;
eladalonf1841382017-06-12 01:16:46 -070097
Sebastian Jansson84848f22018-11-16 10:40:36 +010098 std::vector<VideoCodec> codecs() const override;
99 RtpCapabilities GetCapabilities() const override;
eladalonf1841382017-06-12 01:16:46 -0700100
eladalonf1841382017-06-12 01:16:46 -0700101 private:
Magnus Jedvert59ab3532018-09-03 18:07:56 +0200102 const std::unique_ptr<webrtc::VideoDecoderFactory> decoder_factory_;
Magnus Jedvert07e0d012017-10-31 11:24:54 +0100103 const std::unique_ptr<webrtc::VideoEncoderFactory> encoder_factory_;
Jiawei Ouc2ebe212018-11-08 10:02:56 -0800104 const std::unique_ptr<webrtc::VideoBitrateAllocatorFactory>
105 bitrate_allocator_factory_;
eladalonf1841382017-06-12 01:16:46 -0700106};
107
108class WebRtcVideoChannel : public VideoMediaChannel, public webrtc::Transport {
109 public:
Jiawei Ouc2ebe212018-11-08 10:02:56 -0800110 WebRtcVideoChannel(
111 webrtc::Call* call,
112 const MediaConfig& config,
113 const VideoOptions& options,
114 const webrtc::CryptoOptions& crypto_options,
115 webrtc::VideoEncoderFactory* encoder_factory,
116 webrtc::VideoDecoderFactory* decoder_factory,
117 webrtc::VideoBitrateAllocatorFactory* bitrate_allocator_factory);
eladalonf1841382017-06-12 01:16:46 -0700118 ~WebRtcVideoChannel() override;
119
120 // VideoMediaChannel implementation
eladalonf1841382017-06-12 01:16:46 -0700121 bool SetSendParameters(const VideoSendParameters& params) override;
122 bool SetRecvParameters(const VideoRecvParameters& params) override;
123 webrtc::RtpParameters GetRtpSendParameters(uint32_t ssrc) const override;
Zach Steinba37b4b2018-01-23 15:02:36 -0800124 webrtc::RTCError SetRtpSendParameters(
125 uint32_t ssrc,
126 const webrtc::RtpParameters& parameters) override;
eladalonf1841382017-06-12 01:16:46 -0700127 webrtc::RtpParameters GetRtpReceiveParameters(uint32_t ssrc) const override;
128 bool SetRtpReceiveParameters(
129 uint32_t ssrc,
130 const webrtc::RtpParameters& parameters) override;
131 bool GetSendCodec(VideoCodec* send_codec) override;
132 bool SetSend(bool send) override;
133 bool SetVideoSend(
134 uint32_t ssrc,
eladalonf1841382017-06-12 01:16:46 -0700135 const VideoOptions* options,
136 rtc::VideoSourceInterface<webrtc::VideoFrame>* source) override;
137 bool AddSendStream(const StreamParams& sp) override;
138 bool RemoveSendStream(uint32_t ssrc) override;
139 bool AddRecvStream(const StreamParams& sp) override;
140 bool AddRecvStream(const StreamParams& sp, bool default_stream);
141 bool RemoveRecvStream(uint32_t ssrc) override;
142 bool SetSink(uint32_t ssrc,
143 rtc::VideoSinkInterface<webrtc::VideoFrame>* sink) override;
144 void FillBitrateInfo(BandwidthEstimationInfo* bwe_info) override;
145 bool GetStats(VideoMediaInfo* info) override;
146
147 void OnPacketReceived(rtc::CopyOnWriteBuffer* packet,
Niels Möllere6933812018-11-05 13:01:41 +0100148 int64_t packet_time_us) override;
eladalonf1841382017-06-12 01:16:46 -0700149 void OnRtcpReceived(rtc::CopyOnWriteBuffer* packet,
Niels Möllere6933812018-11-05 13:01:41 +0100150 int64_t packet_time_us) override;
eladalonf1841382017-06-12 01:16:46 -0700151 void OnReadyToSend(bool ready) override;
152 void OnNetworkRouteChanged(const std::string& transport_name,
153 const rtc::NetworkRoute& network_route) override;
Anton Sukhanov98a462c2018-10-17 13:15:42 -0700154 void SetInterface(NetworkInterface* iface,
155 webrtc::MediaTransportInterface* media_transport) override;
eladalonf1841382017-06-12 01:16:46 -0700156
Benjamin Wright192eeec2018-10-17 17:27:25 -0700157 // E2E Encrypted Video Frame API
158 // Set a frame decryptor to a particular ssrc that will intercept all
159 // incoming video frames and attempt to decrypt them before forwarding the
160 // result.
161 void SetFrameDecryptor(uint32_t ssrc,
162 rtc::scoped_refptr<webrtc::FrameDecryptorInterface>
163 frame_decryptor) override;
164 // Set a frame encryptor to a particular ssrc that will intercept all
165 // outgoing video frames and attempt to encrypt them and forward the result
166 // to the packetizer.
167 void SetFrameEncryptor(uint32_t ssrc,
168 rtc::scoped_refptr<webrtc::FrameEncryptorInterface>
169 frame_encryptor) override;
170
Ruslan Burakov493a6502019-02-27 15:32:48 +0100171 bool SetBaseMinimumPlayoutDelayMs(uint32_t ssrc, int delay_ms) override;
172
173 absl::optional<int> GetBaseMinimumPlayoutDelayMs(
174 uint32_t ssrc) const override;
175
eladalonf1841382017-06-12 01:16:46 -0700176 // Implemented for VideoMediaChannelTest.
Steve Antonef50b252019-03-01 15:15:38 -0800177 bool sending() const {
178 RTC_DCHECK_RUN_ON(&thread_checker_);
179 return sending_;
180 }
eladalonf1841382017-06-12 01:16:46 -0700181
Danil Chapovalov00c71832018-06-15 15:58:38 +0200182 absl::optional<uint32_t> GetDefaultReceiveStreamSsrc();
eladalonf1841382017-06-12 01:16:46 -0700183
Steve Antonef50b252019-03-01 15:15:38 -0800184 StreamParams unsignaled_stream_params() {
185 RTC_DCHECK_RUN_ON(&thread_checker_);
186 return unsignaled_stream_params_;
187 }
Seth Hampson5897a6e2018-04-03 11:16:33 -0700188
eladalonf1841382017-06-12 01:16:46 -0700189 // AdaptReason is used for expressing why a WebRtcVideoSendStream request
190 // a lower input frame size than the currently configured camera input frame
191 // size. There can be more than one reason OR:ed together.
192 enum AdaptReason {
193 ADAPTREASON_NONE = 0,
194 ADAPTREASON_CPU = 1,
195 ADAPTREASON_BANDWIDTH = 2,
196 };
197
sprang67561a62017-06-15 06:34:42 -0700198 static constexpr int kDefaultQpMax = 56;
199
Jonas Oreland49ac5952018-09-26 16:04:32 +0200200 std::vector<webrtc::RtpSource> GetSources(uint32_t ssrc) const override;
201
eladalonf1841382017-06-12 01:16:46 -0700202 private:
203 class WebRtcVideoReceiveStream;
204 struct VideoCodecSettings {
205 VideoCodecSettings();
206
207 // Checks if all members of |*this| are equal to the corresponding members
208 // of |other|.
209 bool operator==(const VideoCodecSettings& other) const;
210 bool operator!=(const VideoCodecSettings& other) const;
211
212 // Checks if all members of |a|, except |flexfec_payload_type|, are equal
213 // to the corresponding members of |b|.
214 static bool EqualsDisregardingFlexfec(const VideoCodecSettings& a,
215 const VideoCodecSettings& b);
216
217 VideoCodec codec;
218 webrtc::UlpfecConfig ulpfec;
219 int flexfec_payload_type;
220 int rtx_payload_type;
221 };
222
223 struct ChangedSendParameters {
224 // These optionals are unset if not changed.
Danil Chapovalov00c71832018-06-15 15:58:38 +0200225 absl::optional<VideoCodecSettings> codec;
226 absl::optional<std::vector<webrtc::RtpExtension>> rtp_header_extensions;
227 absl::optional<std::string> mid;
Johannes Kron9190b822018-10-29 11:22:05 +0100228 absl::optional<bool> extmap_allow_mixed;
Danil Chapovalov00c71832018-06-15 15:58:38 +0200229 absl::optional<int> max_bandwidth_bps;
230 absl::optional<bool> conference_mode;
231 absl::optional<webrtc::RtcpMode> rtcp_mode;
eladalonf1841382017-06-12 01:16:46 -0700232 };
233
234 struct ChangedRecvParameters {
235 // These optionals are unset if not changed.
Danil Chapovalov00c71832018-06-15 15:58:38 +0200236 absl::optional<std::vector<VideoCodecSettings>> codec_settings;
237 absl::optional<std::vector<webrtc::RtpExtension>> rtp_header_extensions;
eladalonf1841382017-06-12 01:16:46 -0700238 // Keep track of the FlexFEC payload type separately from |codec_settings|.
239 // This allows us to recreate the FlexfecReceiveStream separately from the
240 // VideoReceiveStream when the FlexFEC payload type is changed.
Danil Chapovalov00c71832018-06-15 15:58:38 +0200241 absl::optional<int> flexfec_payload_type;
eladalonf1841382017-06-12 01:16:46 -0700242 };
243
244 bool GetChangedSendParameters(const VideoSendParameters& params,
Steve Antonef50b252019-03-01 15:15:38 -0800245 ChangedSendParameters* changed_params) const
246 RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_);
eladalonf1841382017-06-12 01:16:46 -0700247 bool GetChangedRecvParameters(const VideoRecvParameters& params,
Steve Antonef50b252019-03-01 15:15:38 -0800248 ChangedRecvParameters* changed_params) const
249 RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_);
eladalonf1841382017-06-12 01:16:46 -0700250
251 void SetMaxSendBandwidth(int bps);
252
253 void ConfigureReceiverRtp(
254 webrtc::VideoReceiveStream::Config* config,
255 webrtc::FlexfecReceiveStream::Config* flexfec_config,
Steve Antonef50b252019-03-01 15:15:38 -0800256 const StreamParams& sp) const
257 RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_);
eladalonf1841382017-06-12 01:16:46 -0700258 bool ValidateSendSsrcAvailability(const StreamParams& sp) const
Steve Antonef50b252019-03-01 15:15:38 -0800259 RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_);
eladalonf1841382017-06-12 01:16:46 -0700260 bool ValidateReceiveSsrcAvailability(const StreamParams& sp) const
Steve Antonef50b252019-03-01 15:15:38 -0800261 RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_);
eladalonf1841382017-06-12 01:16:46 -0700262 void DeleteReceiveStream(WebRtcVideoReceiveStream* stream)
Steve Antonef50b252019-03-01 15:15:38 -0800263 RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_);
eladalonf1841382017-06-12 01:16:46 -0700264
265 static std::string CodecSettingsVectorToString(
266 const std::vector<VideoCodecSettings>& codecs);
267
268 // Wrapper for the sender part.
Christian Fremerey6c025412019-02-13 19:43:28 +0000269 class WebRtcVideoSendStream
270 : public rtc::VideoSourceInterface<webrtc::VideoFrame> {
eladalonf1841382017-06-12 01:16:46 -0700271 public:
272 WebRtcVideoSendStream(
273 webrtc::Call* call,
274 const StreamParams& sp,
275 webrtc::VideoSendStream::Config config,
276 const VideoOptions& options,
eladalonf1841382017-06-12 01:16:46 -0700277 bool enable_cpu_overuse_detection,
278 int max_bitrate_bps,
Danil Chapovalov00c71832018-06-15 15:58:38 +0200279 const absl::optional<VideoCodecSettings>& codec_settings,
280 const absl::optional<std::vector<webrtc::RtpExtension>>& rtp_extensions,
eladalonf1841382017-06-12 01:16:46 -0700281 const VideoSendParameters& send_params);
282 virtual ~WebRtcVideoSendStream();
283
284 void SetSendParameters(const ChangedSendParameters& send_params);
Zach Steinba37b4b2018-01-23 15:02:36 -0800285 webrtc::RTCError SetRtpParameters(const webrtc::RtpParameters& parameters);
eladalonf1841382017-06-12 01:16:46 -0700286 webrtc::RtpParameters GetRtpParameters() const;
287
Benjamin Wright192eeec2018-10-17 17:27:25 -0700288 void SetFrameEncryptor(
289 rtc::scoped_refptr<webrtc::FrameEncryptorInterface> frame_encryptor);
290
Christian Fremerey6c025412019-02-13 19:43:28 +0000291 // Implements rtc::VideoSourceInterface<webrtc::VideoFrame>.
292 // WebRtcVideoSendStream acts as a source to the webrtc::VideoSendStream
293 // in |stream_|. This is done to proxy VideoSinkWants from the encoder to
294 // the worker thread.
295 void AddOrUpdateSink(rtc::VideoSinkInterface<webrtc::VideoFrame>* sink,
296 const rtc::VideoSinkWants& wants) override;
297 void RemoveSink(rtc::VideoSinkInterface<webrtc::VideoFrame>* sink) override;
298
Niels Möllerff40b142018-04-09 08:49:14 +0200299 bool SetVideoSend(const VideoOptions* options,
eladalonf1841382017-06-12 01:16:46 -0700300 rtc::VideoSourceInterface<webrtc::VideoFrame>* source);
301
302 void SetSend(bool send);
303
304 const std::vector<uint32_t>& GetSsrcs() const;
305 VideoSenderInfo GetVideoSenderInfo(bool log_stats);
306 void FillBitrateInfo(BandwidthEstimationInfo* bwe_info);
307
308 private:
309 // Parameters needed to reconstruct the underlying stream.
310 // webrtc::VideoSendStream doesn't support setting a lot of options on the
311 // fly, so when those need to be changed we tear down and reconstruct with
312 // similar parameters depending on which options changed etc.
313 struct VideoSendStreamParameters {
314 VideoSendStreamParameters(
315 webrtc::VideoSendStream::Config config,
316 const VideoOptions& options,
317 int max_bitrate_bps,
Danil Chapovalov00c71832018-06-15 15:58:38 +0200318 const absl::optional<VideoCodecSettings>& codec_settings);
eladalonf1841382017-06-12 01:16:46 -0700319 webrtc::VideoSendStream::Config config;
320 VideoOptions options;
321 int max_bitrate_bps;
322 bool conference_mode;
Danil Chapovalov00c71832018-06-15 15:58:38 +0200323 absl::optional<VideoCodecSettings> codec_settings;
eladalonf1841382017-06-12 01:16:46 -0700324 // Sent resolutions + bitrates etc. by the underlying VideoSendStream,
325 // typically changes when setting a new resolution or reconfiguring
326 // bitrates.
327 webrtc::VideoEncoderConfig encoder_config;
328 };
329
eladalonf1841382017-06-12 01:16:46 -0700330 rtc::scoped_refptr<webrtc::VideoEncoderConfig::EncoderSpecificSettings>
331 ConfigureVideoEncoderSettings(const VideoCodec& codec);
Niels Möller5bf8ccd2018-03-15 14:16:11 +0100332 void SetCodec(const VideoCodecSettings& codec);
eladalonf1841382017-06-12 01:16:46 -0700333 void RecreateWebRtcStream();
334 webrtc::VideoEncoderConfig CreateVideoEncoderConfig(
335 const VideoCodec& codec) const;
336 void ReconfigureEncoder();
eladalonf1841382017-06-12 01:16:46 -0700337
338 // Calls Start or Stop according to whether or not |sending_| is true,
339 // and whether or not the encoding in |rtp_parameters_| is active.
340 void UpdateSendState();
341
Taylor Brandstetter49fcc102018-05-16 14:20:41 -0700342 webrtc::DegradationPreference GetDegradationPreference() const
343 RTC_EXCLUSIVE_LOCKS_REQUIRED(&thread_checker_);
eladalonf1841382017-06-12 01:16:46 -0700344
345 rtc::ThreadChecker thread_checker_;
346 rtc::AsyncInvoker invoker_;
347 rtc::Thread* worker_thread_;
Niels Möller1e062892018-02-07 10:18:32 +0100348 const std::vector<uint32_t> ssrcs_ RTC_GUARDED_BY(&thread_checker_);
349 const std::vector<SsrcGroup> ssrc_groups_ RTC_GUARDED_BY(&thread_checker_);
eladalonf1841382017-06-12 01:16:46 -0700350 webrtc::Call* const call_;
351 const bool enable_cpu_overuse_detection_;
352 rtc::VideoSourceInterface<webrtc::VideoFrame>* source_
Niels Möller1e062892018-02-07 10:18:32 +0100353 RTC_GUARDED_BY(&thread_checker_);
eladalonf1841382017-06-12 01:16:46 -0700354
Niels Möller1e062892018-02-07 10:18:32 +0100355 webrtc::VideoSendStream* stream_ RTC_GUARDED_BY(&thread_checker_);
Christian Fremerey6c025412019-02-13 19:43:28 +0000356 rtc::VideoSinkInterface<webrtc::VideoFrame>* encoder_sink_
357 RTC_GUARDED_BY(&thread_checker_);
eladalonf1841382017-06-12 01:16:46 -0700358 // Contains settings that are the same for all streams in the MediaChannel,
359 // such as codecs, header extensions, and the global bitrate limit for the
360 // entire channel.
Niels Möller1e062892018-02-07 10:18:32 +0100361 VideoSendStreamParameters parameters_ RTC_GUARDED_BY(&thread_checker_);
eladalonf1841382017-06-12 01:16:46 -0700362 // Contains settings that are unique for each stream, such as max_bitrate.
363 // Does *not* contain codecs, however.
364 // TODO(skvlad): Move ssrcs_ and ssrc_groups_ into rtp_parameters_.
365 // TODO(skvlad): Combine parameters_ and rtp_parameters_ once we have only
366 // one stream per MediaChannel.
Niels Möller1e062892018-02-07 10:18:32 +0100367 webrtc::RtpParameters rtp_parameters_ RTC_GUARDED_BY(&thread_checker_);
eladalonf1841382017-06-12 01:16:46 -0700368
Niels Möller1e062892018-02-07 10:18:32 +0100369 bool sending_ RTC_GUARDED_BY(&thread_checker_);
eladalonf1841382017-06-12 01:16:46 -0700370 };
371
372 // Wrapper for the receiver part, contains configs etc. that are needed to
373 // reconstruct the underlying VideoReceiveStream.
374 class WebRtcVideoReceiveStream
375 : public rtc::VideoSinkInterface<webrtc::VideoFrame> {
376 public:
377 WebRtcVideoReceiveStream(
378 webrtc::Call* call,
379 const StreamParams& sp,
380 webrtc::VideoReceiveStream::Config config,
Magnus Jedvert59ab3532018-09-03 18:07:56 +0200381 webrtc::VideoDecoderFactory* decoder_factory,
eladalonf1841382017-06-12 01:16:46 -0700382 bool default_stream,
383 const std::vector<VideoCodecSettings>& recv_codecs,
384 const webrtc::FlexfecReceiveStream::Config& flexfec_config);
385 ~WebRtcVideoReceiveStream();
386
387 const std::vector<uint32_t>& GetSsrcs() const;
Florent Castelliabe301f2018-06-12 18:33:49 +0200388
Jonas Oreland49ac5952018-09-26 16:04:32 +0200389 std::vector<webrtc::RtpSource> GetSources();
390
Florent Castelliabe301f2018-06-12 18:33:49 +0200391 // Does not return codecs, they are filled by the owning WebRtcVideoChannel.
392 webrtc::RtpParameters GetRtpParameters() const;
eladalonf1841382017-06-12 01:16:46 -0700393
394 void SetLocalSsrc(uint32_t local_ssrc);
395 // TODO(deadbeef): Move these feedback parameters into the recv parameters.
396 void SetFeedbackParameters(bool nack_enabled,
397 bool remb_enabled,
398 bool transport_cc_enabled,
399 webrtc::RtcpMode rtcp_mode);
400 void SetRecvParameters(const ChangedRecvParameters& recv_params);
401
402 void OnFrame(const webrtc::VideoFrame& frame) override;
403 bool IsDefaultStream() const;
404
Benjamin Wright192eeec2018-10-17 17:27:25 -0700405 void SetFrameDecryptor(
406 rtc::scoped_refptr<webrtc::FrameDecryptorInterface> frame_decryptor);
407
Ruslan Burakov493a6502019-02-27 15:32:48 +0100408 bool SetBaseMinimumPlayoutDelayMs(int delay_ms);
409
410 int GetBaseMinimumPlayoutDelayMs() const;
411
eladalonf1841382017-06-12 01:16:46 -0700412 void SetSink(rtc::VideoSinkInterface<webrtc::VideoFrame>* sink);
413
414 VideoReceiverInfo GetVideoReceiverInfo(bool log_stats);
415
416 private:
eladalonf1841382017-06-12 01:16:46 -0700417 void RecreateWebRtcVideoStream();
418 void MaybeRecreateWebRtcFlexfecStream();
419
eladalonc0d481a2017-08-02 07:39:07 -0700420 void MaybeAssociateFlexfecWithVideo();
421 void MaybeDissociateFlexfecFromVideo();
422
Niels Möllercbcbc222018-09-28 09:07:24 +0200423 void ConfigureCodecs(const std::vector<VideoCodecSettings>& recv_codecs);
eladalonf1841382017-06-12 01:16:46 -0700424 void ConfigureFlexfecCodec(int flexfec_payload_type);
eladalonf1841382017-06-12 01:16:46 -0700425
426 std::string GetCodecNameFromPayloadType(int payload_type);
427
428 webrtc::Call* const call_;
Niels Möllercbcbc222018-09-28 09:07:24 +0200429 const StreamParams stream_params_;
eladalonf1841382017-06-12 01:16:46 -0700430
431 // Both |stream_| and |flexfec_stream_| are managed by |this|. They are
432 // destroyed by calling call_->DestroyVideoReceiveStream and
433 // call_->DestroyFlexfecReceiveStream, respectively.
434 webrtc::VideoReceiveStream* stream_;
435 const bool default_stream_;
436 webrtc::VideoReceiveStream::Config config_;
437 webrtc::FlexfecReceiveStream::Config flexfec_config_;
438 webrtc::FlexfecReceiveStream* flexfec_stream_;
439
Niels Möllercbcbc222018-09-28 09:07:24 +0200440 webrtc::VideoDecoderFactory* const decoder_factory_;
eladalonf1841382017-06-12 01:16:46 -0700441
442 rtc::CriticalSection sink_lock_;
danilchapa37de392017-09-09 04:17:22 -0700443 rtc::VideoSinkInterface<webrtc::VideoFrame>* sink_
444 RTC_GUARDED_BY(sink_lock_);
eladalonf1841382017-06-12 01:16:46 -0700445 // Expands remote RTP timestamps to int64_t to be able to estimate how long
446 // the stream has been running.
447 rtc::TimestampWrapAroundHandler timestamp_wraparound_handler_
danilchapa37de392017-09-09 04:17:22 -0700448 RTC_GUARDED_BY(sink_lock_);
449 int64_t first_frame_timestamp_ RTC_GUARDED_BY(sink_lock_);
eladalonf1841382017-06-12 01:16:46 -0700450 // Start NTP time is estimated as current remote NTP time (estimated from
451 // RTCP) minus the elapsed time, as soon as remote NTP time is available.
danilchapa37de392017-09-09 04:17:22 -0700452 int64_t estimated_remote_start_ntp_time_ms_ RTC_GUARDED_BY(sink_lock_);
eladalonf1841382017-06-12 01:16:46 -0700453 };
454
455 void Construct(webrtc::Call* call, WebRtcVideoEngine* engine);
456
457 bool SendRtp(const uint8_t* data,
458 size_t len,
459 const webrtc::PacketOptions& options) override;
460 bool SendRtcp(const uint8_t* data, size_t len) override;
461
462 static std::vector<VideoCodecSettings> MapCodecs(
463 const std::vector<VideoCodec>& codecs);
464 // Select what video codec will be used for sending, i.e. what codec is used
465 // for local encoding, based on supported remote codecs. The first remote
466 // codec that is supported locally will be selected.
Danil Chapovalov00c71832018-06-15 15:58:38 +0200467 absl::optional<VideoCodecSettings> SelectSendVideoCodec(
Steve Antonef50b252019-03-01 15:15:38 -0800468 const std::vector<VideoCodecSettings>& remote_mapped_codecs) const
469 RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_);
eladalonf1841382017-06-12 01:16:46 -0700470
471 static bool NonFlexfecReceiveCodecsHaveChanged(
472 std::vector<VideoCodecSettings> before,
473 std::vector<VideoCodecSettings> after);
474
Steve Antonef50b252019-03-01 15:15:38 -0800475 void FillSenderStats(VideoMediaInfo* info, bool log_stats)
476 RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_);
477 void FillReceiverStats(VideoMediaInfo* info, bool log_stats)
478 RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_);
eladalonf1841382017-06-12 01:16:46 -0700479 void FillBandwidthEstimationStats(const webrtc::Call::Stats& stats,
Steve Antonef50b252019-03-01 15:15:38 -0800480 VideoMediaInfo* info)
481 RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_);
482 void FillSendAndReceiveCodecStats(VideoMediaInfo* video_media_info)
483 RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_);
eladalonf1841382017-06-12 01:16:46 -0700484
485 rtc::ThreadChecker thread_checker_;
486
Steve Antonef50b252019-03-01 15:15:38 -0800487 uint32_t rtcp_receiver_report_ssrc_ RTC_GUARDED_BY(thread_checker_);
488 bool sending_ RTC_GUARDED_BY(thread_checker_);
489 webrtc::Call* const call_ RTC_GUARDED_BY(thread_checker_);
eladalonf1841382017-06-12 01:16:46 -0700490
Steve Antonef50b252019-03-01 15:15:38 -0800491 DefaultUnsignalledSsrcHandler default_unsignalled_ssrc_handler_
492 RTC_GUARDED_BY(thread_checker_);
493 UnsignalledSsrcHandler* const unsignalled_ssrc_handler_
494 RTC_GUARDED_BY(thread_checker_);
eladalonf1841382017-06-12 01:16:46 -0700495
Ruslan Burakov493a6502019-02-27 15:32:48 +0100496 // Delay for unsignaled streams, which may be set before the stream exists.
Steve Antonef50b252019-03-01 15:15:38 -0800497 int default_recv_base_minimum_delay_ms_ RTC_GUARDED_BY(thread_checker_) = 0;
Ruslan Burakov493a6502019-02-27 15:32:48 +0100498
Steve Antonef50b252019-03-01 15:15:38 -0800499 const MediaConfig::Video video_config_ RTC_GUARDED_BY(thread_checker_);
eladalonf1841382017-06-12 01:16:46 -0700500
eladalonf1841382017-06-12 01:16:46 -0700501 // Using primary-ssrc (first ssrc) as key.
502 std::map<uint32_t, WebRtcVideoSendStream*> send_streams_
Steve Antonef50b252019-03-01 15:15:38 -0800503 RTC_GUARDED_BY(thread_checker_);
eladalonf1841382017-06-12 01:16:46 -0700504 std::map<uint32_t, WebRtcVideoReceiveStream*> receive_streams_
Steve Antonef50b252019-03-01 15:15:38 -0800505 RTC_GUARDED_BY(thread_checker_);
506 std::set<uint32_t> send_ssrcs_ RTC_GUARDED_BY(thread_checker_);
507 std::set<uint32_t> receive_ssrcs_ RTC_GUARDED_BY(thread_checker_);
eladalonf1841382017-06-12 01:16:46 -0700508
Steve Antonef50b252019-03-01 15:15:38 -0800509 absl::optional<VideoCodecSettings> send_codec_
510 RTC_GUARDED_BY(thread_checker_);
511 absl::optional<std::vector<webrtc::RtpExtension>> send_rtp_extensions_
512 RTC_GUARDED_BY(thread_checker_);
eladalonf1841382017-06-12 01:16:46 -0700513
Steve Antonef50b252019-03-01 15:15:38 -0800514 webrtc::VideoEncoderFactory* const encoder_factory_
515 RTC_GUARDED_BY(thread_checker_);
516 webrtc::VideoDecoderFactory* const decoder_factory_
517 RTC_GUARDED_BY(thread_checker_);
518 webrtc::VideoBitrateAllocatorFactory* const bitrate_allocator_factory_
519 RTC_GUARDED_BY(thread_checker_);
520 std::vector<VideoCodecSettings> recv_codecs_ RTC_GUARDED_BY(thread_checker_);
521 std::vector<webrtc::RtpExtension> recv_rtp_extensions_
522 RTC_GUARDED_BY(thread_checker_);
eladalonf1841382017-06-12 01:16:46 -0700523 // See reason for keeping track of the FlexFEC payload type separately in
524 // comment in WebRtcVideoChannel::ChangedRecvParameters.
Steve Antonef50b252019-03-01 15:15:38 -0800525 int recv_flexfec_payload_type_ RTC_GUARDED_BY(thread_checker_);
526 webrtc::BitrateConstraints bitrate_config_ RTC_GUARDED_BY(thread_checker_);
eladalonf1841382017-06-12 01:16:46 -0700527 // TODO(deadbeef): Don't duplicate information between
528 // send_params/recv_params, rtp_extensions, options, etc.
Steve Antonef50b252019-03-01 15:15:38 -0800529 VideoSendParameters send_params_ RTC_GUARDED_BY(thread_checker_);
Steve Antonef50b252019-03-01 15:15:38 -0800530 VideoOptions default_send_options_ RTC_GUARDED_BY(thread_checker_);
531 VideoRecvParameters recv_params_ RTC_GUARDED_BY(thread_checker_);
532 int64_t last_stats_log_ms_ RTC_GUARDED_BY(thread_checker_);
533 const bool discard_unknown_ssrc_packets_ RTC_GUARDED_BY(thread_checker_);
Seth Hampson5897a6e2018-04-03 11:16:33 -0700534 // This is a stream param that comes from the remote description, but wasn't
535 // signaled with any a=ssrc lines. It holds information that was signaled
536 // before the unsignaled receive stream is created when the first packet is
537 // received.
Steve Antonef50b252019-03-01 15:15:38 -0800538 StreamParams unsignaled_stream_params_ RTC_GUARDED_BY(thread_checker_);
Benjamin Wright192eeec2018-10-17 17:27:25 -0700539 // Per peer connection crypto options that last for the lifetime of the peer
540 // connection.
Steve Antonef50b252019-03-01 15:15:38 -0800541 const webrtc::CryptoOptions crypto_options_ RTC_GUARDED_BY(thread_checker_);
eladalonf1841382017-06-12 01:16:46 -0700542};
543
ilnik6b826ef2017-06-16 06:53:48 -0700544class EncoderStreamFactory
545 : public webrtc::VideoEncoderConfig::VideoStreamFactoryInterface {
546 public:
547 EncoderStreamFactory(std::string codec_name,
548 int max_qp,
Seth Hampson1370e302018-02-07 08:50:36 -0800549 bool is_screenshare,
550 bool screenshare_config_explicitly_enabled);
ilnik6b826ef2017-06-16 06:53:48 -0700551
552 private:
553 std::vector<webrtc::VideoStream> CreateEncoderStreams(
554 int width,
555 int height,
556 const webrtc::VideoEncoderConfig& encoder_config) override;
557
558 const std::string codec_name_;
559 const int max_qp_;
Seth Hampson1370e302018-02-07 08:50:36 -0800560 const bool is_screenshare_;
561 // Allows a screenshare specific configuration, which enables temporal
562 // layering and allows simulcast.
563 const bool screenshare_config_explicitly_enabled_;
ilnik6b826ef2017-06-16 06:53:48 -0700564};
565
eladalonf1841382017-06-12 01:16:46 -0700566} // namespace cricket
567
Steve Anton10542f22019-01-11 09:11:00 -0800568#endif // MEDIA_ENGINE_WEBRTC_VIDEO_ENGINE_H_