blob: c24aad6f0b09188c9a36db370376161db50be694 [file] [log] [blame]
eladalonf1841382017-06-12 01:16:46 -07001/*
2 * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Steve Anton10542f22019-01-11 09:11:00 -080011#ifndef MEDIA_ENGINE_WEBRTC_VIDEO_ENGINE_H_
12#define MEDIA_ENGINE_WEBRTC_VIDEO_ENGINE_H_
eladalonf1841382017-06-12 01:16:46 -070013
14#include <map>
15#include <memory>
16#include <set>
17#include <string>
18#include <vector>
19
Danil Chapovalov00c71832018-06-15 15:58:38 +020020#include "absl/types/optional.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020021#include "api/call/transport.h"
Jiawei Ouc2ebe212018-11-08 10:02:56 -080022#include "api/video/video_bitrate_allocator_factory.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020023#include "api/video/video_frame.h"
Niels Möllerc6ce9c52018-05-11 11:15:30 +020024#include "api/video/video_sink_interface.h"
Niels Möller0327c2d2018-05-21 14:09:31 +020025#include "api/video/video_source_interface.h"
26#include "api/video_codecs/sdp_video_format.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020027#include "call/call.h"
28#include "call/flexfec_receive_stream.h"
29#include "call/video_receive_stream.h"
30#include "call/video_send_stream.h"
Steve Anton10542f22019-01-11 09:11:00 -080031#include "media/base/media_engine.h"
Steve Anton10542f22019-01-11 09:11:00 -080032#include "rtc_base/async_invoker.h"
33#include "rtc_base/critical_section.h"
34#include "rtc_base/network_route.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020035#include "rtc_base/thread_annotations.h"
36#include "rtc_base/thread_checker.h"
eladalonf1841382017-06-12 01:16:46 -070037
38namespace webrtc {
Magnus Jedvertd4b0c052017-09-14 10:24:54 +020039class VideoDecoderFactory;
Magnus Jedvertd4b0c052017-09-14 10:24:54 +020040class VideoEncoderFactory;
eladalonf1841382017-06-12 01:16:46 -070041struct MediaConfig;
Yves Gerey665174f2018-06-19 15:03:05 +020042} // namespace webrtc
eladalonf1841382017-06-12 01:16:46 -070043
44namespace rtc {
45class Thread;
46} // namespace rtc
47
48namespace cricket {
49
eladalonf1841382017-06-12 01:16:46 -070050class WebRtcVideoChannel;
eladalonf1841382017-06-12 01:16:46 -070051
eladalonf1841382017-06-12 01:16:46 -070052class UnsignalledSsrcHandler {
53 public:
54 enum Action {
55 kDropPacket,
56 kDeliverPacket,
57 };
58 virtual Action OnUnsignalledSsrc(WebRtcVideoChannel* channel,
59 uint32_t ssrc) = 0;
60 virtual ~UnsignalledSsrcHandler() = default;
61};
62
63// TODO(pbos): Remove, use external handlers only.
64class DefaultUnsignalledSsrcHandler : public UnsignalledSsrcHandler {
65 public:
66 DefaultUnsignalledSsrcHandler();
Yves Gerey665174f2018-06-19 15:03:05 +020067 Action OnUnsignalledSsrc(WebRtcVideoChannel* channel, uint32_t ssrc) override;
eladalonf1841382017-06-12 01:16:46 -070068
69 rtc::VideoSinkInterface<webrtc::VideoFrame>* GetDefaultSink() const;
70 void SetDefaultSink(WebRtcVideoChannel* channel,
71 rtc::VideoSinkInterface<webrtc::VideoFrame>* sink);
72
73 virtual ~DefaultUnsignalledSsrcHandler() = default;
74
75 private:
76 rtc::VideoSinkInterface<webrtc::VideoFrame>* default_sink_;
77};
78
79// WebRtcVideoEngine is used for the new native WebRTC Video API (webrtc:1667).
Sebastian Jansson84848f22018-11-16 10:40:36 +010080class WebRtcVideoEngine : public VideoEngineInterface {
eladalonf1841382017-06-12 01:16:46 -070081 public:
Magnus Jedvertd4b0c052017-09-14 10:24:54 +020082 // These video codec factories represents all video codecs, i.e. both software
83 // and external hardware codecs.
84 WebRtcVideoEngine(
85 std::unique_ptr<webrtc::VideoEncoderFactory> video_encoder_factory,
Jiawei Ouc2ebe212018-11-08 10:02:56 -080086 std::unique_ptr<webrtc::VideoDecoderFactory> video_decoder_factory,
87 std::unique_ptr<webrtc::VideoBitrateAllocatorFactory>
88 video_bitrate_allocator_factory);
Magnus Jedvertd4b0c052017-09-14 10:24:54 +020089
Sebastian Jansson84848f22018-11-16 10:40:36 +010090 ~WebRtcVideoEngine() override;
eladalonf1841382017-06-12 01:16:46 -070091
Sebastian Jansson84848f22018-11-16 10:40:36 +010092 VideoMediaChannel* CreateMediaChannel(
Benjamin Wrightbfb444c2018-10-15 10:20:24 -070093 webrtc::Call* call,
94 const MediaConfig& config,
95 const VideoOptions& options,
Sebastian Jansson84848f22018-11-16 10:40:36 +010096 const webrtc::CryptoOptions& crypto_options) override;
eladalonf1841382017-06-12 01:16:46 -070097
Sebastian Jansson84848f22018-11-16 10:40:36 +010098 std::vector<VideoCodec> codecs() const override;
99 RtpCapabilities GetCapabilities() const override;
eladalonf1841382017-06-12 01:16:46 -0700100
eladalonf1841382017-06-12 01:16:46 -0700101 private:
Magnus Jedvert59ab3532018-09-03 18:07:56 +0200102 const std::unique_ptr<webrtc::VideoDecoderFactory> decoder_factory_;
Magnus Jedvert07e0d012017-10-31 11:24:54 +0100103 const std::unique_ptr<webrtc::VideoEncoderFactory> encoder_factory_;
Jiawei Ouc2ebe212018-11-08 10:02:56 -0800104 const std::unique_ptr<webrtc::VideoBitrateAllocatorFactory>
105 bitrate_allocator_factory_;
eladalonf1841382017-06-12 01:16:46 -0700106};
107
108class WebRtcVideoChannel : public VideoMediaChannel, public webrtc::Transport {
109 public:
Jiawei Ouc2ebe212018-11-08 10:02:56 -0800110 WebRtcVideoChannel(
111 webrtc::Call* call,
112 const MediaConfig& config,
113 const VideoOptions& options,
114 const webrtc::CryptoOptions& crypto_options,
115 webrtc::VideoEncoderFactory* encoder_factory,
116 webrtc::VideoDecoderFactory* decoder_factory,
117 webrtc::VideoBitrateAllocatorFactory* bitrate_allocator_factory);
eladalonf1841382017-06-12 01:16:46 -0700118 ~WebRtcVideoChannel() override;
119
120 // VideoMediaChannel implementation
121 rtc::DiffServCodePoint PreferredDscp() const override;
122
123 bool SetSendParameters(const VideoSendParameters& params) override;
124 bool SetRecvParameters(const VideoRecvParameters& params) override;
125 webrtc::RtpParameters GetRtpSendParameters(uint32_t ssrc) const override;
Zach Steinba37b4b2018-01-23 15:02:36 -0800126 webrtc::RTCError SetRtpSendParameters(
127 uint32_t ssrc,
128 const webrtc::RtpParameters& parameters) override;
eladalonf1841382017-06-12 01:16:46 -0700129 webrtc::RtpParameters GetRtpReceiveParameters(uint32_t ssrc) const override;
130 bool SetRtpReceiveParameters(
131 uint32_t ssrc,
132 const webrtc::RtpParameters& parameters) override;
133 bool GetSendCodec(VideoCodec* send_codec) override;
134 bool SetSend(bool send) override;
135 bool SetVideoSend(
136 uint32_t ssrc,
eladalonf1841382017-06-12 01:16:46 -0700137 const VideoOptions* options,
138 rtc::VideoSourceInterface<webrtc::VideoFrame>* source) override;
139 bool AddSendStream(const StreamParams& sp) override;
140 bool RemoveSendStream(uint32_t ssrc) override;
141 bool AddRecvStream(const StreamParams& sp) override;
142 bool AddRecvStream(const StreamParams& sp, bool default_stream);
143 bool RemoveRecvStream(uint32_t ssrc) override;
144 bool SetSink(uint32_t ssrc,
145 rtc::VideoSinkInterface<webrtc::VideoFrame>* sink) override;
146 void FillBitrateInfo(BandwidthEstimationInfo* bwe_info) override;
147 bool GetStats(VideoMediaInfo* info) override;
148
149 void OnPacketReceived(rtc::CopyOnWriteBuffer* packet,
Niels Möllere6933812018-11-05 13:01:41 +0100150 int64_t packet_time_us) override;
eladalonf1841382017-06-12 01:16:46 -0700151 void OnRtcpReceived(rtc::CopyOnWriteBuffer* packet,
Niels Möllere6933812018-11-05 13:01:41 +0100152 int64_t packet_time_us) override;
eladalonf1841382017-06-12 01:16:46 -0700153 void OnReadyToSend(bool ready) override;
154 void OnNetworkRouteChanged(const std::string& transport_name,
155 const rtc::NetworkRoute& network_route) override;
Anton Sukhanov98a462c2018-10-17 13:15:42 -0700156 void SetInterface(NetworkInterface* iface,
157 webrtc::MediaTransportInterface* media_transport) override;
eladalonf1841382017-06-12 01:16:46 -0700158
Benjamin Wright192eeec2018-10-17 17:27:25 -0700159 // E2E Encrypted Video Frame API
160 // Set a frame decryptor to a particular ssrc that will intercept all
161 // incoming video frames and attempt to decrypt them before forwarding the
162 // result.
163 void SetFrameDecryptor(uint32_t ssrc,
164 rtc::scoped_refptr<webrtc::FrameDecryptorInterface>
165 frame_decryptor) override;
166 // Set a frame encryptor to a particular ssrc that will intercept all
167 // outgoing video frames and attempt to encrypt them and forward the result
168 // to the packetizer.
169 void SetFrameEncryptor(uint32_t ssrc,
170 rtc::scoped_refptr<webrtc::FrameEncryptorInterface>
171 frame_encryptor) override;
172
Ruslan Burakov493a6502019-02-27 15:32:48 +0100173 bool SetBaseMinimumPlayoutDelayMs(uint32_t ssrc, int delay_ms) override;
174
175 absl::optional<int> GetBaseMinimumPlayoutDelayMs(
176 uint32_t ssrc) const override;
177
eladalonf1841382017-06-12 01:16:46 -0700178 // Implemented for VideoMediaChannelTest.
Steve Antonef50b252019-03-01 15:15:38 -0800179 bool sending() const {
180 RTC_DCHECK_RUN_ON(&thread_checker_);
181 return sending_;
182 }
eladalonf1841382017-06-12 01:16:46 -0700183
Danil Chapovalov00c71832018-06-15 15:58:38 +0200184 absl::optional<uint32_t> GetDefaultReceiveStreamSsrc();
eladalonf1841382017-06-12 01:16:46 -0700185
Steve Antonef50b252019-03-01 15:15:38 -0800186 StreamParams unsignaled_stream_params() {
187 RTC_DCHECK_RUN_ON(&thread_checker_);
188 return unsignaled_stream_params_;
189 }
Seth Hampson5897a6e2018-04-03 11:16:33 -0700190
eladalonf1841382017-06-12 01:16:46 -0700191 // AdaptReason is used for expressing why a WebRtcVideoSendStream request
192 // a lower input frame size than the currently configured camera input frame
193 // size. There can be more than one reason OR:ed together.
194 enum AdaptReason {
195 ADAPTREASON_NONE = 0,
196 ADAPTREASON_CPU = 1,
197 ADAPTREASON_BANDWIDTH = 2,
198 };
199
sprang67561a62017-06-15 06:34:42 -0700200 static constexpr int kDefaultQpMax = 56;
201
Jonas Oreland49ac5952018-09-26 16:04:32 +0200202 std::vector<webrtc::RtpSource> GetSources(uint32_t ssrc) const override;
203
eladalonf1841382017-06-12 01:16:46 -0700204 private:
205 class WebRtcVideoReceiveStream;
206 struct VideoCodecSettings {
207 VideoCodecSettings();
208
209 // Checks if all members of |*this| are equal to the corresponding members
210 // of |other|.
211 bool operator==(const VideoCodecSettings& other) const;
212 bool operator!=(const VideoCodecSettings& other) const;
213
214 // Checks if all members of |a|, except |flexfec_payload_type|, are equal
215 // to the corresponding members of |b|.
216 static bool EqualsDisregardingFlexfec(const VideoCodecSettings& a,
217 const VideoCodecSettings& b);
218
219 VideoCodec codec;
220 webrtc::UlpfecConfig ulpfec;
221 int flexfec_payload_type;
222 int rtx_payload_type;
223 };
224
225 struct ChangedSendParameters {
226 // These optionals are unset if not changed.
Danil Chapovalov00c71832018-06-15 15:58:38 +0200227 absl::optional<VideoCodecSettings> codec;
228 absl::optional<std::vector<webrtc::RtpExtension>> rtp_header_extensions;
229 absl::optional<std::string> mid;
Johannes Kron9190b822018-10-29 11:22:05 +0100230 absl::optional<bool> extmap_allow_mixed;
Danil Chapovalov00c71832018-06-15 15:58:38 +0200231 absl::optional<int> max_bandwidth_bps;
232 absl::optional<bool> conference_mode;
233 absl::optional<webrtc::RtcpMode> rtcp_mode;
eladalonf1841382017-06-12 01:16:46 -0700234 };
235
236 struct ChangedRecvParameters {
237 // These optionals are unset if not changed.
Danil Chapovalov00c71832018-06-15 15:58:38 +0200238 absl::optional<std::vector<VideoCodecSettings>> codec_settings;
239 absl::optional<std::vector<webrtc::RtpExtension>> rtp_header_extensions;
eladalonf1841382017-06-12 01:16:46 -0700240 // Keep track of the FlexFEC payload type separately from |codec_settings|.
241 // This allows us to recreate the FlexfecReceiveStream separately from the
242 // VideoReceiveStream when the FlexFEC payload type is changed.
Danil Chapovalov00c71832018-06-15 15:58:38 +0200243 absl::optional<int> flexfec_payload_type;
eladalonf1841382017-06-12 01:16:46 -0700244 };
245
246 bool GetChangedSendParameters(const VideoSendParameters& params,
Steve Antonef50b252019-03-01 15:15:38 -0800247 ChangedSendParameters* changed_params) const
248 RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_);
eladalonf1841382017-06-12 01:16:46 -0700249 bool GetChangedRecvParameters(const VideoRecvParameters& params,
Steve Antonef50b252019-03-01 15:15:38 -0800250 ChangedRecvParameters* changed_params) const
251 RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_);
eladalonf1841382017-06-12 01:16:46 -0700252
253 void SetMaxSendBandwidth(int bps);
254
255 void ConfigureReceiverRtp(
256 webrtc::VideoReceiveStream::Config* config,
257 webrtc::FlexfecReceiveStream::Config* flexfec_config,
Steve Antonef50b252019-03-01 15:15:38 -0800258 const StreamParams& sp) const
259 RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_);
eladalonf1841382017-06-12 01:16:46 -0700260 bool ValidateSendSsrcAvailability(const StreamParams& sp) const
Steve Antonef50b252019-03-01 15:15:38 -0800261 RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_);
eladalonf1841382017-06-12 01:16:46 -0700262 bool ValidateReceiveSsrcAvailability(const StreamParams& sp) const
Steve Antonef50b252019-03-01 15:15:38 -0800263 RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_);
eladalonf1841382017-06-12 01:16:46 -0700264 void DeleteReceiveStream(WebRtcVideoReceiveStream* stream)
Steve Antonef50b252019-03-01 15:15:38 -0800265 RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_);
eladalonf1841382017-06-12 01:16:46 -0700266
267 static std::string CodecSettingsVectorToString(
268 const std::vector<VideoCodecSettings>& codecs);
269
270 // Wrapper for the sender part.
Christian Fremerey6c025412019-02-13 19:43:28 +0000271 class WebRtcVideoSendStream
272 : public rtc::VideoSourceInterface<webrtc::VideoFrame> {
eladalonf1841382017-06-12 01:16:46 -0700273 public:
274 WebRtcVideoSendStream(
275 webrtc::Call* call,
276 const StreamParams& sp,
277 webrtc::VideoSendStream::Config config,
278 const VideoOptions& options,
eladalonf1841382017-06-12 01:16:46 -0700279 bool enable_cpu_overuse_detection,
280 int max_bitrate_bps,
Danil Chapovalov00c71832018-06-15 15:58:38 +0200281 const absl::optional<VideoCodecSettings>& codec_settings,
282 const absl::optional<std::vector<webrtc::RtpExtension>>& rtp_extensions,
eladalonf1841382017-06-12 01:16:46 -0700283 const VideoSendParameters& send_params);
284 virtual ~WebRtcVideoSendStream();
285
286 void SetSendParameters(const ChangedSendParameters& send_params);
Zach Steinba37b4b2018-01-23 15:02:36 -0800287 webrtc::RTCError SetRtpParameters(const webrtc::RtpParameters& parameters);
eladalonf1841382017-06-12 01:16:46 -0700288 webrtc::RtpParameters GetRtpParameters() const;
289
Benjamin Wright192eeec2018-10-17 17:27:25 -0700290 void SetFrameEncryptor(
291 rtc::scoped_refptr<webrtc::FrameEncryptorInterface> frame_encryptor);
292
Christian Fremerey6c025412019-02-13 19:43:28 +0000293 // Implements rtc::VideoSourceInterface<webrtc::VideoFrame>.
294 // WebRtcVideoSendStream acts as a source to the webrtc::VideoSendStream
295 // in |stream_|. This is done to proxy VideoSinkWants from the encoder to
296 // the worker thread.
297 void AddOrUpdateSink(rtc::VideoSinkInterface<webrtc::VideoFrame>* sink,
298 const rtc::VideoSinkWants& wants) override;
299 void RemoveSink(rtc::VideoSinkInterface<webrtc::VideoFrame>* sink) override;
300
Niels Möllerff40b142018-04-09 08:49:14 +0200301 bool SetVideoSend(const VideoOptions* options,
eladalonf1841382017-06-12 01:16:46 -0700302 rtc::VideoSourceInterface<webrtc::VideoFrame>* source);
303
304 void SetSend(bool send);
305
306 const std::vector<uint32_t>& GetSsrcs() const;
307 VideoSenderInfo GetVideoSenderInfo(bool log_stats);
308 void FillBitrateInfo(BandwidthEstimationInfo* bwe_info);
309
310 private:
311 // Parameters needed to reconstruct the underlying stream.
312 // webrtc::VideoSendStream doesn't support setting a lot of options on the
313 // fly, so when those need to be changed we tear down and reconstruct with
314 // similar parameters depending on which options changed etc.
315 struct VideoSendStreamParameters {
316 VideoSendStreamParameters(
317 webrtc::VideoSendStream::Config config,
318 const VideoOptions& options,
319 int max_bitrate_bps,
Danil Chapovalov00c71832018-06-15 15:58:38 +0200320 const absl::optional<VideoCodecSettings>& codec_settings);
eladalonf1841382017-06-12 01:16:46 -0700321 webrtc::VideoSendStream::Config config;
322 VideoOptions options;
323 int max_bitrate_bps;
324 bool conference_mode;
Danil Chapovalov00c71832018-06-15 15:58:38 +0200325 absl::optional<VideoCodecSettings> codec_settings;
eladalonf1841382017-06-12 01:16:46 -0700326 // Sent resolutions + bitrates etc. by the underlying VideoSendStream,
327 // typically changes when setting a new resolution or reconfiguring
328 // bitrates.
329 webrtc::VideoEncoderConfig encoder_config;
330 };
331
eladalonf1841382017-06-12 01:16:46 -0700332 rtc::scoped_refptr<webrtc::VideoEncoderConfig::EncoderSpecificSettings>
333 ConfigureVideoEncoderSettings(const VideoCodec& codec);
Niels Möller5bf8ccd2018-03-15 14:16:11 +0100334 void SetCodec(const VideoCodecSettings& codec);
eladalonf1841382017-06-12 01:16:46 -0700335 void RecreateWebRtcStream();
336 webrtc::VideoEncoderConfig CreateVideoEncoderConfig(
337 const VideoCodec& codec) const;
338 void ReconfigureEncoder();
eladalonf1841382017-06-12 01:16:46 -0700339
340 // Calls Start or Stop according to whether or not |sending_| is true,
341 // and whether or not the encoding in |rtp_parameters_| is active.
342 void UpdateSendState();
343
Taylor Brandstetter49fcc102018-05-16 14:20:41 -0700344 webrtc::DegradationPreference GetDegradationPreference() const
345 RTC_EXCLUSIVE_LOCKS_REQUIRED(&thread_checker_);
eladalonf1841382017-06-12 01:16:46 -0700346
347 rtc::ThreadChecker thread_checker_;
348 rtc::AsyncInvoker invoker_;
349 rtc::Thread* worker_thread_;
Niels Möller1e062892018-02-07 10:18:32 +0100350 const std::vector<uint32_t> ssrcs_ RTC_GUARDED_BY(&thread_checker_);
351 const std::vector<SsrcGroup> ssrc_groups_ RTC_GUARDED_BY(&thread_checker_);
eladalonf1841382017-06-12 01:16:46 -0700352 webrtc::Call* const call_;
353 const bool enable_cpu_overuse_detection_;
354 rtc::VideoSourceInterface<webrtc::VideoFrame>* source_
Niels Möller1e062892018-02-07 10:18:32 +0100355 RTC_GUARDED_BY(&thread_checker_);
eladalonf1841382017-06-12 01:16:46 -0700356
Niels Möller1e062892018-02-07 10:18:32 +0100357 webrtc::VideoSendStream* stream_ RTC_GUARDED_BY(&thread_checker_);
Christian Fremerey6c025412019-02-13 19:43:28 +0000358 rtc::VideoSinkInterface<webrtc::VideoFrame>* encoder_sink_
359 RTC_GUARDED_BY(&thread_checker_);
eladalonf1841382017-06-12 01:16:46 -0700360 // Contains settings that are the same for all streams in the MediaChannel,
361 // such as codecs, header extensions, and the global bitrate limit for the
362 // entire channel.
Niels Möller1e062892018-02-07 10:18:32 +0100363 VideoSendStreamParameters parameters_ RTC_GUARDED_BY(&thread_checker_);
eladalonf1841382017-06-12 01:16:46 -0700364 // Contains settings that are unique for each stream, such as max_bitrate.
365 // Does *not* contain codecs, however.
366 // TODO(skvlad): Move ssrcs_ and ssrc_groups_ into rtp_parameters_.
367 // TODO(skvlad): Combine parameters_ and rtp_parameters_ once we have only
368 // one stream per MediaChannel.
Niels Möller1e062892018-02-07 10:18:32 +0100369 webrtc::RtpParameters rtp_parameters_ RTC_GUARDED_BY(&thread_checker_);
eladalonf1841382017-06-12 01:16:46 -0700370
Niels Möller1e062892018-02-07 10:18:32 +0100371 bool sending_ RTC_GUARDED_BY(&thread_checker_);
eladalonf1841382017-06-12 01:16:46 -0700372 };
373
374 // Wrapper for the receiver part, contains configs etc. that are needed to
375 // reconstruct the underlying VideoReceiveStream.
376 class WebRtcVideoReceiveStream
377 : public rtc::VideoSinkInterface<webrtc::VideoFrame> {
378 public:
379 WebRtcVideoReceiveStream(
380 webrtc::Call* call,
381 const StreamParams& sp,
382 webrtc::VideoReceiveStream::Config config,
Magnus Jedvert59ab3532018-09-03 18:07:56 +0200383 webrtc::VideoDecoderFactory* decoder_factory,
eladalonf1841382017-06-12 01:16:46 -0700384 bool default_stream,
385 const std::vector<VideoCodecSettings>& recv_codecs,
386 const webrtc::FlexfecReceiveStream::Config& flexfec_config);
387 ~WebRtcVideoReceiveStream();
388
389 const std::vector<uint32_t>& GetSsrcs() const;
Florent Castelliabe301f2018-06-12 18:33:49 +0200390
Jonas Oreland49ac5952018-09-26 16:04:32 +0200391 std::vector<webrtc::RtpSource> GetSources();
392
Florent Castelliabe301f2018-06-12 18:33:49 +0200393 // Does not return codecs, they are filled by the owning WebRtcVideoChannel.
394 webrtc::RtpParameters GetRtpParameters() const;
eladalonf1841382017-06-12 01:16:46 -0700395
396 void SetLocalSsrc(uint32_t local_ssrc);
397 // TODO(deadbeef): Move these feedback parameters into the recv parameters.
398 void SetFeedbackParameters(bool nack_enabled,
399 bool remb_enabled,
400 bool transport_cc_enabled,
401 webrtc::RtcpMode rtcp_mode);
402 void SetRecvParameters(const ChangedRecvParameters& recv_params);
403
404 void OnFrame(const webrtc::VideoFrame& frame) override;
405 bool IsDefaultStream() const;
406
Benjamin Wright192eeec2018-10-17 17:27:25 -0700407 void SetFrameDecryptor(
408 rtc::scoped_refptr<webrtc::FrameDecryptorInterface> frame_decryptor);
409
Ruslan Burakov493a6502019-02-27 15:32:48 +0100410 bool SetBaseMinimumPlayoutDelayMs(int delay_ms);
411
412 int GetBaseMinimumPlayoutDelayMs() const;
413
eladalonf1841382017-06-12 01:16:46 -0700414 void SetSink(rtc::VideoSinkInterface<webrtc::VideoFrame>* sink);
415
416 VideoReceiverInfo GetVideoReceiverInfo(bool log_stats);
417
418 private:
eladalonf1841382017-06-12 01:16:46 -0700419 void RecreateWebRtcVideoStream();
420 void MaybeRecreateWebRtcFlexfecStream();
421
eladalonc0d481a2017-08-02 07:39:07 -0700422 void MaybeAssociateFlexfecWithVideo();
423 void MaybeDissociateFlexfecFromVideo();
424
Niels Möllercbcbc222018-09-28 09:07:24 +0200425 void ConfigureCodecs(const std::vector<VideoCodecSettings>& recv_codecs);
eladalonf1841382017-06-12 01:16:46 -0700426 void ConfigureFlexfecCodec(int flexfec_payload_type);
eladalonf1841382017-06-12 01:16:46 -0700427
428 std::string GetCodecNameFromPayloadType(int payload_type);
429
430 webrtc::Call* const call_;
Niels Möllercbcbc222018-09-28 09:07:24 +0200431 const StreamParams stream_params_;
eladalonf1841382017-06-12 01:16:46 -0700432
433 // Both |stream_| and |flexfec_stream_| are managed by |this|. They are
434 // destroyed by calling call_->DestroyVideoReceiveStream and
435 // call_->DestroyFlexfecReceiveStream, respectively.
436 webrtc::VideoReceiveStream* stream_;
437 const bool default_stream_;
438 webrtc::VideoReceiveStream::Config config_;
439 webrtc::FlexfecReceiveStream::Config flexfec_config_;
440 webrtc::FlexfecReceiveStream* flexfec_stream_;
441
Niels Möllercbcbc222018-09-28 09:07:24 +0200442 webrtc::VideoDecoderFactory* const decoder_factory_;
eladalonf1841382017-06-12 01:16:46 -0700443
444 rtc::CriticalSection sink_lock_;
danilchapa37de392017-09-09 04:17:22 -0700445 rtc::VideoSinkInterface<webrtc::VideoFrame>* sink_
446 RTC_GUARDED_BY(sink_lock_);
eladalonf1841382017-06-12 01:16:46 -0700447 // Expands remote RTP timestamps to int64_t to be able to estimate how long
448 // the stream has been running.
449 rtc::TimestampWrapAroundHandler timestamp_wraparound_handler_
danilchapa37de392017-09-09 04:17:22 -0700450 RTC_GUARDED_BY(sink_lock_);
451 int64_t first_frame_timestamp_ RTC_GUARDED_BY(sink_lock_);
eladalonf1841382017-06-12 01:16:46 -0700452 // Start NTP time is estimated as current remote NTP time (estimated from
453 // RTCP) minus the elapsed time, as soon as remote NTP time is available.
danilchapa37de392017-09-09 04:17:22 -0700454 int64_t estimated_remote_start_ntp_time_ms_ RTC_GUARDED_BY(sink_lock_);
eladalonf1841382017-06-12 01:16:46 -0700455 };
456
457 void Construct(webrtc::Call* call, WebRtcVideoEngine* engine);
458
459 bool SendRtp(const uint8_t* data,
460 size_t len,
461 const webrtc::PacketOptions& options) override;
462 bool SendRtcp(const uint8_t* data, size_t len) override;
463
464 static std::vector<VideoCodecSettings> MapCodecs(
465 const std::vector<VideoCodec>& codecs);
466 // Select what video codec will be used for sending, i.e. what codec is used
467 // for local encoding, based on supported remote codecs. The first remote
468 // codec that is supported locally will be selected.
Danil Chapovalov00c71832018-06-15 15:58:38 +0200469 absl::optional<VideoCodecSettings> SelectSendVideoCodec(
Steve Antonef50b252019-03-01 15:15:38 -0800470 const std::vector<VideoCodecSettings>& remote_mapped_codecs) const
471 RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_);
eladalonf1841382017-06-12 01:16:46 -0700472
473 static bool NonFlexfecReceiveCodecsHaveChanged(
474 std::vector<VideoCodecSettings> before,
475 std::vector<VideoCodecSettings> after);
476
Steve Antonef50b252019-03-01 15:15:38 -0800477 void FillSenderStats(VideoMediaInfo* info, bool log_stats)
478 RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_);
479 void FillReceiverStats(VideoMediaInfo* info, bool log_stats)
480 RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_);
eladalonf1841382017-06-12 01:16:46 -0700481 void FillBandwidthEstimationStats(const webrtc::Call::Stats& stats,
Steve Antonef50b252019-03-01 15:15:38 -0800482 VideoMediaInfo* info)
483 RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_);
484 void FillSendAndReceiveCodecStats(VideoMediaInfo* video_media_info)
485 RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_);
eladalonf1841382017-06-12 01:16:46 -0700486
487 rtc::ThreadChecker thread_checker_;
488
Steve Antonef50b252019-03-01 15:15:38 -0800489 uint32_t rtcp_receiver_report_ssrc_ RTC_GUARDED_BY(thread_checker_);
490 bool sending_ RTC_GUARDED_BY(thread_checker_);
491 webrtc::Call* const call_ RTC_GUARDED_BY(thread_checker_);
eladalonf1841382017-06-12 01:16:46 -0700492
Steve Antonef50b252019-03-01 15:15:38 -0800493 DefaultUnsignalledSsrcHandler default_unsignalled_ssrc_handler_
494 RTC_GUARDED_BY(thread_checker_);
495 UnsignalledSsrcHandler* const unsignalled_ssrc_handler_
496 RTC_GUARDED_BY(thread_checker_);
eladalonf1841382017-06-12 01:16:46 -0700497
Ruslan Burakov493a6502019-02-27 15:32:48 +0100498 // Delay for unsignaled streams, which may be set before the stream exists.
Steve Antonef50b252019-03-01 15:15:38 -0800499 int default_recv_base_minimum_delay_ms_ RTC_GUARDED_BY(thread_checker_) = 0;
Ruslan Burakov493a6502019-02-27 15:32:48 +0100500
Steve Antonef50b252019-03-01 15:15:38 -0800501 const MediaConfig::Video video_config_ RTC_GUARDED_BY(thread_checker_);
eladalonf1841382017-06-12 01:16:46 -0700502
eladalonf1841382017-06-12 01:16:46 -0700503 // Using primary-ssrc (first ssrc) as key.
504 std::map<uint32_t, WebRtcVideoSendStream*> send_streams_
Steve Antonef50b252019-03-01 15:15:38 -0800505 RTC_GUARDED_BY(thread_checker_);
eladalonf1841382017-06-12 01:16:46 -0700506 std::map<uint32_t, WebRtcVideoReceiveStream*> receive_streams_
Steve Antonef50b252019-03-01 15:15:38 -0800507 RTC_GUARDED_BY(thread_checker_);
508 std::set<uint32_t> send_ssrcs_ RTC_GUARDED_BY(thread_checker_);
509 std::set<uint32_t> receive_ssrcs_ RTC_GUARDED_BY(thread_checker_);
eladalonf1841382017-06-12 01:16:46 -0700510
Steve Antonef50b252019-03-01 15:15:38 -0800511 absl::optional<VideoCodecSettings> send_codec_
512 RTC_GUARDED_BY(thread_checker_);
513 absl::optional<std::vector<webrtc::RtpExtension>> send_rtp_extensions_
514 RTC_GUARDED_BY(thread_checker_);
eladalonf1841382017-06-12 01:16:46 -0700515
Steve Antonef50b252019-03-01 15:15:38 -0800516 webrtc::VideoEncoderFactory* const encoder_factory_
517 RTC_GUARDED_BY(thread_checker_);
518 webrtc::VideoDecoderFactory* const decoder_factory_
519 RTC_GUARDED_BY(thread_checker_);
520 webrtc::VideoBitrateAllocatorFactory* const bitrate_allocator_factory_
521 RTC_GUARDED_BY(thread_checker_);
522 std::vector<VideoCodecSettings> recv_codecs_ RTC_GUARDED_BY(thread_checker_);
523 std::vector<webrtc::RtpExtension> recv_rtp_extensions_
524 RTC_GUARDED_BY(thread_checker_);
eladalonf1841382017-06-12 01:16:46 -0700525 // See reason for keeping track of the FlexFEC payload type separately in
526 // comment in WebRtcVideoChannel::ChangedRecvParameters.
Steve Antonef50b252019-03-01 15:15:38 -0800527 int recv_flexfec_payload_type_ RTC_GUARDED_BY(thread_checker_);
528 webrtc::BitrateConstraints bitrate_config_ RTC_GUARDED_BY(thread_checker_);
eladalonf1841382017-06-12 01:16:46 -0700529 // TODO(deadbeef): Don't duplicate information between
530 // send_params/recv_params, rtp_extensions, options, etc.
Steve Antonef50b252019-03-01 15:15:38 -0800531 VideoSendParameters send_params_ RTC_GUARDED_BY(thread_checker_);
532 rtc::DiffServCodePoint preferred_dscp_ RTC_GUARDED_BY(thread_checker_);
533 VideoOptions default_send_options_ RTC_GUARDED_BY(thread_checker_);
534 VideoRecvParameters recv_params_ RTC_GUARDED_BY(thread_checker_);
535 int64_t last_stats_log_ms_ RTC_GUARDED_BY(thread_checker_);
536 const bool discard_unknown_ssrc_packets_ RTC_GUARDED_BY(thread_checker_);
Seth Hampson5897a6e2018-04-03 11:16:33 -0700537 // This is a stream param that comes from the remote description, but wasn't
538 // signaled with any a=ssrc lines. It holds information that was signaled
539 // before the unsignaled receive stream is created when the first packet is
540 // received.
Steve Antonef50b252019-03-01 15:15:38 -0800541 StreamParams unsignaled_stream_params_ RTC_GUARDED_BY(thread_checker_);
Benjamin Wright192eeec2018-10-17 17:27:25 -0700542 // Per peer connection crypto options that last for the lifetime of the peer
543 // connection.
Steve Antonef50b252019-03-01 15:15:38 -0800544 const webrtc::CryptoOptions crypto_options_ RTC_GUARDED_BY(thread_checker_);
eladalonf1841382017-06-12 01:16:46 -0700545};
546
ilnik6b826ef2017-06-16 06:53:48 -0700547class EncoderStreamFactory
548 : public webrtc::VideoEncoderConfig::VideoStreamFactoryInterface {
549 public:
550 EncoderStreamFactory(std::string codec_name,
551 int max_qp,
Seth Hampson1370e302018-02-07 08:50:36 -0800552 bool is_screenshare,
553 bool screenshare_config_explicitly_enabled);
ilnik6b826ef2017-06-16 06:53:48 -0700554
555 private:
556 std::vector<webrtc::VideoStream> CreateEncoderStreams(
557 int width,
558 int height,
559 const webrtc::VideoEncoderConfig& encoder_config) override;
560
561 const std::string codec_name_;
562 const int max_qp_;
Seth Hampson1370e302018-02-07 08:50:36 -0800563 const bool is_screenshare_;
564 // Allows a screenshare specific configuration, which enables temporal
565 // layering and allows simulcast.
566 const bool screenshare_config_explicitly_enabled_;
ilnik6b826ef2017-06-16 06:53:48 -0700567};
568
eladalonf1841382017-06-12 01:16:46 -0700569} // namespace cricket
570
Steve Anton10542f22019-01-11 09:11:00 -0800571#endif // MEDIA_ENGINE_WEBRTC_VIDEO_ENGINE_H_