blob: 0b209873c888fc10d0c5a2a6529d2544163db314 [file] [log] [blame]
eladalonf1841382017-06-12 01:16:46 -07001/*
2 * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Steve Anton10542f22019-01-11 09:11:00 -080011#ifndef MEDIA_ENGINE_WEBRTC_VIDEO_ENGINE_H_
12#define MEDIA_ENGINE_WEBRTC_VIDEO_ENGINE_H_
eladalonf1841382017-06-12 01:16:46 -070013
14#include <map>
15#include <memory>
16#include <set>
17#include <string>
18#include <vector>
19
Danil Chapovalov00c71832018-06-15 15:58:38 +020020#include "absl/types/optional.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020021#include "api/call/transport.h"
Jiawei Ouc2ebe212018-11-08 10:02:56 -080022#include "api/video/video_bitrate_allocator_factory.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020023#include "api/video/video_frame.h"
Niels Möllerc6ce9c52018-05-11 11:15:30 +020024#include "api/video/video_sink_interface.h"
Niels Möller0327c2d2018-05-21 14:09:31 +020025#include "api/video/video_source_interface.h"
26#include "api/video_codecs/sdp_video_format.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020027#include "call/call.h"
28#include "call/flexfec_receive_stream.h"
29#include "call/video_receive_stream.h"
30#include "call/video_send_stream.h"
Steve Anton10542f22019-01-11 09:11:00 -080031#include "media/base/media_engine.h"
Jonas Oreland6d835922019-03-18 10:59:40 +010032#include "media/engine/unhandled_packets_buffer.h"
Steve Anton10542f22019-01-11 09:11:00 -080033#include "rtc_base/async_invoker.h"
34#include "rtc_base/critical_section.h"
35#include "rtc_base/network_route.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020036#include "rtc_base/thread_annotations.h"
37#include "rtc_base/thread_checker.h"
eladalonf1841382017-06-12 01:16:46 -070038
39namespace webrtc {
Magnus Jedvertd4b0c052017-09-14 10:24:54 +020040class VideoDecoderFactory;
Magnus Jedvertd4b0c052017-09-14 10:24:54 +020041class VideoEncoderFactory;
eladalonf1841382017-06-12 01:16:46 -070042struct MediaConfig;
Yves Gerey665174f2018-06-19 15:03:05 +020043} // namespace webrtc
eladalonf1841382017-06-12 01:16:46 -070044
45namespace rtc {
46class Thread;
47} // namespace rtc
48
49namespace cricket {
50
eladalonf1841382017-06-12 01:16:46 -070051class WebRtcVideoChannel;
eladalonf1841382017-06-12 01:16:46 -070052
eladalonf1841382017-06-12 01:16:46 -070053class UnsignalledSsrcHandler {
54 public:
55 enum Action {
56 kDropPacket,
57 kDeliverPacket,
58 };
59 virtual Action OnUnsignalledSsrc(WebRtcVideoChannel* channel,
60 uint32_t ssrc) = 0;
61 virtual ~UnsignalledSsrcHandler() = default;
62};
63
64// TODO(pbos): Remove, use external handlers only.
65class DefaultUnsignalledSsrcHandler : public UnsignalledSsrcHandler {
66 public:
67 DefaultUnsignalledSsrcHandler();
Yves Gerey665174f2018-06-19 15:03:05 +020068 Action OnUnsignalledSsrc(WebRtcVideoChannel* channel, uint32_t ssrc) override;
eladalonf1841382017-06-12 01:16:46 -070069
70 rtc::VideoSinkInterface<webrtc::VideoFrame>* GetDefaultSink() const;
71 void SetDefaultSink(WebRtcVideoChannel* channel,
72 rtc::VideoSinkInterface<webrtc::VideoFrame>* sink);
73
74 virtual ~DefaultUnsignalledSsrcHandler() = default;
75
76 private:
77 rtc::VideoSinkInterface<webrtc::VideoFrame>* default_sink_;
78};
79
80// WebRtcVideoEngine is used for the new native WebRTC Video API (webrtc:1667).
Sebastian Jansson84848f22018-11-16 10:40:36 +010081class WebRtcVideoEngine : public VideoEngineInterface {
eladalonf1841382017-06-12 01:16:46 -070082 public:
Magnus Jedvertd4b0c052017-09-14 10:24:54 +020083 // These video codec factories represents all video codecs, i.e. both software
84 // and external hardware codecs.
85 WebRtcVideoEngine(
86 std::unique_ptr<webrtc::VideoEncoderFactory> video_encoder_factory,
Jiawei Ouc2ebe212018-11-08 10:02:56 -080087 std::unique_ptr<webrtc::VideoDecoderFactory> video_decoder_factory,
88 std::unique_ptr<webrtc::VideoBitrateAllocatorFactory>
89 video_bitrate_allocator_factory);
Magnus Jedvertd4b0c052017-09-14 10:24:54 +020090
Sebastian Jansson84848f22018-11-16 10:40:36 +010091 ~WebRtcVideoEngine() override;
eladalonf1841382017-06-12 01:16:46 -070092
Sebastian Jansson84848f22018-11-16 10:40:36 +010093 VideoMediaChannel* CreateMediaChannel(
Benjamin Wrightbfb444c2018-10-15 10:20:24 -070094 webrtc::Call* call,
95 const MediaConfig& config,
96 const VideoOptions& options,
Sebastian Jansson84848f22018-11-16 10:40:36 +010097 const webrtc::CryptoOptions& crypto_options) override;
eladalonf1841382017-06-12 01:16:46 -070098
Sebastian Jansson84848f22018-11-16 10:40:36 +010099 std::vector<VideoCodec> codecs() const override;
100 RtpCapabilities GetCapabilities() const override;
eladalonf1841382017-06-12 01:16:46 -0700101
eladalonf1841382017-06-12 01:16:46 -0700102 private:
Magnus Jedvert59ab3532018-09-03 18:07:56 +0200103 const std::unique_ptr<webrtc::VideoDecoderFactory> decoder_factory_;
Magnus Jedvert07e0d012017-10-31 11:24:54 +0100104 const std::unique_ptr<webrtc::VideoEncoderFactory> encoder_factory_;
Jiawei Ouc2ebe212018-11-08 10:02:56 -0800105 const std::unique_ptr<webrtc::VideoBitrateAllocatorFactory>
106 bitrate_allocator_factory_;
eladalonf1841382017-06-12 01:16:46 -0700107};
108
109class WebRtcVideoChannel : public VideoMediaChannel, public webrtc::Transport {
110 public:
Jiawei Ouc2ebe212018-11-08 10:02:56 -0800111 WebRtcVideoChannel(
112 webrtc::Call* call,
113 const MediaConfig& config,
114 const VideoOptions& options,
115 const webrtc::CryptoOptions& crypto_options,
116 webrtc::VideoEncoderFactory* encoder_factory,
117 webrtc::VideoDecoderFactory* decoder_factory,
118 webrtc::VideoBitrateAllocatorFactory* bitrate_allocator_factory);
eladalonf1841382017-06-12 01:16:46 -0700119 ~WebRtcVideoChannel() override;
120
121 // VideoMediaChannel implementation
eladalonf1841382017-06-12 01:16:46 -0700122 bool SetSendParameters(const VideoSendParameters& params) override;
123 bool SetRecvParameters(const VideoRecvParameters& params) override;
124 webrtc::RtpParameters GetRtpSendParameters(uint32_t ssrc) const override;
Zach Steinba37b4b2018-01-23 15:02:36 -0800125 webrtc::RTCError SetRtpSendParameters(
126 uint32_t ssrc,
127 const webrtc::RtpParameters& parameters) override;
eladalonf1841382017-06-12 01:16:46 -0700128 webrtc::RtpParameters GetRtpReceiveParameters(uint32_t ssrc) const override;
129 bool SetRtpReceiveParameters(
130 uint32_t ssrc,
131 const webrtc::RtpParameters& parameters) override;
132 bool GetSendCodec(VideoCodec* send_codec) override;
133 bool SetSend(bool send) override;
134 bool SetVideoSend(
135 uint32_t ssrc,
eladalonf1841382017-06-12 01:16:46 -0700136 const VideoOptions* options,
137 rtc::VideoSourceInterface<webrtc::VideoFrame>* source) override;
138 bool AddSendStream(const StreamParams& sp) override;
139 bool RemoveSendStream(uint32_t ssrc) override;
140 bool AddRecvStream(const StreamParams& sp) override;
141 bool AddRecvStream(const StreamParams& sp, bool default_stream);
142 bool RemoveRecvStream(uint32_t ssrc) override;
143 bool SetSink(uint32_t ssrc,
144 rtc::VideoSinkInterface<webrtc::VideoFrame>* sink) override;
145 void FillBitrateInfo(BandwidthEstimationInfo* bwe_info) override;
146 bool GetStats(VideoMediaInfo* info) override;
147
Amit Hilbuche7a5f7b2019-03-12 11:10:27 -0700148 void OnPacketReceived(rtc::CopyOnWriteBuffer packet,
Niels Möllere6933812018-11-05 13:01:41 +0100149 int64_t packet_time_us) override;
Amit Hilbuche7a5f7b2019-03-12 11:10:27 -0700150 void OnRtcpReceived(rtc::CopyOnWriteBuffer packet,
Niels Möllere6933812018-11-05 13:01:41 +0100151 int64_t packet_time_us) override;
eladalonf1841382017-06-12 01:16:46 -0700152 void OnReadyToSend(bool ready) override;
153 void OnNetworkRouteChanged(const std::string& transport_name,
154 const rtc::NetworkRoute& network_route) override;
Anton Sukhanov98a462c2018-10-17 13:15:42 -0700155 void SetInterface(NetworkInterface* iface,
156 webrtc::MediaTransportInterface* media_transport) override;
eladalonf1841382017-06-12 01:16:46 -0700157
Benjamin Wright192eeec2018-10-17 17:27:25 -0700158 // E2E Encrypted Video Frame API
159 // Set a frame decryptor to a particular ssrc that will intercept all
160 // incoming video frames and attempt to decrypt them before forwarding the
161 // result.
162 void SetFrameDecryptor(uint32_t ssrc,
163 rtc::scoped_refptr<webrtc::FrameDecryptorInterface>
164 frame_decryptor) override;
165 // Set a frame encryptor to a particular ssrc that will intercept all
166 // outgoing video frames and attempt to encrypt them and forward the result
167 // to the packetizer.
168 void SetFrameEncryptor(uint32_t ssrc,
169 rtc::scoped_refptr<webrtc::FrameEncryptorInterface>
170 frame_encryptor) override;
171
Ruslan Burakov493a6502019-02-27 15:32:48 +0100172 bool SetBaseMinimumPlayoutDelayMs(uint32_t ssrc, int delay_ms) override;
173
174 absl::optional<int> GetBaseMinimumPlayoutDelayMs(
175 uint32_t ssrc) const override;
176
eladalonf1841382017-06-12 01:16:46 -0700177 // Implemented for VideoMediaChannelTest.
Steve Antonef50b252019-03-01 15:15:38 -0800178 bool sending() const {
179 RTC_DCHECK_RUN_ON(&thread_checker_);
180 return sending_;
181 }
eladalonf1841382017-06-12 01:16:46 -0700182
Danil Chapovalov00c71832018-06-15 15:58:38 +0200183 absl::optional<uint32_t> GetDefaultReceiveStreamSsrc();
eladalonf1841382017-06-12 01:16:46 -0700184
Steve Antonef50b252019-03-01 15:15:38 -0800185 StreamParams unsignaled_stream_params() {
186 RTC_DCHECK_RUN_ON(&thread_checker_);
187 return unsignaled_stream_params_;
188 }
Seth Hampson5897a6e2018-04-03 11:16:33 -0700189
eladalonf1841382017-06-12 01:16:46 -0700190 // AdaptReason is used for expressing why a WebRtcVideoSendStream request
191 // a lower input frame size than the currently configured camera input frame
192 // size. There can be more than one reason OR:ed together.
193 enum AdaptReason {
194 ADAPTREASON_NONE = 0,
195 ADAPTREASON_CPU = 1,
196 ADAPTREASON_BANDWIDTH = 2,
197 };
198
sprang67561a62017-06-15 06:34:42 -0700199 static constexpr int kDefaultQpMax = 56;
200
Jonas Oreland49ac5952018-09-26 16:04:32 +0200201 std::vector<webrtc::RtpSource> GetSources(uint32_t ssrc) const override;
202
Jonas Oreland6d835922019-03-18 10:59:40 +0100203 // Take the buffered packets for |ssrcs| and feed them into DeliverPacket.
204 // This method does nothing unless unknown_ssrc_packet_buffer_ is configured.
205 void BackfillBufferedPackets(rtc::ArrayView<const uint32_t> ssrcs);
206
eladalonf1841382017-06-12 01:16:46 -0700207 private:
208 class WebRtcVideoReceiveStream;
209 struct VideoCodecSettings {
210 VideoCodecSettings();
211
212 // Checks if all members of |*this| are equal to the corresponding members
213 // of |other|.
214 bool operator==(const VideoCodecSettings& other) const;
215 bool operator!=(const VideoCodecSettings& other) const;
216
217 // Checks if all members of |a|, except |flexfec_payload_type|, are equal
218 // to the corresponding members of |b|.
219 static bool EqualsDisregardingFlexfec(const VideoCodecSettings& a,
220 const VideoCodecSettings& b);
221
222 VideoCodec codec;
223 webrtc::UlpfecConfig ulpfec;
224 int flexfec_payload_type;
225 int rtx_payload_type;
226 };
227
228 struct ChangedSendParameters {
229 // These optionals are unset if not changed.
Danil Chapovalov00c71832018-06-15 15:58:38 +0200230 absl::optional<VideoCodecSettings> codec;
231 absl::optional<std::vector<webrtc::RtpExtension>> rtp_header_extensions;
232 absl::optional<std::string> mid;
Johannes Kron9190b822018-10-29 11:22:05 +0100233 absl::optional<bool> extmap_allow_mixed;
Danil Chapovalov00c71832018-06-15 15:58:38 +0200234 absl::optional<int> max_bandwidth_bps;
235 absl::optional<bool> conference_mode;
236 absl::optional<webrtc::RtcpMode> rtcp_mode;
eladalonf1841382017-06-12 01:16:46 -0700237 };
238
239 struct ChangedRecvParameters {
240 // These optionals are unset if not changed.
Danil Chapovalov00c71832018-06-15 15:58:38 +0200241 absl::optional<std::vector<VideoCodecSettings>> codec_settings;
242 absl::optional<std::vector<webrtc::RtpExtension>> rtp_header_extensions;
eladalonf1841382017-06-12 01:16:46 -0700243 // Keep track of the FlexFEC payload type separately from |codec_settings|.
244 // This allows us to recreate the FlexfecReceiveStream separately from the
245 // VideoReceiveStream when the FlexFEC payload type is changed.
Danil Chapovalov00c71832018-06-15 15:58:38 +0200246 absl::optional<int> flexfec_payload_type;
eladalonf1841382017-06-12 01:16:46 -0700247 };
248
249 bool GetChangedSendParameters(const VideoSendParameters& params,
Steve Antonef50b252019-03-01 15:15:38 -0800250 ChangedSendParameters* changed_params) const
251 RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_);
eladalonf1841382017-06-12 01:16:46 -0700252 bool GetChangedRecvParameters(const VideoRecvParameters& params,
Steve Antonef50b252019-03-01 15:15:38 -0800253 ChangedRecvParameters* changed_params) const
254 RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_);
eladalonf1841382017-06-12 01:16:46 -0700255
256 void SetMaxSendBandwidth(int bps);
257
258 void ConfigureReceiverRtp(
259 webrtc::VideoReceiveStream::Config* config,
260 webrtc::FlexfecReceiveStream::Config* flexfec_config,
Steve Antonef50b252019-03-01 15:15:38 -0800261 const StreamParams& sp) const
262 RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_);
eladalonf1841382017-06-12 01:16:46 -0700263 bool ValidateSendSsrcAvailability(const StreamParams& sp) const
Steve Antonef50b252019-03-01 15:15:38 -0800264 RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_);
eladalonf1841382017-06-12 01:16:46 -0700265 bool ValidateReceiveSsrcAvailability(const StreamParams& sp) const
Steve Antonef50b252019-03-01 15:15:38 -0800266 RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_);
eladalonf1841382017-06-12 01:16:46 -0700267 void DeleteReceiveStream(WebRtcVideoReceiveStream* stream)
Steve Antonef50b252019-03-01 15:15:38 -0800268 RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_);
eladalonf1841382017-06-12 01:16:46 -0700269
270 static std::string CodecSettingsVectorToString(
271 const std::vector<VideoCodecSettings>& codecs);
272
273 // Wrapper for the sender part.
Christian Fremerey6c025412019-02-13 19:43:28 +0000274 class WebRtcVideoSendStream
275 : public rtc::VideoSourceInterface<webrtc::VideoFrame> {
eladalonf1841382017-06-12 01:16:46 -0700276 public:
277 WebRtcVideoSendStream(
278 webrtc::Call* call,
279 const StreamParams& sp,
280 webrtc::VideoSendStream::Config config,
281 const VideoOptions& options,
eladalonf1841382017-06-12 01:16:46 -0700282 bool enable_cpu_overuse_detection,
283 int max_bitrate_bps,
Danil Chapovalov00c71832018-06-15 15:58:38 +0200284 const absl::optional<VideoCodecSettings>& codec_settings,
285 const absl::optional<std::vector<webrtc::RtpExtension>>& rtp_extensions,
eladalonf1841382017-06-12 01:16:46 -0700286 const VideoSendParameters& send_params);
287 virtual ~WebRtcVideoSendStream();
288
289 void SetSendParameters(const ChangedSendParameters& send_params);
Zach Steinba37b4b2018-01-23 15:02:36 -0800290 webrtc::RTCError SetRtpParameters(const webrtc::RtpParameters& parameters);
eladalonf1841382017-06-12 01:16:46 -0700291 webrtc::RtpParameters GetRtpParameters() const;
292
Benjamin Wright192eeec2018-10-17 17:27:25 -0700293 void SetFrameEncryptor(
294 rtc::scoped_refptr<webrtc::FrameEncryptorInterface> frame_encryptor);
295
Christian Fremerey6c025412019-02-13 19:43:28 +0000296 // Implements rtc::VideoSourceInterface<webrtc::VideoFrame>.
297 // WebRtcVideoSendStream acts as a source to the webrtc::VideoSendStream
298 // in |stream_|. This is done to proxy VideoSinkWants from the encoder to
299 // the worker thread.
300 void AddOrUpdateSink(rtc::VideoSinkInterface<webrtc::VideoFrame>* sink,
301 const rtc::VideoSinkWants& wants) override;
302 void RemoveSink(rtc::VideoSinkInterface<webrtc::VideoFrame>* sink) override;
303
Niels Möllerff40b142018-04-09 08:49:14 +0200304 bool SetVideoSend(const VideoOptions* options,
eladalonf1841382017-06-12 01:16:46 -0700305 rtc::VideoSourceInterface<webrtc::VideoFrame>* source);
306
307 void SetSend(bool send);
308
309 const std::vector<uint32_t>& GetSsrcs() const;
310 VideoSenderInfo GetVideoSenderInfo(bool log_stats);
311 void FillBitrateInfo(BandwidthEstimationInfo* bwe_info);
312
313 private:
314 // Parameters needed to reconstruct the underlying stream.
315 // webrtc::VideoSendStream doesn't support setting a lot of options on the
316 // fly, so when those need to be changed we tear down and reconstruct with
317 // similar parameters depending on which options changed etc.
318 struct VideoSendStreamParameters {
319 VideoSendStreamParameters(
320 webrtc::VideoSendStream::Config config,
321 const VideoOptions& options,
322 int max_bitrate_bps,
Danil Chapovalov00c71832018-06-15 15:58:38 +0200323 const absl::optional<VideoCodecSettings>& codec_settings);
eladalonf1841382017-06-12 01:16:46 -0700324 webrtc::VideoSendStream::Config config;
325 VideoOptions options;
326 int max_bitrate_bps;
327 bool conference_mode;
Danil Chapovalov00c71832018-06-15 15:58:38 +0200328 absl::optional<VideoCodecSettings> codec_settings;
eladalonf1841382017-06-12 01:16:46 -0700329 // Sent resolutions + bitrates etc. by the underlying VideoSendStream,
330 // typically changes when setting a new resolution or reconfiguring
331 // bitrates.
332 webrtc::VideoEncoderConfig encoder_config;
333 };
334
eladalonf1841382017-06-12 01:16:46 -0700335 rtc::scoped_refptr<webrtc::VideoEncoderConfig::EncoderSpecificSettings>
336 ConfigureVideoEncoderSettings(const VideoCodec& codec);
Niels Möller5bf8ccd2018-03-15 14:16:11 +0100337 void SetCodec(const VideoCodecSettings& codec);
eladalonf1841382017-06-12 01:16:46 -0700338 void RecreateWebRtcStream();
339 webrtc::VideoEncoderConfig CreateVideoEncoderConfig(
340 const VideoCodec& codec) const;
341 void ReconfigureEncoder();
eladalonf1841382017-06-12 01:16:46 -0700342
343 // Calls Start or Stop according to whether or not |sending_| is true,
344 // and whether or not the encoding in |rtp_parameters_| is active.
345 void UpdateSendState();
346
Taylor Brandstetter49fcc102018-05-16 14:20:41 -0700347 webrtc::DegradationPreference GetDegradationPreference() const
348 RTC_EXCLUSIVE_LOCKS_REQUIRED(&thread_checker_);
eladalonf1841382017-06-12 01:16:46 -0700349
350 rtc::ThreadChecker thread_checker_;
351 rtc::AsyncInvoker invoker_;
352 rtc::Thread* worker_thread_;
Niels Möller1e062892018-02-07 10:18:32 +0100353 const std::vector<uint32_t> ssrcs_ RTC_GUARDED_BY(&thread_checker_);
354 const std::vector<SsrcGroup> ssrc_groups_ RTC_GUARDED_BY(&thread_checker_);
eladalonf1841382017-06-12 01:16:46 -0700355 webrtc::Call* const call_;
356 const bool enable_cpu_overuse_detection_;
357 rtc::VideoSourceInterface<webrtc::VideoFrame>* source_
Niels Möller1e062892018-02-07 10:18:32 +0100358 RTC_GUARDED_BY(&thread_checker_);
eladalonf1841382017-06-12 01:16:46 -0700359
Niels Möller1e062892018-02-07 10:18:32 +0100360 webrtc::VideoSendStream* stream_ RTC_GUARDED_BY(&thread_checker_);
Christian Fremerey6c025412019-02-13 19:43:28 +0000361 rtc::VideoSinkInterface<webrtc::VideoFrame>* encoder_sink_
362 RTC_GUARDED_BY(&thread_checker_);
eladalonf1841382017-06-12 01:16:46 -0700363 // Contains settings that are the same for all streams in the MediaChannel,
364 // such as codecs, header extensions, and the global bitrate limit for the
365 // entire channel.
Niels Möller1e062892018-02-07 10:18:32 +0100366 VideoSendStreamParameters parameters_ RTC_GUARDED_BY(&thread_checker_);
eladalonf1841382017-06-12 01:16:46 -0700367 // Contains settings that are unique for each stream, such as max_bitrate.
368 // Does *not* contain codecs, however.
369 // TODO(skvlad): Move ssrcs_ and ssrc_groups_ into rtp_parameters_.
370 // TODO(skvlad): Combine parameters_ and rtp_parameters_ once we have only
371 // one stream per MediaChannel.
Niels Möller1e062892018-02-07 10:18:32 +0100372 webrtc::RtpParameters rtp_parameters_ RTC_GUARDED_BY(&thread_checker_);
eladalonf1841382017-06-12 01:16:46 -0700373
Niels Möller1e062892018-02-07 10:18:32 +0100374 bool sending_ RTC_GUARDED_BY(&thread_checker_);
eladalonf1841382017-06-12 01:16:46 -0700375 };
376
377 // Wrapper for the receiver part, contains configs etc. that are needed to
378 // reconstruct the underlying VideoReceiveStream.
379 class WebRtcVideoReceiveStream
380 : public rtc::VideoSinkInterface<webrtc::VideoFrame> {
381 public:
382 WebRtcVideoReceiveStream(
Jonas Oreland6d835922019-03-18 10:59:40 +0100383 WebRtcVideoChannel* channel,
eladalonf1841382017-06-12 01:16:46 -0700384 webrtc::Call* call,
385 const StreamParams& sp,
386 webrtc::VideoReceiveStream::Config config,
Magnus Jedvert59ab3532018-09-03 18:07:56 +0200387 webrtc::VideoDecoderFactory* decoder_factory,
eladalonf1841382017-06-12 01:16:46 -0700388 bool default_stream,
389 const std::vector<VideoCodecSettings>& recv_codecs,
390 const webrtc::FlexfecReceiveStream::Config& flexfec_config);
391 ~WebRtcVideoReceiveStream();
392
393 const std::vector<uint32_t>& GetSsrcs() const;
Florent Castelliabe301f2018-06-12 18:33:49 +0200394
Jonas Oreland49ac5952018-09-26 16:04:32 +0200395 std::vector<webrtc::RtpSource> GetSources();
396
Florent Castelliabe301f2018-06-12 18:33:49 +0200397 // Does not return codecs, they are filled by the owning WebRtcVideoChannel.
398 webrtc::RtpParameters GetRtpParameters() const;
eladalonf1841382017-06-12 01:16:46 -0700399
400 void SetLocalSsrc(uint32_t local_ssrc);
401 // TODO(deadbeef): Move these feedback parameters into the recv parameters.
402 void SetFeedbackParameters(bool nack_enabled,
403 bool remb_enabled,
404 bool transport_cc_enabled,
405 webrtc::RtcpMode rtcp_mode);
406 void SetRecvParameters(const ChangedRecvParameters& recv_params);
407
408 void OnFrame(const webrtc::VideoFrame& frame) override;
409 bool IsDefaultStream() const;
410
Benjamin Wright192eeec2018-10-17 17:27:25 -0700411 void SetFrameDecryptor(
412 rtc::scoped_refptr<webrtc::FrameDecryptorInterface> frame_decryptor);
413
Ruslan Burakov493a6502019-02-27 15:32:48 +0100414 bool SetBaseMinimumPlayoutDelayMs(int delay_ms);
415
416 int GetBaseMinimumPlayoutDelayMs() const;
417
eladalonf1841382017-06-12 01:16:46 -0700418 void SetSink(rtc::VideoSinkInterface<webrtc::VideoFrame>* sink);
419
420 VideoReceiverInfo GetVideoReceiverInfo(bool log_stats);
421
422 private:
eladalonf1841382017-06-12 01:16:46 -0700423 void RecreateWebRtcVideoStream();
424 void MaybeRecreateWebRtcFlexfecStream();
425
eladalonc0d481a2017-08-02 07:39:07 -0700426 void MaybeAssociateFlexfecWithVideo();
427 void MaybeDissociateFlexfecFromVideo();
428
Niels Möllercbcbc222018-09-28 09:07:24 +0200429 void ConfigureCodecs(const std::vector<VideoCodecSettings>& recv_codecs);
eladalonf1841382017-06-12 01:16:46 -0700430 void ConfigureFlexfecCodec(int flexfec_payload_type);
eladalonf1841382017-06-12 01:16:46 -0700431
432 std::string GetCodecNameFromPayloadType(int payload_type);
433
Jonas Oreland6d835922019-03-18 10:59:40 +0100434 WebRtcVideoChannel* const channel_;
eladalonf1841382017-06-12 01:16:46 -0700435 webrtc::Call* const call_;
Niels Möllercbcbc222018-09-28 09:07:24 +0200436 const StreamParams stream_params_;
eladalonf1841382017-06-12 01:16:46 -0700437
438 // Both |stream_| and |flexfec_stream_| are managed by |this|. They are
439 // destroyed by calling call_->DestroyVideoReceiveStream and
440 // call_->DestroyFlexfecReceiveStream, respectively.
441 webrtc::VideoReceiveStream* stream_;
442 const bool default_stream_;
443 webrtc::VideoReceiveStream::Config config_;
444 webrtc::FlexfecReceiveStream::Config flexfec_config_;
445 webrtc::FlexfecReceiveStream* flexfec_stream_;
446
Niels Möllercbcbc222018-09-28 09:07:24 +0200447 webrtc::VideoDecoderFactory* const decoder_factory_;
eladalonf1841382017-06-12 01:16:46 -0700448
449 rtc::CriticalSection sink_lock_;
danilchapa37de392017-09-09 04:17:22 -0700450 rtc::VideoSinkInterface<webrtc::VideoFrame>* sink_
451 RTC_GUARDED_BY(sink_lock_);
eladalonf1841382017-06-12 01:16:46 -0700452 // Expands remote RTP timestamps to int64_t to be able to estimate how long
453 // the stream has been running.
454 rtc::TimestampWrapAroundHandler timestamp_wraparound_handler_
danilchapa37de392017-09-09 04:17:22 -0700455 RTC_GUARDED_BY(sink_lock_);
456 int64_t first_frame_timestamp_ RTC_GUARDED_BY(sink_lock_);
eladalonf1841382017-06-12 01:16:46 -0700457 // Start NTP time is estimated as current remote NTP time (estimated from
458 // RTCP) minus the elapsed time, as soon as remote NTP time is available.
danilchapa37de392017-09-09 04:17:22 -0700459 int64_t estimated_remote_start_ntp_time_ms_ RTC_GUARDED_BY(sink_lock_);
eladalonf1841382017-06-12 01:16:46 -0700460 };
461
462 void Construct(webrtc::Call* call, WebRtcVideoEngine* engine);
463
464 bool SendRtp(const uint8_t* data,
465 size_t len,
466 const webrtc::PacketOptions& options) override;
467 bool SendRtcp(const uint8_t* data, size_t len) override;
468
469 static std::vector<VideoCodecSettings> MapCodecs(
470 const std::vector<VideoCodec>& codecs);
471 // Select what video codec will be used for sending, i.e. what codec is used
472 // for local encoding, based on supported remote codecs. The first remote
473 // codec that is supported locally will be selected.
Danil Chapovalov00c71832018-06-15 15:58:38 +0200474 absl::optional<VideoCodecSettings> SelectSendVideoCodec(
Steve Antonef50b252019-03-01 15:15:38 -0800475 const std::vector<VideoCodecSettings>& remote_mapped_codecs) const
476 RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_);
eladalonf1841382017-06-12 01:16:46 -0700477
478 static bool NonFlexfecReceiveCodecsHaveChanged(
479 std::vector<VideoCodecSettings> before,
480 std::vector<VideoCodecSettings> after);
481
Steve Antonef50b252019-03-01 15:15:38 -0800482 void FillSenderStats(VideoMediaInfo* info, bool log_stats)
483 RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_);
484 void FillReceiverStats(VideoMediaInfo* info, bool log_stats)
485 RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_);
eladalonf1841382017-06-12 01:16:46 -0700486 void FillBandwidthEstimationStats(const webrtc::Call::Stats& stats,
Steve Antonef50b252019-03-01 15:15:38 -0800487 VideoMediaInfo* info)
488 RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_);
489 void FillSendAndReceiveCodecStats(VideoMediaInfo* video_media_info)
490 RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_);
eladalonf1841382017-06-12 01:16:46 -0700491
492 rtc::ThreadChecker thread_checker_;
493
Steve Antonef50b252019-03-01 15:15:38 -0800494 uint32_t rtcp_receiver_report_ssrc_ RTC_GUARDED_BY(thread_checker_);
495 bool sending_ RTC_GUARDED_BY(thread_checker_);
496 webrtc::Call* const call_ RTC_GUARDED_BY(thread_checker_);
eladalonf1841382017-06-12 01:16:46 -0700497
Steve Antonef50b252019-03-01 15:15:38 -0800498 DefaultUnsignalledSsrcHandler default_unsignalled_ssrc_handler_
499 RTC_GUARDED_BY(thread_checker_);
500 UnsignalledSsrcHandler* const unsignalled_ssrc_handler_
501 RTC_GUARDED_BY(thread_checker_);
eladalonf1841382017-06-12 01:16:46 -0700502
Ruslan Burakov493a6502019-02-27 15:32:48 +0100503 // Delay for unsignaled streams, which may be set before the stream exists.
Steve Antonef50b252019-03-01 15:15:38 -0800504 int default_recv_base_minimum_delay_ms_ RTC_GUARDED_BY(thread_checker_) = 0;
Ruslan Burakov493a6502019-02-27 15:32:48 +0100505
Steve Antonef50b252019-03-01 15:15:38 -0800506 const MediaConfig::Video video_config_ RTC_GUARDED_BY(thread_checker_);
eladalonf1841382017-06-12 01:16:46 -0700507
eladalonf1841382017-06-12 01:16:46 -0700508 // Using primary-ssrc (first ssrc) as key.
509 std::map<uint32_t, WebRtcVideoSendStream*> send_streams_
Steve Antonef50b252019-03-01 15:15:38 -0800510 RTC_GUARDED_BY(thread_checker_);
eladalonf1841382017-06-12 01:16:46 -0700511 std::map<uint32_t, WebRtcVideoReceiveStream*> receive_streams_
Steve Antonef50b252019-03-01 15:15:38 -0800512 RTC_GUARDED_BY(thread_checker_);
513 std::set<uint32_t> send_ssrcs_ RTC_GUARDED_BY(thread_checker_);
514 std::set<uint32_t> receive_ssrcs_ RTC_GUARDED_BY(thread_checker_);
eladalonf1841382017-06-12 01:16:46 -0700515
Steve Antonef50b252019-03-01 15:15:38 -0800516 absl::optional<VideoCodecSettings> send_codec_
517 RTC_GUARDED_BY(thread_checker_);
518 absl::optional<std::vector<webrtc::RtpExtension>> send_rtp_extensions_
519 RTC_GUARDED_BY(thread_checker_);
eladalonf1841382017-06-12 01:16:46 -0700520
Steve Antonef50b252019-03-01 15:15:38 -0800521 webrtc::VideoEncoderFactory* const encoder_factory_
522 RTC_GUARDED_BY(thread_checker_);
523 webrtc::VideoDecoderFactory* const decoder_factory_
524 RTC_GUARDED_BY(thread_checker_);
525 webrtc::VideoBitrateAllocatorFactory* const bitrate_allocator_factory_
526 RTC_GUARDED_BY(thread_checker_);
527 std::vector<VideoCodecSettings> recv_codecs_ RTC_GUARDED_BY(thread_checker_);
528 std::vector<webrtc::RtpExtension> recv_rtp_extensions_
529 RTC_GUARDED_BY(thread_checker_);
eladalonf1841382017-06-12 01:16:46 -0700530 // See reason for keeping track of the FlexFEC payload type separately in
531 // comment in WebRtcVideoChannel::ChangedRecvParameters.
Steve Antonef50b252019-03-01 15:15:38 -0800532 int recv_flexfec_payload_type_ RTC_GUARDED_BY(thread_checker_);
533 webrtc::BitrateConstraints bitrate_config_ RTC_GUARDED_BY(thread_checker_);
eladalonf1841382017-06-12 01:16:46 -0700534 // TODO(deadbeef): Don't duplicate information between
535 // send_params/recv_params, rtp_extensions, options, etc.
Steve Antonef50b252019-03-01 15:15:38 -0800536 VideoSendParameters send_params_ RTC_GUARDED_BY(thread_checker_);
Steve Antonef50b252019-03-01 15:15:38 -0800537 VideoOptions default_send_options_ RTC_GUARDED_BY(thread_checker_);
538 VideoRecvParameters recv_params_ RTC_GUARDED_BY(thread_checker_);
539 int64_t last_stats_log_ms_ RTC_GUARDED_BY(thread_checker_);
540 const bool discard_unknown_ssrc_packets_ RTC_GUARDED_BY(thread_checker_);
Seth Hampson5897a6e2018-04-03 11:16:33 -0700541 // This is a stream param that comes from the remote description, but wasn't
542 // signaled with any a=ssrc lines. It holds information that was signaled
543 // before the unsignaled receive stream is created when the first packet is
544 // received.
Steve Antonef50b252019-03-01 15:15:38 -0800545 StreamParams unsignaled_stream_params_ RTC_GUARDED_BY(thread_checker_);
Benjamin Wright192eeec2018-10-17 17:27:25 -0700546 // Per peer connection crypto options that last for the lifetime of the peer
547 // connection.
Steve Antonef50b252019-03-01 15:15:38 -0800548 const webrtc::CryptoOptions crypto_options_ RTC_GUARDED_BY(thread_checker_);
Jonas Oreland6d835922019-03-18 10:59:40 +0100549
550 // Buffer for unhandled packets.
551 std::unique_ptr<UnhandledPacketsBuffer> unknown_ssrc_packet_buffer_
552 RTC_GUARDED_BY(thread_checker_);
eladalonf1841382017-06-12 01:16:46 -0700553};
554
ilnik6b826ef2017-06-16 06:53:48 -0700555class EncoderStreamFactory
556 : public webrtc::VideoEncoderConfig::VideoStreamFactoryInterface {
557 public:
558 EncoderStreamFactory(std::string codec_name,
559 int max_qp,
Seth Hampson1370e302018-02-07 08:50:36 -0800560 bool is_screenshare,
561 bool screenshare_config_explicitly_enabled);
ilnik6b826ef2017-06-16 06:53:48 -0700562
563 private:
564 std::vector<webrtc::VideoStream> CreateEncoderStreams(
565 int width,
566 int height,
567 const webrtc::VideoEncoderConfig& encoder_config) override;
568
569 const std::string codec_name_;
570 const int max_qp_;
Seth Hampson1370e302018-02-07 08:50:36 -0800571 const bool is_screenshare_;
572 // Allows a screenshare specific configuration, which enables temporal
573 // layering and allows simulcast.
574 const bool screenshare_config_explicitly_enabled_;
ilnik6b826ef2017-06-16 06:53:48 -0700575};
576
eladalonf1841382017-06-12 01:16:46 -0700577} // namespace cricket
578
Steve Anton10542f22019-01-11 09:11:00 -0800579#endif // MEDIA_ENGINE_WEBRTC_VIDEO_ENGINE_H_