blob: ccec8cc1d4921277965a130f537beab1e9c98f6e [file] [log] [blame]
eladalonf1841382017-06-12 01:16:46 -07001/*
2 * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Steve Anton10542f22019-01-11 09:11:00 -080011#ifndef MEDIA_ENGINE_WEBRTC_VIDEO_ENGINE_H_
12#define MEDIA_ENGINE_WEBRTC_VIDEO_ENGINE_H_
eladalonf1841382017-06-12 01:16:46 -070013
14#include <map>
15#include <memory>
16#include <set>
17#include <string>
18#include <vector>
19
Danil Chapovalov00c71832018-06-15 15:58:38 +020020#include "absl/types/optional.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020021#include "api/call/transport.h"
Jiawei Ouc2ebe212018-11-08 10:02:56 -080022#include "api/video/video_bitrate_allocator_factory.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020023#include "api/video/video_frame.h"
Niels Möllerc6ce9c52018-05-11 11:15:30 +020024#include "api/video/video_sink_interface.h"
Niels Möller0327c2d2018-05-21 14:09:31 +020025#include "api/video/video_source_interface.h"
26#include "api/video_codecs/sdp_video_format.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020027#include "call/call.h"
28#include "call/flexfec_receive_stream.h"
29#include "call/video_receive_stream.h"
30#include "call/video_send_stream.h"
Steve Anton10542f22019-01-11 09:11:00 -080031#include "media/base/media_engine.h"
Jonas Oreland6d835922019-03-18 10:59:40 +010032#include "media/engine/unhandled_packets_buffer.h"
Steve Anton10542f22019-01-11 09:11:00 -080033#include "rtc_base/async_invoker.h"
34#include "rtc_base/critical_section.h"
35#include "rtc_base/network_route.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020036#include "rtc_base/thread_annotations.h"
37#include "rtc_base/thread_checker.h"
eladalonf1841382017-06-12 01:16:46 -070038
39namespace webrtc {
Magnus Jedvertd4b0c052017-09-14 10:24:54 +020040class VideoDecoderFactory;
Magnus Jedvertd4b0c052017-09-14 10:24:54 +020041class VideoEncoderFactory;
eladalonf1841382017-06-12 01:16:46 -070042struct MediaConfig;
Yves Gerey665174f2018-06-19 15:03:05 +020043} // namespace webrtc
eladalonf1841382017-06-12 01:16:46 -070044
45namespace rtc {
46class Thread;
47} // namespace rtc
48
49namespace cricket {
50
eladalonf1841382017-06-12 01:16:46 -070051class WebRtcVideoChannel;
eladalonf1841382017-06-12 01:16:46 -070052
eladalonf1841382017-06-12 01:16:46 -070053class UnsignalledSsrcHandler {
54 public:
55 enum Action {
56 kDropPacket,
57 kDeliverPacket,
58 };
59 virtual Action OnUnsignalledSsrc(WebRtcVideoChannel* channel,
60 uint32_t ssrc) = 0;
61 virtual ~UnsignalledSsrcHandler() = default;
62};
63
64// TODO(pbos): Remove, use external handlers only.
65class DefaultUnsignalledSsrcHandler : public UnsignalledSsrcHandler {
66 public:
67 DefaultUnsignalledSsrcHandler();
Yves Gerey665174f2018-06-19 15:03:05 +020068 Action OnUnsignalledSsrc(WebRtcVideoChannel* channel, uint32_t ssrc) override;
eladalonf1841382017-06-12 01:16:46 -070069
70 rtc::VideoSinkInterface<webrtc::VideoFrame>* GetDefaultSink() const;
71 void SetDefaultSink(WebRtcVideoChannel* channel,
72 rtc::VideoSinkInterface<webrtc::VideoFrame>* sink);
73
74 virtual ~DefaultUnsignalledSsrcHandler() = default;
75
76 private:
77 rtc::VideoSinkInterface<webrtc::VideoFrame>* default_sink_;
78};
79
80// WebRtcVideoEngine is used for the new native WebRTC Video API (webrtc:1667).
Sebastian Jansson84848f22018-11-16 10:40:36 +010081class WebRtcVideoEngine : public VideoEngineInterface {
eladalonf1841382017-06-12 01:16:46 -070082 public:
Magnus Jedvertd4b0c052017-09-14 10:24:54 +020083 // These video codec factories represents all video codecs, i.e. both software
84 // and external hardware codecs.
85 WebRtcVideoEngine(
86 std::unique_ptr<webrtc::VideoEncoderFactory> video_encoder_factory,
Jonas Orelanda3aa9bd2019-04-17 07:38:40 +020087 std::unique_ptr<webrtc::VideoDecoderFactory> video_decoder_factory);
Magnus Jedvertd4b0c052017-09-14 10:24:54 +020088
Sebastian Jansson84848f22018-11-16 10:40:36 +010089 ~WebRtcVideoEngine() override;
eladalonf1841382017-06-12 01:16:46 -070090
Sebastian Jansson84848f22018-11-16 10:40:36 +010091 VideoMediaChannel* CreateMediaChannel(
Benjamin Wrightbfb444c2018-10-15 10:20:24 -070092 webrtc::Call* call,
93 const MediaConfig& config,
94 const VideoOptions& options,
Jonas Orelanda3aa9bd2019-04-17 07:38:40 +020095 const webrtc::CryptoOptions& crypto_options,
96 webrtc::VideoBitrateAllocatorFactory* video_bitrate_allocator_factory)
97 override;
eladalonf1841382017-06-12 01:16:46 -070098
Sebastian Jansson84848f22018-11-16 10:40:36 +010099 std::vector<VideoCodec> codecs() const override;
100 RtpCapabilities GetCapabilities() const override;
eladalonf1841382017-06-12 01:16:46 -0700101
eladalonf1841382017-06-12 01:16:46 -0700102 private:
Magnus Jedvert59ab3532018-09-03 18:07:56 +0200103 const std::unique_ptr<webrtc::VideoDecoderFactory> decoder_factory_;
Magnus Jedvert07e0d012017-10-31 11:24:54 +0100104 const std::unique_ptr<webrtc::VideoEncoderFactory> encoder_factory_;
Jiawei Ouc2ebe212018-11-08 10:02:56 -0800105 const std::unique_ptr<webrtc::VideoBitrateAllocatorFactory>
106 bitrate_allocator_factory_;
eladalonf1841382017-06-12 01:16:46 -0700107};
108
109class WebRtcVideoChannel : public VideoMediaChannel, public webrtc::Transport {
110 public:
Jiawei Ouc2ebe212018-11-08 10:02:56 -0800111 WebRtcVideoChannel(
112 webrtc::Call* call,
113 const MediaConfig& config,
114 const VideoOptions& options,
115 const webrtc::CryptoOptions& crypto_options,
116 webrtc::VideoEncoderFactory* encoder_factory,
117 webrtc::VideoDecoderFactory* decoder_factory,
118 webrtc::VideoBitrateAllocatorFactory* bitrate_allocator_factory);
eladalonf1841382017-06-12 01:16:46 -0700119 ~WebRtcVideoChannel() override;
120
121 // VideoMediaChannel implementation
eladalonf1841382017-06-12 01:16:46 -0700122 bool SetSendParameters(const VideoSendParameters& params) override;
123 bool SetRecvParameters(const VideoRecvParameters& params) override;
124 webrtc::RtpParameters GetRtpSendParameters(uint32_t ssrc) const override;
Zach Steinba37b4b2018-01-23 15:02:36 -0800125 webrtc::RTCError SetRtpSendParameters(
126 uint32_t ssrc,
127 const webrtc::RtpParameters& parameters) override;
eladalonf1841382017-06-12 01:16:46 -0700128 webrtc::RtpParameters GetRtpReceiveParameters(uint32_t ssrc) const override;
129 bool SetRtpReceiveParameters(
130 uint32_t ssrc,
131 const webrtc::RtpParameters& parameters) override;
132 bool GetSendCodec(VideoCodec* send_codec) override;
133 bool SetSend(bool send) override;
134 bool SetVideoSend(
135 uint32_t ssrc,
eladalonf1841382017-06-12 01:16:46 -0700136 const VideoOptions* options,
137 rtc::VideoSourceInterface<webrtc::VideoFrame>* source) override;
138 bool AddSendStream(const StreamParams& sp) override;
139 bool RemoveSendStream(uint32_t ssrc) override;
140 bool AddRecvStream(const StreamParams& sp) override;
141 bool AddRecvStream(const StreamParams& sp, bool default_stream);
142 bool RemoveRecvStream(uint32_t ssrc) override;
143 bool SetSink(uint32_t ssrc,
144 rtc::VideoSinkInterface<webrtc::VideoFrame>* sink) override;
145 void FillBitrateInfo(BandwidthEstimationInfo* bwe_info) override;
146 bool GetStats(VideoMediaInfo* info) override;
147
Amit Hilbuche7a5f7b2019-03-12 11:10:27 -0700148 void OnPacketReceived(rtc::CopyOnWriteBuffer packet,
Niels Möllere6933812018-11-05 13:01:41 +0100149 int64_t packet_time_us) override;
Amit Hilbuche7a5f7b2019-03-12 11:10:27 -0700150 void OnRtcpReceived(rtc::CopyOnWriteBuffer packet,
Niels Möllere6933812018-11-05 13:01:41 +0100151 int64_t packet_time_us) override;
eladalonf1841382017-06-12 01:16:46 -0700152 void OnReadyToSend(bool ready) override;
153 void OnNetworkRouteChanged(const std::string& transport_name,
154 const rtc::NetworkRoute& network_route) override;
Anton Sukhanov4f08faa2019-05-21 11:12:57 -0700155 void SetInterface(
156 NetworkInterface* iface,
157 const webrtc::MediaTransportConfig& media_transport_config) override;
eladalonf1841382017-06-12 01:16:46 -0700158
Benjamin Wright192eeec2018-10-17 17:27:25 -0700159 // E2E Encrypted Video Frame API
160 // Set a frame decryptor to a particular ssrc that will intercept all
161 // incoming video frames and attempt to decrypt them before forwarding the
162 // result.
163 void SetFrameDecryptor(uint32_t ssrc,
164 rtc::scoped_refptr<webrtc::FrameDecryptorInterface>
165 frame_decryptor) override;
166 // Set a frame encryptor to a particular ssrc that will intercept all
167 // outgoing video frames and attempt to encrypt them and forward the result
168 // to the packetizer.
169 void SetFrameEncryptor(uint32_t ssrc,
170 rtc::scoped_refptr<webrtc::FrameEncryptorInterface>
171 frame_encryptor) override;
172
Ruslan Burakov493a6502019-02-27 15:32:48 +0100173 bool SetBaseMinimumPlayoutDelayMs(uint32_t ssrc, int delay_ms) override;
174
175 absl::optional<int> GetBaseMinimumPlayoutDelayMs(
176 uint32_t ssrc) const override;
177
eladalonf1841382017-06-12 01:16:46 -0700178 // Implemented for VideoMediaChannelTest.
Steve Antonef50b252019-03-01 15:15:38 -0800179 bool sending() const {
180 RTC_DCHECK_RUN_ON(&thread_checker_);
181 return sending_;
182 }
eladalonf1841382017-06-12 01:16:46 -0700183
Danil Chapovalov00c71832018-06-15 15:58:38 +0200184 absl::optional<uint32_t> GetDefaultReceiveStreamSsrc();
eladalonf1841382017-06-12 01:16:46 -0700185
Steve Antonef50b252019-03-01 15:15:38 -0800186 StreamParams unsignaled_stream_params() {
187 RTC_DCHECK_RUN_ON(&thread_checker_);
188 return unsignaled_stream_params_;
189 }
Seth Hampson5897a6e2018-04-03 11:16:33 -0700190
eladalonf1841382017-06-12 01:16:46 -0700191 // AdaptReason is used for expressing why a WebRtcVideoSendStream request
192 // a lower input frame size than the currently configured camera input frame
193 // size. There can be more than one reason OR:ed together.
194 enum AdaptReason {
195 ADAPTREASON_NONE = 0,
196 ADAPTREASON_CPU = 1,
197 ADAPTREASON_BANDWIDTH = 2,
198 };
199
sprang67561a62017-06-15 06:34:42 -0700200 static constexpr int kDefaultQpMax = 56;
201
Jonas Oreland49ac5952018-09-26 16:04:32 +0200202 std::vector<webrtc::RtpSource> GetSources(uint32_t ssrc) const override;
203
Jonas Oreland6d835922019-03-18 10:59:40 +0100204 // Take the buffered packets for |ssrcs| and feed them into DeliverPacket.
205 // This method does nothing unless unknown_ssrc_packet_buffer_ is configured.
206 void BackfillBufferedPackets(rtc::ArrayView<const uint32_t> ssrcs);
207
eladalonf1841382017-06-12 01:16:46 -0700208 private:
209 class WebRtcVideoReceiveStream;
210 struct VideoCodecSettings {
211 VideoCodecSettings();
212
213 // Checks if all members of |*this| are equal to the corresponding members
214 // of |other|.
215 bool operator==(const VideoCodecSettings& other) const;
216 bool operator!=(const VideoCodecSettings& other) const;
217
218 // Checks if all members of |a|, except |flexfec_payload_type|, are equal
219 // to the corresponding members of |b|.
220 static bool EqualsDisregardingFlexfec(const VideoCodecSettings& a,
221 const VideoCodecSettings& b);
222
223 VideoCodec codec;
224 webrtc::UlpfecConfig ulpfec;
225 int flexfec_payload_type;
226 int rtx_payload_type;
227 };
228
229 struct ChangedSendParameters {
230 // These optionals are unset if not changed.
Danil Chapovalov00c71832018-06-15 15:58:38 +0200231 absl::optional<VideoCodecSettings> codec;
232 absl::optional<std::vector<webrtc::RtpExtension>> rtp_header_extensions;
233 absl::optional<std::string> mid;
Johannes Kron9190b822018-10-29 11:22:05 +0100234 absl::optional<bool> extmap_allow_mixed;
Danil Chapovalov00c71832018-06-15 15:58:38 +0200235 absl::optional<int> max_bandwidth_bps;
236 absl::optional<bool> conference_mode;
237 absl::optional<webrtc::RtcpMode> rtcp_mode;
eladalonf1841382017-06-12 01:16:46 -0700238 };
239
240 struct ChangedRecvParameters {
241 // These optionals are unset if not changed.
Danil Chapovalov00c71832018-06-15 15:58:38 +0200242 absl::optional<std::vector<VideoCodecSettings>> codec_settings;
243 absl::optional<std::vector<webrtc::RtpExtension>> rtp_header_extensions;
eladalonf1841382017-06-12 01:16:46 -0700244 // Keep track of the FlexFEC payload type separately from |codec_settings|.
245 // This allows us to recreate the FlexfecReceiveStream separately from the
246 // VideoReceiveStream when the FlexFEC payload type is changed.
Danil Chapovalov00c71832018-06-15 15:58:38 +0200247 absl::optional<int> flexfec_payload_type;
eladalonf1841382017-06-12 01:16:46 -0700248 };
249
250 bool GetChangedSendParameters(const VideoSendParameters& params,
Steve Antonef50b252019-03-01 15:15:38 -0800251 ChangedSendParameters* changed_params) const
252 RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_);
eladalonf1841382017-06-12 01:16:46 -0700253 bool GetChangedRecvParameters(const VideoRecvParameters& params,
Steve Antonef50b252019-03-01 15:15:38 -0800254 ChangedRecvParameters* changed_params) const
255 RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_);
eladalonf1841382017-06-12 01:16:46 -0700256
257 void SetMaxSendBandwidth(int bps);
258
259 void ConfigureReceiverRtp(
260 webrtc::VideoReceiveStream::Config* config,
261 webrtc::FlexfecReceiveStream::Config* flexfec_config,
Steve Antonef50b252019-03-01 15:15:38 -0800262 const StreamParams& sp) const
263 RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_);
eladalonf1841382017-06-12 01:16:46 -0700264 bool ValidateSendSsrcAvailability(const StreamParams& sp) const
Steve Antonef50b252019-03-01 15:15:38 -0800265 RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_);
eladalonf1841382017-06-12 01:16:46 -0700266 bool ValidateReceiveSsrcAvailability(const StreamParams& sp) const
Steve Antonef50b252019-03-01 15:15:38 -0800267 RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_);
eladalonf1841382017-06-12 01:16:46 -0700268 void DeleteReceiveStream(WebRtcVideoReceiveStream* stream)
Steve Antonef50b252019-03-01 15:15:38 -0800269 RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_);
eladalonf1841382017-06-12 01:16:46 -0700270
271 static std::string CodecSettingsVectorToString(
272 const std::vector<VideoCodecSettings>& codecs);
273
274 // Wrapper for the sender part.
Christian Fremerey6c025412019-02-13 19:43:28 +0000275 class WebRtcVideoSendStream
276 : public rtc::VideoSourceInterface<webrtc::VideoFrame> {
eladalonf1841382017-06-12 01:16:46 -0700277 public:
278 WebRtcVideoSendStream(
279 webrtc::Call* call,
280 const StreamParams& sp,
281 webrtc::VideoSendStream::Config config,
282 const VideoOptions& options,
eladalonf1841382017-06-12 01:16:46 -0700283 bool enable_cpu_overuse_detection,
284 int max_bitrate_bps,
Danil Chapovalov00c71832018-06-15 15:58:38 +0200285 const absl::optional<VideoCodecSettings>& codec_settings,
286 const absl::optional<std::vector<webrtc::RtpExtension>>& rtp_extensions,
eladalonf1841382017-06-12 01:16:46 -0700287 const VideoSendParameters& send_params);
288 virtual ~WebRtcVideoSendStream();
289
290 void SetSendParameters(const ChangedSendParameters& send_params);
Zach Steinba37b4b2018-01-23 15:02:36 -0800291 webrtc::RTCError SetRtpParameters(const webrtc::RtpParameters& parameters);
eladalonf1841382017-06-12 01:16:46 -0700292 webrtc::RtpParameters GetRtpParameters() const;
293
Benjamin Wright192eeec2018-10-17 17:27:25 -0700294 void SetFrameEncryptor(
295 rtc::scoped_refptr<webrtc::FrameEncryptorInterface> frame_encryptor);
296
Christian Fremerey6c025412019-02-13 19:43:28 +0000297 // Implements rtc::VideoSourceInterface<webrtc::VideoFrame>.
298 // WebRtcVideoSendStream acts as a source to the webrtc::VideoSendStream
299 // in |stream_|. This is done to proxy VideoSinkWants from the encoder to
300 // the worker thread.
301 void AddOrUpdateSink(rtc::VideoSinkInterface<webrtc::VideoFrame>* sink,
302 const rtc::VideoSinkWants& wants) override;
303 void RemoveSink(rtc::VideoSinkInterface<webrtc::VideoFrame>* sink) override;
304
Niels Möllerff40b142018-04-09 08:49:14 +0200305 bool SetVideoSend(const VideoOptions* options,
eladalonf1841382017-06-12 01:16:46 -0700306 rtc::VideoSourceInterface<webrtc::VideoFrame>* source);
307
308 void SetSend(bool send);
309
310 const std::vector<uint32_t>& GetSsrcs() const;
311 VideoSenderInfo GetVideoSenderInfo(bool log_stats);
312 void FillBitrateInfo(BandwidthEstimationInfo* bwe_info);
313
314 private:
315 // Parameters needed to reconstruct the underlying stream.
316 // webrtc::VideoSendStream doesn't support setting a lot of options on the
317 // fly, so when those need to be changed we tear down and reconstruct with
318 // similar parameters depending on which options changed etc.
319 struct VideoSendStreamParameters {
320 VideoSendStreamParameters(
321 webrtc::VideoSendStream::Config config,
322 const VideoOptions& options,
323 int max_bitrate_bps,
Danil Chapovalov00c71832018-06-15 15:58:38 +0200324 const absl::optional<VideoCodecSettings>& codec_settings);
eladalonf1841382017-06-12 01:16:46 -0700325 webrtc::VideoSendStream::Config config;
326 VideoOptions options;
327 int max_bitrate_bps;
328 bool conference_mode;
Danil Chapovalov00c71832018-06-15 15:58:38 +0200329 absl::optional<VideoCodecSettings> codec_settings;
eladalonf1841382017-06-12 01:16:46 -0700330 // Sent resolutions + bitrates etc. by the underlying VideoSendStream,
331 // typically changes when setting a new resolution or reconfiguring
332 // bitrates.
333 webrtc::VideoEncoderConfig encoder_config;
334 };
335
eladalonf1841382017-06-12 01:16:46 -0700336 rtc::scoped_refptr<webrtc::VideoEncoderConfig::EncoderSpecificSettings>
337 ConfigureVideoEncoderSettings(const VideoCodec& codec);
Niels Möller5bf8ccd2018-03-15 14:16:11 +0100338 void SetCodec(const VideoCodecSettings& codec);
eladalonf1841382017-06-12 01:16:46 -0700339 void RecreateWebRtcStream();
340 webrtc::VideoEncoderConfig CreateVideoEncoderConfig(
341 const VideoCodec& codec) const;
342 void ReconfigureEncoder();
eladalonf1841382017-06-12 01:16:46 -0700343
344 // Calls Start or Stop according to whether or not |sending_| is true,
345 // and whether or not the encoding in |rtp_parameters_| is active.
346 void UpdateSendState();
347
Taylor Brandstetter49fcc102018-05-16 14:20:41 -0700348 webrtc::DegradationPreference GetDegradationPreference() const
349 RTC_EXCLUSIVE_LOCKS_REQUIRED(&thread_checker_);
eladalonf1841382017-06-12 01:16:46 -0700350
351 rtc::ThreadChecker thread_checker_;
352 rtc::AsyncInvoker invoker_;
353 rtc::Thread* worker_thread_;
Niels Möller1e062892018-02-07 10:18:32 +0100354 const std::vector<uint32_t> ssrcs_ RTC_GUARDED_BY(&thread_checker_);
355 const std::vector<SsrcGroup> ssrc_groups_ RTC_GUARDED_BY(&thread_checker_);
eladalonf1841382017-06-12 01:16:46 -0700356 webrtc::Call* const call_;
357 const bool enable_cpu_overuse_detection_;
358 rtc::VideoSourceInterface<webrtc::VideoFrame>* source_
Niels Möller1e062892018-02-07 10:18:32 +0100359 RTC_GUARDED_BY(&thread_checker_);
eladalonf1841382017-06-12 01:16:46 -0700360
Niels Möller1e062892018-02-07 10:18:32 +0100361 webrtc::VideoSendStream* stream_ RTC_GUARDED_BY(&thread_checker_);
Christian Fremerey6c025412019-02-13 19:43:28 +0000362 rtc::VideoSinkInterface<webrtc::VideoFrame>* encoder_sink_
363 RTC_GUARDED_BY(&thread_checker_);
eladalonf1841382017-06-12 01:16:46 -0700364 // Contains settings that are the same for all streams in the MediaChannel,
365 // such as codecs, header extensions, and the global bitrate limit for the
366 // entire channel.
Niels Möller1e062892018-02-07 10:18:32 +0100367 VideoSendStreamParameters parameters_ RTC_GUARDED_BY(&thread_checker_);
eladalonf1841382017-06-12 01:16:46 -0700368 // Contains settings that are unique for each stream, such as max_bitrate.
369 // Does *not* contain codecs, however.
370 // TODO(skvlad): Move ssrcs_ and ssrc_groups_ into rtp_parameters_.
371 // TODO(skvlad): Combine parameters_ and rtp_parameters_ once we have only
372 // one stream per MediaChannel.
Niels Möller1e062892018-02-07 10:18:32 +0100373 webrtc::RtpParameters rtp_parameters_ RTC_GUARDED_BY(&thread_checker_);
eladalonf1841382017-06-12 01:16:46 -0700374
Niels Möller1e062892018-02-07 10:18:32 +0100375 bool sending_ RTC_GUARDED_BY(&thread_checker_);
eladalonf1841382017-06-12 01:16:46 -0700376 };
377
378 // Wrapper for the receiver part, contains configs etc. that are needed to
379 // reconstruct the underlying VideoReceiveStream.
380 class WebRtcVideoReceiveStream
381 : public rtc::VideoSinkInterface<webrtc::VideoFrame> {
382 public:
383 WebRtcVideoReceiveStream(
Jonas Oreland6d835922019-03-18 10:59:40 +0100384 WebRtcVideoChannel* channel,
eladalonf1841382017-06-12 01:16:46 -0700385 webrtc::Call* call,
386 const StreamParams& sp,
387 webrtc::VideoReceiveStream::Config config,
Magnus Jedvert59ab3532018-09-03 18:07:56 +0200388 webrtc::VideoDecoderFactory* decoder_factory,
eladalonf1841382017-06-12 01:16:46 -0700389 bool default_stream,
390 const std::vector<VideoCodecSettings>& recv_codecs,
391 const webrtc::FlexfecReceiveStream::Config& flexfec_config);
392 ~WebRtcVideoReceiveStream();
393
394 const std::vector<uint32_t>& GetSsrcs() const;
Florent Castelliabe301f2018-06-12 18:33:49 +0200395
Jonas Oreland49ac5952018-09-26 16:04:32 +0200396 std::vector<webrtc::RtpSource> GetSources();
397
Florent Castelliabe301f2018-06-12 18:33:49 +0200398 // Does not return codecs, they are filled by the owning WebRtcVideoChannel.
399 webrtc::RtpParameters GetRtpParameters() const;
eladalonf1841382017-06-12 01:16:46 -0700400
401 void SetLocalSsrc(uint32_t local_ssrc);
402 // TODO(deadbeef): Move these feedback parameters into the recv parameters.
403 void SetFeedbackParameters(bool nack_enabled,
404 bool remb_enabled,
405 bool transport_cc_enabled,
406 webrtc::RtcpMode rtcp_mode);
407 void SetRecvParameters(const ChangedRecvParameters& recv_params);
408
409 void OnFrame(const webrtc::VideoFrame& frame) override;
410 bool IsDefaultStream() const;
411
Benjamin Wright192eeec2018-10-17 17:27:25 -0700412 void SetFrameDecryptor(
413 rtc::scoped_refptr<webrtc::FrameDecryptorInterface> frame_decryptor);
414
Ruslan Burakov493a6502019-02-27 15:32:48 +0100415 bool SetBaseMinimumPlayoutDelayMs(int delay_ms);
416
417 int GetBaseMinimumPlayoutDelayMs() const;
418
eladalonf1841382017-06-12 01:16:46 -0700419 void SetSink(rtc::VideoSinkInterface<webrtc::VideoFrame>* sink);
420
421 VideoReceiverInfo GetVideoReceiverInfo(bool log_stats);
422
423 private:
eladalonf1841382017-06-12 01:16:46 -0700424 void RecreateWebRtcVideoStream();
425 void MaybeRecreateWebRtcFlexfecStream();
426
eladalonc0d481a2017-08-02 07:39:07 -0700427 void MaybeAssociateFlexfecWithVideo();
428 void MaybeDissociateFlexfecFromVideo();
429
Niels Möllercbcbc222018-09-28 09:07:24 +0200430 void ConfigureCodecs(const std::vector<VideoCodecSettings>& recv_codecs);
eladalonf1841382017-06-12 01:16:46 -0700431 void ConfigureFlexfecCodec(int flexfec_payload_type);
eladalonf1841382017-06-12 01:16:46 -0700432
433 std::string GetCodecNameFromPayloadType(int payload_type);
434
Jonas Oreland6d835922019-03-18 10:59:40 +0100435 WebRtcVideoChannel* const channel_;
eladalonf1841382017-06-12 01:16:46 -0700436 webrtc::Call* const call_;
Niels Möllercbcbc222018-09-28 09:07:24 +0200437 const StreamParams stream_params_;
eladalonf1841382017-06-12 01:16:46 -0700438
439 // Both |stream_| and |flexfec_stream_| are managed by |this|. They are
440 // destroyed by calling call_->DestroyVideoReceiveStream and
441 // call_->DestroyFlexfecReceiveStream, respectively.
442 webrtc::VideoReceiveStream* stream_;
443 const bool default_stream_;
444 webrtc::VideoReceiveStream::Config config_;
445 webrtc::FlexfecReceiveStream::Config flexfec_config_;
446 webrtc::FlexfecReceiveStream* flexfec_stream_;
447
Niels Möllercbcbc222018-09-28 09:07:24 +0200448 webrtc::VideoDecoderFactory* const decoder_factory_;
eladalonf1841382017-06-12 01:16:46 -0700449
450 rtc::CriticalSection sink_lock_;
danilchapa37de392017-09-09 04:17:22 -0700451 rtc::VideoSinkInterface<webrtc::VideoFrame>* sink_
452 RTC_GUARDED_BY(sink_lock_);
eladalonf1841382017-06-12 01:16:46 -0700453 // Expands remote RTP timestamps to int64_t to be able to estimate how long
454 // the stream has been running.
455 rtc::TimestampWrapAroundHandler timestamp_wraparound_handler_
danilchapa37de392017-09-09 04:17:22 -0700456 RTC_GUARDED_BY(sink_lock_);
457 int64_t first_frame_timestamp_ RTC_GUARDED_BY(sink_lock_);
eladalonf1841382017-06-12 01:16:46 -0700458 // Start NTP time is estimated as current remote NTP time (estimated from
459 // RTCP) minus the elapsed time, as soon as remote NTP time is available.
danilchapa37de392017-09-09 04:17:22 -0700460 int64_t estimated_remote_start_ntp_time_ms_ RTC_GUARDED_BY(sink_lock_);
eladalonf1841382017-06-12 01:16:46 -0700461 };
462
463 void Construct(webrtc::Call* call, WebRtcVideoEngine* engine);
464
465 bool SendRtp(const uint8_t* data,
466 size_t len,
467 const webrtc::PacketOptions& options) override;
468 bool SendRtcp(const uint8_t* data, size_t len) override;
469
470 static std::vector<VideoCodecSettings> MapCodecs(
471 const std::vector<VideoCodec>& codecs);
472 // Select what video codec will be used for sending, i.e. what codec is used
473 // for local encoding, based on supported remote codecs. The first remote
474 // codec that is supported locally will be selected.
Danil Chapovalov00c71832018-06-15 15:58:38 +0200475 absl::optional<VideoCodecSettings> SelectSendVideoCodec(
Steve Antonef50b252019-03-01 15:15:38 -0800476 const std::vector<VideoCodecSettings>& remote_mapped_codecs) const
477 RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_);
eladalonf1841382017-06-12 01:16:46 -0700478
479 static bool NonFlexfecReceiveCodecsHaveChanged(
480 std::vector<VideoCodecSettings> before,
481 std::vector<VideoCodecSettings> after);
482
Steve Antonef50b252019-03-01 15:15:38 -0800483 void FillSenderStats(VideoMediaInfo* info, bool log_stats)
484 RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_);
485 void FillReceiverStats(VideoMediaInfo* info, bool log_stats)
486 RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_);
eladalonf1841382017-06-12 01:16:46 -0700487 void FillBandwidthEstimationStats(const webrtc::Call::Stats& stats,
Steve Antonef50b252019-03-01 15:15:38 -0800488 VideoMediaInfo* info)
489 RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_);
490 void FillSendAndReceiveCodecStats(VideoMediaInfo* video_media_info)
491 RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_);
eladalonf1841382017-06-12 01:16:46 -0700492
493 rtc::ThreadChecker thread_checker_;
494
Steve Antonef50b252019-03-01 15:15:38 -0800495 uint32_t rtcp_receiver_report_ssrc_ RTC_GUARDED_BY(thread_checker_);
496 bool sending_ RTC_GUARDED_BY(thread_checker_);
497 webrtc::Call* const call_ RTC_GUARDED_BY(thread_checker_);
eladalonf1841382017-06-12 01:16:46 -0700498
Steve Antonef50b252019-03-01 15:15:38 -0800499 DefaultUnsignalledSsrcHandler default_unsignalled_ssrc_handler_
500 RTC_GUARDED_BY(thread_checker_);
501 UnsignalledSsrcHandler* const unsignalled_ssrc_handler_
502 RTC_GUARDED_BY(thread_checker_);
eladalonf1841382017-06-12 01:16:46 -0700503
Ruslan Burakov493a6502019-02-27 15:32:48 +0100504 // Delay for unsignaled streams, which may be set before the stream exists.
Steve Antonef50b252019-03-01 15:15:38 -0800505 int default_recv_base_minimum_delay_ms_ RTC_GUARDED_BY(thread_checker_) = 0;
Ruslan Burakov493a6502019-02-27 15:32:48 +0100506
Steve Antonef50b252019-03-01 15:15:38 -0800507 const MediaConfig::Video video_config_ RTC_GUARDED_BY(thread_checker_);
eladalonf1841382017-06-12 01:16:46 -0700508
eladalonf1841382017-06-12 01:16:46 -0700509 // Using primary-ssrc (first ssrc) as key.
510 std::map<uint32_t, WebRtcVideoSendStream*> send_streams_
Steve Antonef50b252019-03-01 15:15:38 -0800511 RTC_GUARDED_BY(thread_checker_);
eladalonf1841382017-06-12 01:16:46 -0700512 std::map<uint32_t, WebRtcVideoReceiveStream*> receive_streams_
Steve Antonef50b252019-03-01 15:15:38 -0800513 RTC_GUARDED_BY(thread_checker_);
514 std::set<uint32_t> send_ssrcs_ RTC_GUARDED_BY(thread_checker_);
515 std::set<uint32_t> receive_ssrcs_ RTC_GUARDED_BY(thread_checker_);
eladalonf1841382017-06-12 01:16:46 -0700516
Steve Antonef50b252019-03-01 15:15:38 -0800517 absl::optional<VideoCodecSettings> send_codec_
518 RTC_GUARDED_BY(thread_checker_);
519 absl::optional<std::vector<webrtc::RtpExtension>> send_rtp_extensions_
520 RTC_GUARDED_BY(thread_checker_);
eladalonf1841382017-06-12 01:16:46 -0700521
Steve Antonef50b252019-03-01 15:15:38 -0800522 webrtc::VideoEncoderFactory* const encoder_factory_
523 RTC_GUARDED_BY(thread_checker_);
524 webrtc::VideoDecoderFactory* const decoder_factory_
525 RTC_GUARDED_BY(thread_checker_);
526 webrtc::VideoBitrateAllocatorFactory* const bitrate_allocator_factory_
527 RTC_GUARDED_BY(thread_checker_);
528 std::vector<VideoCodecSettings> recv_codecs_ RTC_GUARDED_BY(thread_checker_);
529 std::vector<webrtc::RtpExtension> recv_rtp_extensions_
530 RTC_GUARDED_BY(thread_checker_);
eladalonf1841382017-06-12 01:16:46 -0700531 // See reason for keeping track of the FlexFEC payload type separately in
532 // comment in WebRtcVideoChannel::ChangedRecvParameters.
Steve Antonef50b252019-03-01 15:15:38 -0800533 int recv_flexfec_payload_type_ RTC_GUARDED_BY(thread_checker_);
534 webrtc::BitrateConstraints bitrate_config_ RTC_GUARDED_BY(thread_checker_);
eladalonf1841382017-06-12 01:16:46 -0700535 // TODO(deadbeef): Don't duplicate information between
536 // send_params/recv_params, rtp_extensions, options, etc.
Steve Antonef50b252019-03-01 15:15:38 -0800537 VideoSendParameters send_params_ RTC_GUARDED_BY(thread_checker_);
Steve Antonef50b252019-03-01 15:15:38 -0800538 VideoOptions default_send_options_ RTC_GUARDED_BY(thread_checker_);
539 VideoRecvParameters recv_params_ RTC_GUARDED_BY(thread_checker_);
540 int64_t last_stats_log_ms_ RTC_GUARDED_BY(thread_checker_);
541 const bool discard_unknown_ssrc_packets_ RTC_GUARDED_BY(thread_checker_);
Seth Hampson5897a6e2018-04-03 11:16:33 -0700542 // This is a stream param that comes from the remote description, but wasn't
543 // signaled with any a=ssrc lines. It holds information that was signaled
544 // before the unsignaled receive stream is created when the first packet is
545 // received.
Steve Antonef50b252019-03-01 15:15:38 -0800546 StreamParams unsignaled_stream_params_ RTC_GUARDED_BY(thread_checker_);
Benjamin Wright192eeec2018-10-17 17:27:25 -0700547 // Per peer connection crypto options that last for the lifetime of the peer
548 // connection.
Steve Antonef50b252019-03-01 15:15:38 -0800549 const webrtc::CryptoOptions crypto_options_ RTC_GUARDED_BY(thread_checker_);
Jonas Oreland6d835922019-03-18 10:59:40 +0100550
551 // Buffer for unhandled packets.
552 std::unique_ptr<UnhandledPacketsBuffer> unknown_ssrc_packet_buffer_
553 RTC_GUARDED_BY(thread_checker_);
eladalonf1841382017-06-12 01:16:46 -0700554};
555
ilnik6b826ef2017-06-16 06:53:48 -0700556class EncoderStreamFactory
557 : public webrtc::VideoEncoderConfig::VideoStreamFactoryInterface {
558 public:
559 EncoderStreamFactory(std::string codec_name,
560 int max_qp,
Seth Hampson1370e302018-02-07 08:50:36 -0800561 bool is_screenshare,
562 bool screenshare_config_explicitly_enabled);
ilnik6b826ef2017-06-16 06:53:48 -0700563
564 private:
565 std::vector<webrtc::VideoStream> CreateEncoderStreams(
566 int width,
567 int height,
568 const webrtc::VideoEncoderConfig& encoder_config) override;
569
570 const std::string codec_name_;
571 const int max_qp_;
Seth Hampson1370e302018-02-07 08:50:36 -0800572 const bool is_screenshare_;
573 // Allows a screenshare specific configuration, which enables temporal
574 // layering and allows simulcast.
575 const bool screenshare_config_explicitly_enabled_;
ilnik6b826ef2017-06-16 06:53:48 -0700576};
577
eladalonf1841382017-06-12 01:16:46 -0700578} // namespace cricket
579
Steve Anton10542f22019-01-11 09:11:00 -0800580#endif // MEDIA_ENGINE_WEBRTC_VIDEO_ENGINE_H_