blob: a0fde8f9b2907f7a9a7120b47715f8eae3502cb7 [file] [log] [blame]
pbos@webrtc.orgb5a22b12014-05-13 11:07:01 +00001/*
2 * libjingle
3 * Copyright 2014 Google Inc.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions are met:
7 *
8 * 1. Redistributions of source code must retain the above copyright notice,
9 * this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright notice,
11 * this list of conditions and the following disclaimer in the documentation
12 * and/or other materials provided with the distribution.
13 * 3. The name of the author may not be used to endorse or promote products
14 * derived from this software without specific prior written permission.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
17 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
18 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
19 * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
20 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
21 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
22 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
23 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
24 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
25 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 */
27
28#ifndef TALK_MEDIA_WEBRTC_WEBRTCVIDEOENGINE2_H_
29#define TALK_MEDIA_WEBRTC_WEBRTCVIDEOENGINE2_H_
30
31#include <map>
pbos@webrtc.orgb5a22b12014-05-13 11:07:01 +000032#include <string>
buildbot@webrtc.orga09a9992014-08-13 17:26:08 +000033#include <vector>
pbos@webrtc.orgb5a22b12014-05-13 11:07:01 +000034
pbos@webrtc.orgb5a22b12014-05-13 11:07:01 +000035#include "talk/media/base/mediaengine.h"
36#include "talk/media/webrtc/webrtcvideochannelfactory.h"
pbos@webrtc.org0a2087a2014-09-23 09:40:22 +000037#include "talk/media/webrtc/webrtcvideodecoderfactory.h"
38#include "talk/media/webrtc/webrtcvideoencoderfactory.h"
buildbot@webrtc.orga09a9992014-08-13 17:26:08 +000039#include "webrtc/base/cpumonitor.h"
pbos@webrtc.org575d1262014-10-08 14:48:08 +000040#include "webrtc/base/criticalsection.h"
buildbot@webrtc.orga09a9992014-08-13 17:26:08 +000041#include "webrtc/base/scoped_ptr.h"
pbos@webrtc.org38344ed2014-09-24 06:05:00 +000042#include "webrtc/base/thread_annotations.h"
pbos@webrtc.org42684be2014-10-03 11:25:45 +000043#include "webrtc/call.h"
pbos@webrtc.orgb5a22b12014-05-13 11:07:01 +000044#include "webrtc/transport.h"
Thiago Farina9bfe3da2015-04-10 12:52:13 +020045#include "webrtc/video_frame.h"
buildbot@webrtc.orga09a9992014-08-13 17:26:08 +000046#include "webrtc/video_receive_stream.h"
pbos@webrtc.orgb5a22b12014-05-13 11:07:01 +000047#include "webrtc/video_renderer.h"
48#include "webrtc/video_send_stream.h"
49
50namespace webrtc {
pbos@webrtc.orgb5a22b12014-05-13 11:07:01 +000051class VideoDecoder;
52class VideoEncoder;
pbos@webrtc.orgb5a22b12014-05-13 11:07:01 +000053}
54
buildbot@webrtc.orgd4e598d2014-07-29 17:36:52 +000055namespace rtc {
pbos@webrtc.orgb5a22b12014-05-13 11:07:01 +000056class CpuMonitor;
57class Thread;
buildbot@webrtc.orgd4e598d2014-07-29 17:36:52 +000058} // namespace rtc
pbos@webrtc.orgb5a22b12014-05-13 11:07:01 +000059
60namespace cricket {
61
62class VideoCapturer;
63class VideoFrame;
64class VideoProcessor;
65class VideoRenderer;
66class VoiceMediaChannel;
pbos@webrtc.orgb5a22b12014-05-13 11:07:01 +000067class WebRtcDecoderObserver;
68class WebRtcEncoderObserver;
69class WebRtcLocalStreamInfo;
70class WebRtcRenderAdapter;
71class WebRtcVideoChannelRecvInfo;
72class WebRtcVideoChannelSendInfo;
pbos@webrtc.orgb5a22b12014-05-13 11:07:01 +000073class WebRtcVoiceEngine;
74
75struct CapturedFrame;
76struct Device;
77
Peter Boström81ea54e2015-05-07 11:41:09 +020078// Exposed here for unittests.
79std::vector<VideoCodec> DefaultVideoCodecList();
80
pbos@webrtc.orgafb554f42014-08-12 23:17:13 +000081class UnsignalledSsrcHandler {
82 public:
83 enum Action {
84 kDropPacket,
85 kDeliverPacket,
86 };
pbos@webrtc.orga2a6fe62015-03-06 15:35:19 +000087 virtual Action OnUnsignalledSsrc(WebRtcVideoChannel2* channel,
pbos@webrtc.orgafb554f42014-08-12 23:17:13 +000088 uint32_t ssrc) = 0;
89};
90
91// TODO(pbos): Remove, use external handlers only.
92class DefaultUnsignalledSsrcHandler : public UnsignalledSsrcHandler {
93 public:
94 DefaultUnsignalledSsrcHandler();
pbos@webrtc.orga2a6fe62015-03-06 15:35:19 +000095 Action OnUnsignalledSsrc(WebRtcVideoChannel2* channel,
96 uint32_t ssrc) override;
pbos@webrtc.orgafb554f42014-08-12 23:17:13 +000097
98 VideoRenderer* GetDefaultRenderer() const;
99 void SetDefaultRenderer(VideoMediaChannel* channel, VideoRenderer* renderer);
100
101 private:
102 uint32_t default_recv_ssrc_;
103 VideoRenderer* default_renderer_;
104};
105
pbos@webrtc.org42684be2014-10-03 11:25:45 +0000106// CallFactory, overridden for testing to verify that webrtc::Call is configured
107// properly.
108class WebRtcCallFactory {
109 public:
110 virtual ~WebRtcCallFactory();
111 virtual webrtc::Call* CreateCall(const webrtc::Call::Config& config);
112};
113
pbos@webrtc.orgb5a22b12014-05-13 11:07:01 +0000114// WebRtcVideoEngine2 is used for the new native WebRTC Video API (webrtc:1667).
buildbot@webrtc.org3c16d8b2014-10-13 06:35:10 +0000115class WebRtcVideoEngine2 : public sigslot::has_slots<> {
pbos@webrtc.orgb5a22b12014-05-13 11:07:01 +0000116 public:
pbos@webrtc.orgf1f0d9a2015-03-02 13:30:15 +0000117 explicit WebRtcVideoEngine2(WebRtcVoiceEngine* voice_engine);
pbos@webrtc.orgb648b9d2014-08-26 11:08:06 +0000118 virtual ~WebRtcVideoEngine2();
119
pbos@webrtc.org42684be2014-10-03 11:25:45 +0000120 // Used for testing to be able to check and use the webrtc::Call config.
121 void SetCallFactory(WebRtcCallFactory* call_factory);
pbos@webrtc.orgb5a22b12014-05-13 11:07:01 +0000122
123 // Basic video engine implementation.
buildbot@webrtc.orgd4e598d2014-07-29 17:36:52 +0000124 bool Init(rtc::Thread* worker_thread);
pbos@webrtc.orgb5a22b12014-05-13 11:07:01 +0000125 void Terminate();
126
127 int GetCapabilities();
pbos@webrtc.orgb5a22b12014-05-13 11:07:01 +0000128 bool SetDefaultEncoderConfig(const VideoEncoderConfig& config);
pbos@webrtc.orgb5a22b12014-05-13 11:07:01 +0000129
buildbot@webrtc.org1ecbe452014-10-14 20:29:28 +0000130 WebRtcVideoChannel2* CreateChannel(const VideoOptions& options,
131 VoiceMediaChannel* voice_channel);
pbos@webrtc.orgb5a22b12014-05-13 11:07:01 +0000132
133 const std::vector<VideoCodec>& codecs() const;
134 const std::vector<RtpHeaderExtension>& rtp_header_extensions() const;
135 void SetLogging(int min_sev, const char* filter);
136
pbos@webrtc.org0a2087a2014-09-23 09:40:22 +0000137 // Set a WebRtcVideoDecoderFactory for external decoding. Video engine does
138 // not take the ownership of |decoder_factory|. The caller needs to make sure
139 // that |decoder_factory| outlives the video engine.
140 void SetExternalDecoderFactory(WebRtcVideoDecoderFactory* decoder_factory);
141 // Set a WebRtcVideoEncoderFactory for external encoding. Video engine does
142 // not take the ownership of |encoder_factory|. The caller needs to make sure
143 // that |encoder_factory| outlives the video engine.
144 virtual void SetExternalEncoderFactory(
145 WebRtcVideoEncoderFactory* encoder_factory);
146
pbos@webrtc.orgb5a22b12014-05-13 11:07:01 +0000147 bool EnableTimedRender();
pbos@webrtc.orgb5a22b12014-05-13 11:07:01 +0000148
pbos@webrtc.orgb5a22b12014-05-13 11:07:01 +0000149 bool FindCodec(const VideoCodec& in);
150 bool CanSendCodec(const VideoCodec& in,
151 const VideoCodec& current,
152 VideoCodec* out);
153 // Check whether the supplied trace should be ignored.
154 bool ShouldIgnoreTrace(const std::string& trace);
155
pbos@webrtc.orgb5a22b12014-05-13 11:07:01 +0000156 private:
pbos@webrtc.org7fe1e032014-10-14 04:25:33 +0000157 std::vector<VideoCodec> GetSupportedCodecs() const;
158
buildbot@webrtc.orgd4e598d2014-07-29 17:36:52 +0000159 rtc::Thread* worker_thread_;
pbos@webrtc.orgb5a22b12014-05-13 11:07:01 +0000160 WebRtcVoiceEngine* voice_engine_;
161 std::vector<VideoCodec> video_codecs_;
162 std::vector<RtpHeaderExtension> rtp_header_extensions_;
pbos@webrtc.orgb5a22b12014-05-13 11:07:01 +0000163
164 bool initialized_;
165
pbos@webrtc.org42684be2014-10-03 11:25:45 +0000166 WebRtcCallFactory default_call_factory_;
167 WebRtcCallFactory* call_factory_;
168
pbos@webrtc.org0a2087a2014-09-23 09:40:22 +0000169 WebRtcVideoDecoderFactory* external_decoder_factory_;
170 WebRtcVideoEncoderFactory* external_encoder_factory_;
pbos@webrtc.orgf18fba22015-01-14 16:26:23 +0000171 rtc::scoped_ptr<WebRtcVideoEncoderFactory> simulcast_encoder_factory_;
pbos@webrtc.orgb5a22b12014-05-13 11:07:01 +0000172};
173
buildbot@webrtc.orgd4e598d2014-07-29 17:36:52 +0000174class WebRtcVideoChannel2 : public rtc::MessageHandler,
pbos@webrtc.orgb5a22b12014-05-13 11:07:01 +0000175 public VideoMediaChannel,
pbos@webrtc.org42684be2014-10-03 11:25:45 +0000176 public webrtc::newapi::Transport,
177 public webrtc::LoadObserver {
pbos@webrtc.orgb5a22b12014-05-13 11:07:01 +0000178 public:
pbos@webrtc.org42684be2014-10-03 11:25:45 +0000179 WebRtcVideoChannel2(WebRtcCallFactory* call_factory,
pbos@webrtc.org3bf3d232014-10-31 12:59:34 +0000180 WebRtcVoiceEngine* voice_engine,
pbos@webrtc.orgb5a22b12014-05-13 11:07:01 +0000181 VoiceMediaChannel* voice_channel,
pbos@webrtc.orgfa553ef2014-10-20 11:07:07 +0000182 const VideoOptions& options,
pbos@webrtc.org7fe1e032014-10-14 04:25:33 +0000183 WebRtcVideoEncoderFactory* external_encoder_factory,
pbos@webrtc.orgf1c8b902015-01-14 17:29:27 +0000184 WebRtcVideoDecoderFactory* external_decoder_factory);
pbos@webrtc.orgb5a22b12014-05-13 11:07:01 +0000185 ~WebRtcVideoChannel2();
186 bool Init();
187
188 // VideoMediaChannel implementation
kjellander@webrtc.org14665ff2015-03-04 12:58:35 +0000189 bool SetRecvCodecs(const std::vector<VideoCodec>& codecs) override;
190 bool SetSendCodecs(const std::vector<VideoCodec>& codecs) override;
191 bool GetSendCodec(VideoCodec* send_codec) override;
192 bool SetSendStreamFormat(uint32 ssrc, const VideoFormat& format) override;
193 bool SetRender(bool render) override;
194 bool SetSend(bool send) override;
pbos@webrtc.orgb5a22b12014-05-13 11:07:01 +0000195
kjellander@webrtc.org14665ff2015-03-04 12:58:35 +0000196 bool AddSendStream(const StreamParams& sp) override;
197 bool RemoveSendStream(uint32 ssrc) override;
198 bool AddRecvStream(const StreamParams& sp) override;
pbos@webrtc.orga2a6fe62015-03-06 15:35:19 +0000199 bool AddRecvStream(const StreamParams& sp, bool default_stream);
kjellander@webrtc.org14665ff2015-03-04 12:58:35 +0000200 bool RemoveRecvStream(uint32 ssrc) override;
201 bool SetRenderer(uint32 ssrc, VideoRenderer* renderer) override;
202 bool GetStats(VideoMediaInfo* info) override;
203 bool SetCapturer(uint32 ssrc, VideoCapturer* capturer) override;
204 bool SendIntraFrame() override;
205 bool RequestIntraFrame() override;
pbos@webrtc.orgb5a22b12014-05-13 11:07:01 +0000206
kjellander@webrtc.org14665ff2015-03-04 12:58:35 +0000207 void OnPacketReceived(rtc::Buffer* packet,
208 const rtc::PacketTime& packet_time) override;
209 void OnRtcpReceived(rtc::Buffer* packet,
210 const rtc::PacketTime& packet_time) override;
211 void OnReadyToSend(bool ready) override;
212 bool MuteStream(uint32 ssrc, bool mute) override;
pbos@webrtc.org587ef602014-06-16 17:32:02 +0000213
214 // Set send/receive RTP header extensions. This must be done before creating
215 // streams as it only has effect on future streams.
kjellander@webrtc.org14665ff2015-03-04 12:58:35 +0000216 bool SetRecvRtpHeaderExtensions(
pbos@webrtc.org0d852d52015-02-09 15:14:36 +0000217 const std::vector<RtpHeaderExtension>& extensions) override;
kjellander@webrtc.org14665ff2015-03-04 12:58:35 +0000218 bool SetSendRtpHeaderExtensions(
pbos@webrtc.org0d852d52015-02-09 15:14:36 +0000219 const std::vector<RtpHeaderExtension>& extensions) override;
kjellander@webrtc.org14665ff2015-03-04 12:58:35 +0000220 bool SetMaxSendBandwidth(int bps) override;
221 bool SetOptions(const VideoOptions& options) override;
222 bool GetOptions(VideoOptions* options) const override {
pbos@webrtc.orgb5a22b12014-05-13 11:07:01 +0000223 *options = options_;
224 return true;
225 }
kjellander@webrtc.org14665ff2015-03-04 12:58:35 +0000226 void SetInterface(NetworkInterface* iface) override;
227 void UpdateAspectRatio(int ratio_w, int ratio_h) override;
pbos@webrtc.orgb5a22b12014-05-13 11:07:01 +0000228
kjellander@webrtc.org14665ff2015-03-04 12:58:35 +0000229 void OnMessage(rtc::Message* msg) override;
pbos@webrtc.orgb5a22b12014-05-13 11:07:01 +0000230
kjellander@webrtc.org14665ff2015-03-04 12:58:35 +0000231 void OnLoadUpdate(Load load) override;
pbos@webrtc.org42684be2014-10-03 11:25:45 +0000232
pbos@webrtc.orgb5a22b12014-05-13 11:07:01 +0000233 // Implemented for VideoMediaChannelTest.
234 bool sending() const { return sending_; }
buildbot@webrtc.org2c0fb052014-08-13 16:47:12 +0000235 uint32 GetDefaultSendChannelSsrc() { return default_send_ssrc_; }
pbos@webrtc.orgb5a22b12014-05-13 11:07:01 +0000236 bool GetRenderer(uint32 ssrc, VideoRenderer** renderer);
237
238 private:
Peter Boströmd6f4c252015-03-26 16:23:04 +0100239 class WebRtcVideoReceiveStream;
pbos@webrtc.orgd1ea06b2014-07-18 09:35:58 +0000240 void ConfigureReceiverRtp(webrtc::VideoReceiveStream::Config* config,
241 const StreamParams& sp) const;
pbos@webrtc.org96a93252014-11-03 14:46:44 +0000242 bool CodecIsExternallySupported(const std::string& name) const;
Peter Boströmd6f4c252015-03-26 16:23:04 +0100243 bool ValidateSendSsrcAvailability(const StreamParams& sp) const
244 EXCLUSIVE_LOCKS_REQUIRED(stream_crit_);
245 bool ValidateReceiveSsrcAvailability(const StreamParams& sp) const
246 EXCLUSIVE_LOCKS_REQUIRED(stream_crit_);
247 void DeleteReceiveStream(WebRtcVideoReceiveStream* stream)
248 EXCLUSIVE_LOCKS_REQUIRED(stream_crit_);
pbos@webrtc.orgd1ea06b2014-07-18 09:35:58 +0000249
pbos@webrtc.orgb5a22b12014-05-13 11:07:01 +0000250 struct VideoCodecSettings {
251 VideoCodecSettings();
Peter Boströmee0b00e2015-04-22 18:41:14 +0200252
253 bool operator==(const VideoCodecSettings& other) const;
254 bool operator!=(const VideoCodecSettings& other) const;
pbos@webrtc.orga2ef4fe2014-11-07 10:54:43 +0000255
pbos@webrtc.org6ae48c62014-06-06 10:49:19 +0000256 VideoCodec codec;
pbos@webrtc.orgb5a22b12014-05-13 11:07:01 +0000257 webrtc::FecConfig fec;
258 int rtx_payload_type;
259 };
260
pbos@webrtc.orgd1ea06b2014-07-18 09:35:58 +0000261 // Wrapper for the sender part, this is where the capturer is connected and
262 // frames are then converted from cricket frames to webrtc frames.
pbos@webrtc.orgb5a22b12014-05-13 11:07:01 +0000263 class WebRtcVideoSendStream : public sigslot::has_slots<> {
264 public:
pbos@webrtc.org5301b0f2014-07-17 08:51:46 +0000265 WebRtcVideoSendStream(
266 webrtc::Call* call,
pbos@webrtc.org7fe1e032014-10-14 04:25:33 +0000267 WebRtcVideoEncoderFactory* external_encoder_factory,
pbos@webrtc.org5301b0f2014-07-17 08:51:46 +0000268 const VideoOptions& options,
Peter Boströmdfd53fe2015-03-27 15:58:11 +0100269 int max_bitrate_bps,
pbos@webrtc.org5301b0f2014-07-17 08:51:46 +0000270 const Settable<VideoCodecSettings>& codec_settings,
271 const StreamParams& sp,
272 const std::vector<webrtc::RtpExtension>& rtp_extensions);
pbos@webrtc.orgb5a22b12014-05-13 11:07:01 +0000273 ~WebRtcVideoSendStream();
Peter Boströmd6f4c252015-03-26 16:23:04 +0100274
pbos@webrtc.org5301b0f2014-07-17 08:51:46 +0000275 void SetOptions(const VideoOptions& options);
276 void SetCodec(const VideoCodecSettings& codec);
pbos@webrtc.orgd1ea06b2014-07-18 09:35:58 +0000277 void SetRtpExtensions(
278 const std::vector<webrtc::RtpExtension>& rtp_extensions);
pbos@webrtc.orgb5a22b12014-05-13 11:07:01 +0000279
280 void InputFrame(VideoCapturer* capturer, const VideoFrame* frame);
281 bool SetCapturer(VideoCapturer* capturer);
282 bool SetVideoFormat(const VideoFormat& format);
pbos@webrtc.orgef8bb8d2014-08-13 21:36:18 +0000283 void MuteStream(bool mute);
pbos@webrtc.orgb5a22b12014-05-13 11:07:01 +0000284 bool DisconnectCapturer();
285
Guo-wei Shieh64c1e8c2015-04-01 15:33:06 -0700286 void SetApplyRotation(bool apply_rotation);
287
pbos@webrtc.orgb5a22b12014-05-13 11:07:01 +0000288 void Start();
289 void Stop();
290
Peter Boströmd6f4c252015-03-26 16:23:04 +0100291 const std::vector<uint32>& GetSsrcs() const;
pbos@webrtc.orge6f84ae2014-07-18 11:11:55 +0000292 VideoSenderInfo GetVideoSenderInfo();
stefan@webrtc.org0bae1fa2014-11-05 14:05:29 +0000293 void FillBandwidthEstimationInfo(BandwidthEstimationInfo* bwe_info);
pbos@webrtc.orge6f84ae2014-07-18 11:11:55 +0000294
Peter Boströmdfd53fe2015-03-27 15:58:11 +0100295 void SetMaxBitrateBps(int max_bitrate_bps);
296
pbos@webrtc.orgb5a22b12014-05-13 11:07:01 +0000297 private:
pbos@webrtc.org6ae48c62014-06-06 10:49:19 +0000298 // Parameters needed to reconstruct the underlying stream.
299 // webrtc::VideoSendStream doesn't support setting a lot of options on the
300 // fly, so when those need to be changed we tear down and reconstruct with
301 // similar parameters depending on which options changed etc.
302 struct VideoSendStreamParameters {
303 VideoSendStreamParameters(
304 const webrtc::VideoSendStream::Config& config,
305 const VideoOptions& options,
Peter Boströmdfd53fe2015-03-27 15:58:11 +0100306 int max_bitrate_bps,
pbos@webrtc.org5301b0f2014-07-17 08:51:46 +0000307 const Settable<VideoCodecSettings>& codec_settings);
pbos@webrtc.org6ae48c62014-06-06 10:49:19 +0000308 webrtc::VideoSendStream::Config config;
309 VideoOptions options;
Peter Boströmdfd53fe2015-03-27 15:58:11 +0100310 int max_bitrate_bps;
pbos@webrtc.org5301b0f2014-07-17 08:51:46 +0000311 Settable<VideoCodecSettings> codec_settings;
pbos@webrtc.org6ae48c62014-06-06 10:49:19 +0000312 // Sent resolutions + bitrates etc. by the underlying VideoSendStream,
313 // typically changes when setting a new resolution or reconfiguring
314 // bitrates.
pbos@webrtc.orgbbe0a852014-09-19 12:30:25 +0000315 webrtc::VideoEncoderConfig encoder_config;
pbos@webrtc.org6ae48c62014-06-06 10:49:19 +0000316 };
317
pbos@webrtc.org7fe1e032014-10-14 04:25:33 +0000318 struct AllocatedEncoder {
319 AllocatedEncoder(webrtc::VideoEncoder* encoder,
320 webrtc::VideoCodecType type,
321 bool external)
322 : encoder(encoder), type(type), external(external) {}
323 webrtc::VideoEncoder* encoder;
324 webrtc::VideoCodecType type;
325 bool external;
326 };
327
pbos@webrtc.orga2ef4fe2014-11-07 10:54:43 +0000328 struct Dimensions {
pbos@webrtc.orgb4987bf2015-02-18 10:13:09 +0000329 // Initial encoder configuration (QCIF, 176x144) frame (to ensure that
330 // hardware encoders can be initialized). This gives us low memory usage
331 // but also makes it so configuration errors are discovered at the time we
332 // apply the settings rather than when we get the first frame (waiting for
333 // the first frame to know that you gave a bad codec parameter could make
334 // debugging hard).
335 // TODO(pbos): Consider setting up encoders lazily.
336 Dimensions() : width(176), height(144), is_screencast(false) {}
pbos@webrtc.orgefc82c22014-10-27 13:58:00 +0000337 int width;
338 int height;
339 bool is_screencast;
340 };
341
pbos@webrtc.orgf1c8b902015-01-14 17:29:27 +0000342 union VideoEncoderSettings {
343 webrtc::VideoCodecVP8 vp8;
344 webrtc::VideoCodecVP9 vp9;
345 };
346
347 static std::vector<webrtc::VideoStream> CreateVideoStreams(
348 const VideoCodec& codec,
349 const VideoOptions& options,
Peter Boströmdfd53fe2015-03-27 15:58:11 +0100350 int max_bitrate_bps,
pbos@webrtc.orgf1c8b902015-01-14 17:29:27 +0000351 size_t num_streams);
352 static std::vector<webrtc::VideoStream> CreateSimulcastVideoStreams(
353 const VideoCodec& codec,
354 const VideoOptions& options,
Peter Boströmdfd53fe2015-03-27 15:58:11 +0100355 int max_bitrate_bps,
pbos@webrtc.orgf1c8b902015-01-14 17:29:27 +0000356 size_t num_streams);
357
358 void* ConfigureVideoEncoderSettings(const VideoCodec& codec,
Erik Språng143cec12015-04-28 10:01:41 +0200359 const VideoOptions& options,
360 bool is_screencast)
pbos@webrtc.orgf1c8b902015-01-14 17:29:27 +0000361 EXCLUSIVE_LOCKS_REQUIRED(lock_);
362
pbos@webrtc.org7fe1e032014-10-14 04:25:33 +0000363 AllocatedEncoder CreateVideoEncoder(const VideoCodec& codec)
364 EXCLUSIVE_LOCKS_REQUIRED(lock_);
pbos@webrtc.orga2ef4fe2014-11-07 10:54:43 +0000365 void DestroyVideoEncoder(AllocatedEncoder* encoder)
366 EXCLUSIVE_LOCKS_REQUIRED(lock_);
pbos@webrtc.org5301b0f2014-07-17 08:51:46 +0000367 void SetCodecAndOptions(const VideoCodecSettings& codec,
pbos@webrtc.orgd60d79a2014-09-24 07:10:57 +0000368 const VideoOptions& options)
369 EXCLUSIVE_LOCKS_REQUIRED(lock_);
370 void RecreateWebRtcStream() EXCLUSIVE_LOCKS_REQUIRED(lock_);
pbos@webrtc.orga2ef4fe2014-11-07 10:54:43 +0000371 webrtc::VideoEncoderConfig CreateVideoEncoderConfig(
372 const Dimensions& dimensions,
373 const VideoCodec& codec) const EXCLUSIVE_LOCKS_REQUIRED(lock_);
pbos@webrtc.orgefc82c22014-10-27 13:58:00 +0000374 void SetDimensions(int width, int height, bool is_screencast)
pbos@webrtc.orgd60d79a2014-09-24 07:10:57 +0000375 EXCLUSIVE_LOCKS_REQUIRED(lock_);
pbos@webrtc.orgb5a22b12014-05-13 11:07:01 +0000376
Peter Boströmd6f4c252015-03-26 16:23:04 +0100377 const std::vector<uint32> ssrcs_;
pbos@webrtc.orgb5a22b12014-05-13 11:07:01 +0000378 webrtc::Call* const call_;
pbos@webrtc.orga2ef4fe2014-11-07 10:54:43 +0000379 WebRtcVideoEncoderFactory* const external_encoder_factory_
380 GUARDED_BY(lock_);
pbos@webrtc.orgb5a22b12014-05-13 11:07:01 +0000381
buildbot@webrtc.orgd4e598d2014-07-29 17:36:52 +0000382 rtc::CriticalSection lock_;
pbos@webrtc.orgb5a22b12014-05-13 11:07:01 +0000383 webrtc::VideoSendStream* stream_ GUARDED_BY(lock_);
pbos@webrtc.org6ae48c62014-06-06 10:49:19 +0000384 VideoSendStreamParameters parameters_ GUARDED_BY(lock_);
pbos@webrtc.orgf1c8b902015-01-14 17:29:27 +0000385 VideoEncoderSettings encoder_settings_ GUARDED_BY(lock_);
pbos@webrtc.org7fe1e032014-10-14 04:25:33 +0000386 AllocatedEncoder allocated_encoder_ GUARDED_BY(lock_);
pbos@webrtc.orga2ef4fe2014-11-07 10:54:43 +0000387 Dimensions last_dimensions_ GUARDED_BY(lock_);
pbos@webrtc.org6ae48c62014-06-06 10:49:19 +0000388
pbos@webrtc.orgb5a22b12014-05-13 11:07:01 +0000389 VideoCapturer* capturer_ GUARDED_BY(lock_);
390 bool sending_ GUARDED_BY(lock_);
391 bool muted_ GUARDED_BY(lock_);
392 VideoFormat format_ GUARDED_BY(lock_);
pbos@webrtc.org9a4410e2015-02-26 10:03:39 +0000393 int old_adapt_changes_ GUARDED_BY(lock_);
pbos@webrtc.orgb5a22b12014-05-13 11:07:01 +0000394 };
395
pbos@webrtc.orgd1ea06b2014-07-18 09:35:58 +0000396 // Wrapper for the receiver part, contains configs etc. that are needed to
397 // reconstruct the underlying VideoReceiveStream. Also serves as a wrapper
398 // between webrtc::VideoRenderer and cricket::VideoRenderer.
399 class WebRtcVideoReceiveStream : public webrtc::VideoRenderer {
400 public:
401 WebRtcVideoReceiveStream(
402 webrtc::Call*,
Peter Boströmd6f4c252015-03-26 16:23:04 +0100403 const std::vector<uint32>& ssrcs,
pbos@webrtc.org776e6f22014-10-29 15:28:39 +0000404 WebRtcVideoDecoderFactory* external_decoder_factory,
pbos@webrtc.orga2a6fe62015-03-06 15:35:19 +0000405 bool default_stream,
pbos@webrtc.orgd1ea06b2014-07-18 09:35:58 +0000406 const webrtc::VideoReceiveStream::Config& config,
407 const std::vector<VideoCodecSettings>& recv_codecs);
408 ~WebRtcVideoReceiveStream();
409
Peter Boströmd6f4c252015-03-26 16:23:04 +0100410 const std::vector<uint32>& GetSsrcs() const;
411
pbos@webrtc.orgd1ea06b2014-07-18 09:35:58 +0000412 void SetRecvCodecs(const std::vector<VideoCodecSettings>& recv_codecs);
413 void SetRtpExtensions(const std::vector<webrtc::RtpExtension>& extensions);
414
kjellander@webrtc.org14665ff2015-03-04 12:58:35 +0000415 void RenderFrame(const webrtc::I420VideoFrame& frame,
416 int time_to_render_ms) override;
417 bool IsTextureSupported() const override;
pbos@webrtc.orga2a6fe62015-03-06 15:35:19 +0000418 bool IsDefaultStream() const;
pbos@webrtc.orgd1ea06b2014-07-18 09:35:58 +0000419
420 void SetRenderer(cricket::VideoRenderer* renderer);
421 cricket::VideoRenderer* GetRenderer();
422
pbos@webrtc.orge6f84ae2014-07-18 11:11:55 +0000423 VideoReceiverInfo GetVideoReceiverInfo();
424
pbos@webrtc.orgd1ea06b2014-07-18 09:35:58 +0000425 private:
pbos@webrtc.org776e6f22014-10-29 15:28:39 +0000426 struct AllocatedDecoder {
pbos@webrtc.org96a93252014-11-03 14:46:44 +0000427 AllocatedDecoder(webrtc::VideoDecoder* decoder,
428 webrtc::VideoCodecType type,
429 bool external)
430 : decoder(decoder), type(type), external(external) {}
pbos@webrtc.org776e6f22014-10-29 15:28:39 +0000431 webrtc::VideoDecoder* decoder;
pbos@webrtc.org96a93252014-11-03 14:46:44 +0000432 webrtc::VideoCodecType type;
pbos@webrtc.org776e6f22014-10-29 15:28:39 +0000433 bool external;
434 };
435
pbos@webrtc.orgd1ea06b2014-07-18 09:35:58 +0000436 void SetSize(int width, int height);
437 void RecreateWebRtcStream();
438
pbos@webrtc.org96a93252014-11-03 14:46:44 +0000439 AllocatedDecoder CreateOrReuseVideoDecoder(
440 std::vector<AllocatedDecoder>* old_decoder,
441 const VideoCodec& codec);
442 void ClearDecoders(std::vector<AllocatedDecoder>* allocated_decoders);
pbos@webrtc.org776e6f22014-10-29 15:28:39 +0000443
pbos@webrtc.orgd1ea06b2014-07-18 09:35:58 +0000444 webrtc::Call* const call_;
Peter Boströmd6f4c252015-03-26 16:23:04 +0100445 const std::vector<uint32> ssrcs_;
pbos@webrtc.orgd1ea06b2014-07-18 09:35:58 +0000446
447 webrtc::VideoReceiveStream* stream_;
pbos@webrtc.orga2a6fe62015-03-06 15:35:19 +0000448 const bool default_stream_;
pbos@webrtc.orgd1ea06b2014-07-18 09:35:58 +0000449 webrtc::VideoReceiveStream::Config config_;
450
pbos@webrtc.org776e6f22014-10-29 15:28:39 +0000451 WebRtcVideoDecoderFactory* const external_decoder_factory_;
452 std::vector<AllocatedDecoder> allocated_decoders_;
453
buildbot@webrtc.orgd4e598d2014-07-29 17:36:52 +0000454 rtc::CriticalSection renderer_lock_;
pbos@webrtc.orgd1ea06b2014-07-18 09:35:58 +0000455 cricket::VideoRenderer* renderer_ GUARDED_BY(renderer_lock_);
pbos@webrtc.orge6f84ae2014-07-18 11:11:55 +0000456 int last_width_ GUARDED_BY(renderer_lock_);
457 int last_height_ GUARDED_BY(renderer_lock_);
magjed@webrtc.orgfc5ad952015-01-27 09:57:01 +0000458 // Expands remote RTP timestamps to int64_t to be able to estimate how long
459 // the stream has been running.
460 rtc::TimestampWrapAroundHandler timestamp_wraparound_handler_
461 GUARDED_BY(renderer_lock_);
462 int64_t first_frame_timestamp_ GUARDED_BY(renderer_lock_);
463 // Start NTP time is estimated as current remote NTP time (estimated from
464 // RTCP) minus the elapsed time, as soon as remote NTP time is available.
465 int64_t estimated_remote_start_ntp_time_ms_ GUARDED_BY(renderer_lock_);
pbos@webrtc.orgd1ea06b2014-07-18 09:35:58 +0000466 };
467
pbos@webrtc.orgb5a22b12014-05-13 11:07:01 +0000468 void Construct(webrtc::Call* call, WebRtcVideoEngine2* engine);
pbos@webrtc.org6f48f1b2014-07-22 16:29:54 +0000469 void SetDefaultOptions();
pbos@webrtc.orgb5a22b12014-05-13 11:07:01 +0000470
kjellander@webrtc.org14665ff2015-03-04 12:58:35 +0000471 bool SendRtp(const uint8_t* data, size_t len) override;
472 bool SendRtcp(const uint8_t* data, size_t len) override;
pbos@webrtc.orgb5a22b12014-05-13 11:07:01 +0000473
474 void StartAllSendStreams();
475 void StopAllSendStreams();
pbos@webrtc.orgd1ea06b2014-07-18 09:35:58 +0000476
pbos@webrtc.orgb5a22b12014-05-13 11:07:01 +0000477 static std::vector<VideoCodecSettings> MapCodecs(
478 const std::vector<VideoCodec>& codecs);
479 std::vector<VideoCodecSettings> FilterSupportedCodecs(
pbos@webrtc.org96a93252014-11-03 14:46:44 +0000480 const std::vector<VideoCodecSettings>& mapped_codecs) const;
pbos@webrtc.orgb5a22b12014-05-13 11:07:01 +0000481
pbos@webrtc.orge6f84ae2014-07-18 11:11:55 +0000482 void FillSenderStats(VideoMediaInfo* info);
483 void FillReceiverStats(VideoMediaInfo* info);
pbos@webrtc.org2b19f062014-12-11 13:26:09 +0000484 void FillBandwidthEstimationStats(const webrtc::Call::Stats& stats,
485 VideoMediaInfo* info);
pbos@webrtc.orge6f84ae2014-07-18 11:11:55 +0000486
pbos@webrtc.orgb5a22b12014-05-13 11:07:01 +0000487 uint32_t rtcp_receiver_report_ssrc_;
488 bool sending_;
buildbot@webrtc.orgd4e598d2014-07-29 17:36:52 +0000489 rtc::scoped_ptr<webrtc::Call> call_;
pbos@webrtc.org42684be2014-10-03 11:25:45 +0000490 WebRtcCallFactory* call_factory_;
491
pbos@webrtc.orgb5a22b12014-05-13 11:07:01 +0000492 uint32_t default_send_ssrc_;
pbos@webrtc.orgafb554f42014-08-12 23:17:13 +0000493
494 DefaultUnsignalledSsrcHandler default_unsignalled_ssrc_handler_;
495 UnsignalledSsrcHandler* const unsignalled_ssrc_handler_;
pbos@webrtc.orgb5a22b12014-05-13 11:07:01 +0000496
Peter Boströme7b221f2015-04-13 15:34:32 +0200497 // Separate list of set capturers used to signal CPU adaptation. These should
498 // not be locked while calling methods that take other locks to prevent
499 // lock-order inversions.
500 rtc::CriticalSection capturer_crit_;
501 bool signal_cpu_adaptation_ GUARDED_BY(capturer_crit_);
502 std::map<uint32, VideoCapturer*> capturers_ GUARDED_BY(capturer_crit_);
503
pbos@webrtc.org575d1262014-10-08 14:48:08 +0000504 rtc::CriticalSection stream_crit_;
pbos@webrtc.orgb5a22b12014-05-13 11:07:01 +0000505 // Using primary-ssrc (first ssrc) as key.
pbos@webrtc.org575d1262014-10-08 14:48:08 +0000506 std::map<uint32, WebRtcVideoSendStream*> send_streams_
507 GUARDED_BY(stream_crit_);
508 std::map<uint32, WebRtcVideoReceiveStream*> receive_streams_
509 GUARDED_BY(stream_crit_);
Peter Boströmd6f4c252015-03-26 16:23:04 +0100510 std::set<uint32> send_ssrcs_ GUARDED_BY(stream_crit_);
511 std::set<uint32> receive_ssrcs_ GUARDED_BY(stream_crit_);
pbos@webrtc.orgb5a22b12014-05-13 11:07:01 +0000512
513 Settable<VideoCodecSettings> send_codec_;
pbos@webrtc.org587ef602014-06-16 17:32:02 +0000514 std::vector<webrtc::RtpExtension> send_rtp_extensions_;
515
pbos@webrtc.org8296ec52015-03-20 14:27:49 +0000516 const int voice_channel_id_;
pbos@webrtc.org7fe1e032014-10-14 04:25:33 +0000517 WebRtcVideoEncoderFactory* const external_encoder_factory_;
518 WebRtcVideoDecoderFactory* const external_decoder_factory_;
pbos@webrtc.orgb5a22b12014-05-13 11:07:01 +0000519 std::vector<VideoCodecSettings> recv_codecs_;
pbos@webrtc.org587ef602014-06-16 17:32:02 +0000520 std::vector<webrtc::RtpExtension> recv_rtp_extensions_;
pbos@webrtc.org00873182014-11-25 14:03:34 +0000521 webrtc::Call::Config::BitrateConfig bitrate_config_;
pbos@webrtc.orgb5a22b12014-05-13 11:07:01 +0000522 VideoOptions options_;
523};
524
525} // namespace cricket
526
527#endif // TALK_MEDIA_WEBRTC_WEBRTCVIDEOENGINE2_H_