blob: eeda8a2e34a03905ac3d0efffab6959e3b2358e5 [file] [log] [blame]
Henrik Kjellander2557b862015-11-18 22:00:21 +01001/*
2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11#ifndef WEBRTC_MODULES_VIDEO_CODING_INCLUDE_VIDEO_CODING_H_
12#define WEBRTC_MODULES_VIDEO_CODING_INCLUDE_VIDEO_CODING_H_
13
14#if defined(WEBRTC_WIN)
15// This is a workaround on Windows due to the fact that some Windows
16// headers define CreateEvent as a macro to either CreateEventW or CreateEventA.
17// This can cause problems since we use that name as well and could
18// declare them as one thing here whereas in another place a windows header
19// may have been included and then implementing CreateEvent() causes compilation
20// errors. So for consistency, we include the main windows header here.
21#include <windows.h>
22#endif
23
24#include "webrtc/modules/include/module.h"
25#include "webrtc/modules/include/module_common_types.h"
26#include "webrtc/modules/video_coding/include/video_coding_defines.h"
27#include "webrtc/system_wrappers/include/event_wrapper.h"
28#include "webrtc/video_frame.h"
29
philipel9d3ab612015-12-21 04:12:39 -080030namespace webrtc {
Henrik Kjellander2557b862015-11-18 22:00:21 +010031
32class Clock;
33class EncodedImageCallback;
Peter Boströmad6fc5a2016-05-12 03:01:31 +020034// TODO(pbos): Remove VCMQMSettingsCallback completely. This might be done by
35// removing the VCM and use VideoSender/VideoReceiver as a public interface
36// directly.
37class VCMQMSettingsCallback;
Henrik Kjellander2557b862015-11-18 22:00:21 +010038class VideoEncoder;
39class VideoDecoder;
40struct CodecSpecificInfo;
41
42class EventFactory {
43 public:
44 virtual ~EventFactory() {}
45
46 virtual EventWrapper* CreateEvent() = 0;
47};
48
49class EventFactoryImpl : public EventFactory {
50 public:
51 virtual ~EventFactoryImpl() {}
52
philipel9d3ab612015-12-21 04:12:39 -080053 virtual EventWrapper* CreateEvent() { return EventWrapper::Create(); }
Henrik Kjellander2557b862015-11-18 22:00:21 +010054};
55
56// Used to indicate which decode with errors mode should be used.
57enum VCMDecodeErrorMode {
philipel9d3ab612015-12-21 04:12:39 -080058 kNoErrors, // Never decode with errors. Video will freeze
59 // if nack is disabled.
60 kSelectiveErrors, // Frames that are determined decodable in
61 // VCMSessionInfo may be decoded with missing
62 // packets. As not all incomplete frames will be
63 // decodable, video will freeze if nack is disabled.
64 kWithErrors // Release frames as needed. Errors may be
65 // introduced as some encoded frames may not be
66 // complete.
Henrik Kjellander2557b862015-11-18 22:00:21 +010067};
68
philipel9d3ab612015-12-21 04:12:39 -080069class VideoCodingModule : public Module {
70 public:
71 enum SenderNackMode { kNackNone, kNackAll, kNackSelective };
Henrik Kjellander2557b862015-11-18 22:00:21 +010072
philipel9d3ab612015-12-21 04:12:39 -080073 enum ReceiverRobustness { kNone, kHardNack, kSoftNack, kReferenceSelection };
Henrik Kjellander2557b862015-11-18 22:00:21 +010074
perkjf5b2e512016-07-05 08:34:04 -070075 static VideoCodingModule* Create(Clock* clock, EventFactory* event_factory);
Henrik Kjellander2557b862015-11-18 22:00:21 +010076
philipel83f831a2016-03-12 03:30:23 -080077 static VideoCodingModule* Create(
78 Clock* clock,
philipel83f831a2016-03-12 03:30:23 -080079 VCMQMSettingsCallback* qm_settings_callback,
80 NackSender* nack_sender,
sprang3911c262016-04-15 01:24:14 -070081 KeyFrameRequestSender* keyframe_request_sender,
82 EncodedImageCallback* pre_decode_image_callback);
philipel83f831a2016-03-12 03:30:23 -080083
philipel83f831a2016-03-12 03:30:23 -080084 static VideoCodingModule* Create(
85 Clock* clock,
86 EventFactory* event_factory,
87 NackSender* nack_sender,
88 KeyFrameRequestSender* keyframe_request_sender);
89
philipel9d3ab612015-12-21 04:12:39 -080090 // Get supported codec settings using codec type
91 //
92 // Input:
93 // - codecType : The codec type to get settings for
94 // - codec : Memory where the codec settings will be stored
95 //
96 // Return value : VCM_OK, on success
97 // VCM_PARAMETER_ERROR if codec not supported
Peter Boström7776e782016-01-07 15:42:47 +010098 static void Codec(VideoCodecType codecType, VideoCodec* codec);
Henrik Kjellander2557b862015-11-18 22:00:21 +010099
philipel9d3ab612015-12-21 04:12:39 -0800100 /*
101 * Sender
102 */
Henrik Kjellander2557b862015-11-18 22:00:21 +0100103
philipel9d3ab612015-12-21 04:12:39 -0800104 // Registers a codec to be used for encoding. Calling this
105 // API multiple times overwrites any previously registered codecs.
106 //
107 // NOTE: Must be called on the thread that constructed the VCM instance.
108 //
109 // Input:
110 // - sendCodec : Settings for the codec to be registered.
111 // - numberOfCores : The number of cores the codec is allowed
112 // to use.
113 // - maxPayloadSize : The maximum size each payload is allowed
114 // to have. Usually MTU - overhead.
115 //
116 // Return value : VCM_OK, on success.
117 // < 0, on error.
118 virtual int32_t RegisterSendCodec(const VideoCodec* sendCodec,
119 uint32_t numberOfCores,
120 uint32_t maxPayloadSize) = 0;
Henrik Kjellander2557b862015-11-18 22:00:21 +0100121
philipel9d3ab612015-12-21 04:12:39 -0800122 // Register an external encoder object. This can not be used together with
123 // external decoder callbacks.
124 //
125 // Input:
126 // - externalEncoder : Encoder object to be used for encoding frames
127 // inserted
128 // with the AddVideoFrame API.
129 // - payloadType : The payload type bound which this encoder is bound
130 // to.
131 //
132 // Return value : VCM_OK, on success.
133 // < 0, on error.
134 // TODO(pbos): Remove return type when unused elsewhere.
135 virtual int32_t RegisterExternalEncoder(VideoEncoder* externalEncoder,
136 uint8_t payloadType,
137 bool internalSource = false) = 0;
Henrik Kjellander2557b862015-11-18 22:00:21 +0100138
philipel9d3ab612015-12-21 04:12:39 -0800139 // API to get currently configured encoder target bitrate in bits/s.
140 //
141 // Return value : 0, on success.
142 // < 0, on error.
143 virtual int Bitrate(unsigned int* bitrate) const = 0;
Henrik Kjellander2557b862015-11-18 22:00:21 +0100144
philipel9d3ab612015-12-21 04:12:39 -0800145 // API to get currently configured encoder target frame rate.
146 //
147 // Return value : 0, on success.
148 // < 0, on error.
149 virtual int FrameRate(unsigned int* framerate) const = 0;
Henrik Kjellander2557b862015-11-18 22:00:21 +0100150
philipel9d3ab612015-12-21 04:12:39 -0800151 // Sets the parameters describing the send channel. These parameters are
152 // inputs to the
153 // Media Optimization inside the VCM and also specifies the target bit rate
154 // for the
155 // encoder. Bit rate used by NACK should already be compensated for by the
156 // user.
157 //
158 // Input:
159 // - target_bitrate : The target bitrate for VCM in bits/s.
160 // - lossRate : Fractions of lost packets the past second.
161 // (loss rate in percent = 100 * packetLoss /
162 // 255)
163 // - rtt : Current round-trip time in ms.
164 //
165 // Return value : VCM_OK, on success.
166 // < 0, on error.
167 virtual int32_t SetChannelParameters(uint32_t target_bitrate,
168 uint8_t lossRate,
169 int64_t rtt) = 0;
Henrik Kjellander2557b862015-11-18 22:00:21 +0100170
philipel9d3ab612015-12-21 04:12:39 -0800171 // Sets the parameters describing the receive channel. These parameters are
172 // inputs to the
173 // Media Optimization inside the VCM.
174 //
175 // Input:
176 // - rtt : Current round-trip time in ms.
177 // with the most amount available bandwidth in
178 // a conference
179 // scenario
180 //
181 // Return value : VCM_OK, on success.
182 // < 0, on error.
183 virtual int32_t SetReceiveChannelParameters(int64_t rtt) = 0;
Henrik Kjellander2557b862015-11-18 22:00:21 +0100184
Per69b332d2016-06-02 15:45:42 +0200185 // Deprecated: This method currently does not have any effect.
philipel9d3ab612015-12-21 04:12:39 -0800186 // Register a video protection callback which will be called to deliver
187 // the requested FEC rate and NACK status (on/off).
Per69b332d2016-06-02 15:45:42 +0200188 // TODO(perkj): Remove once no projects use it.
philipel9d3ab612015-12-21 04:12:39 -0800189 virtual int32_t RegisterProtectionCallback(
190 VCMProtectionCallback* protection) = 0;
Henrik Kjellander2557b862015-11-18 22:00:21 +0100191
philipel9d3ab612015-12-21 04:12:39 -0800192 // Enable or disable a video protection method.
193 //
194 // Input:
195 // - videoProtection : The method to enable or disable.
196 // - enable : True if the method should be enabled, false if
197 // it should be disabled.
198 //
199 // Return value : VCM_OK, on success.
200 // < 0, on error.
201 virtual int32_t SetVideoProtection(VCMVideoProtection videoProtection,
202 bool enable) = 0;
Henrik Kjellander2557b862015-11-18 22:00:21 +0100203
philipel9d3ab612015-12-21 04:12:39 -0800204 // Add one raw video frame to the encoder. This function does all the
205 // necessary
206 // processing, then decides what frame type to encode, or if the frame should
207 // be
208 // dropped. If the frame should be encoded it passes the frame to the encoder
209 // before it returns.
210 //
211 // Input:
212 // - videoFrame : Video frame to encode.
213 // - codecSpecificInfo : Extra codec information, e.g., pre-parsed
214 // in-band signaling.
215 //
216 // Return value : VCM_OK, on success.
217 // < 0, on error.
218 virtual int32_t AddVideoFrame(
219 const VideoFrame& videoFrame,
philipel9d3ab612015-12-21 04:12:39 -0800220 const CodecSpecificInfo* codecSpecificInfo = NULL) = 0;
Henrik Kjellander2557b862015-11-18 22:00:21 +0100221
philipel9d3ab612015-12-21 04:12:39 -0800222 // Next frame encoded should be an intra frame (keyframe).
223 //
224 // Return value : VCM_OK, on success.
225 // < 0, on error.
perkj600246e2016-05-04 11:26:51 -0700226 virtual int32_t IntraFrameRequest(size_t stream_index) = 0;
Henrik Kjellander2557b862015-11-18 22:00:21 +0100227
philipel9d3ab612015-12-21 04:12:39 -0800228 // Frame Dropper enable. Can be used to disable the frame dropping when the
229 // encoder
230 // over-uses its bit rate. This API is designed to be used when the encoded
231 // frames
232 // are supposed to be stored to an AVI file, or when the I420 codec is used
233 // and the
234 // target bit rate shouldn't affect the frame rate.
235 //
236 // Input:
237 // - enable : True to enable the setting, false to disable it.
238 //
239 // Return value : VCM_OK, on success.
240 // < 0, on error.
241 virtual int32_t EnableFrameDropper(bool enable) = 0;
Henrik Kjellander2557b862015-11-18 22:00:21 +0100242
philipel9d3ab612015-12-21 04:12:39 -0800243 /*
244 * Receiver
245 */
Henrik Kjellander2557b862015-11-18 22:00:21 +0100246
philipel9d3ab612015-12-21 04:12:39 -0800247 // Register possible receive codecs, can be called multiple times for
248 // different codecs.
249 // The module will automatically switch between registered codecs depending on
250 // the
251 // payload type of incoming frames. The actual decoder will be created when
252 // needed.
253 //
254 // Input:
255 // - receiveCodec : Settings for the codec to be registered.
256 // - numberOfCores : Number of CPU cores that the decoder is allowed
257 // to use.
258 // - requireKeyFrame : Set this to true if you don't want any delta
259 // frames
260 // to be decoded until the first key frame has been
261 // decoded.
262 //
263 // Return value : VCM_OK, on success.
264 // < 0, on error.
265 virtual int32_t RegisterReceiveCodec(const VideoCodec* receiveCodec,
266 int32_t numberOfCores,
267 bool requireKeyFrame = false) = 0;
Henrik Kjellander2557b862015-11-18 22:00:21 +0100268
philipel9d3ab612015-12-21 04:12:39 -0800269 // Register an externally defined decoder/renderer object. Can be a decoder
270 // only or a
271 // decoder coupled with a renderer. Note that RegisterReceiveCodec must be
272 // called to
273 // be used for decoding incoming streams.
274 //
275 // Input:
276 // - externalDecoder : The external decoder/renderer object.
277 // - payloadType : The payload type which this decoder should
278 // be
279 // registered to.
280 //
281 virtual void RegisterExternalDecoder(VideoDecoder* externalDecoder,
282 uint8_t payloadType) = 0;
Henrik Kjellander2557b862015-11-18 22:00:21 +0100283
philipel9d3ab612015-12-21 04:12:39 -0800284 // Register a receive callback. Will be called whenever there is a new frame
285 // ready
286 // for rendering.
287 //
288 // Input:
289 // - receiveCallback : The callback object to be used by the
290 // module when a
291 // frame is ready for rendering.
292 // De-register with a NULL pointer.
293 //
294 // Return value : VCM_OK, on success.
295 // < 0, on error.
296 virtual int32_t RegisterReceiveCallback(
297 VCMReceiveCallback* receiveCallback) = 0;
Henrik Kjellander2557b862015-11-18 22:00:21 +0100298
philipel9d3ab612015-12-21 04:12:39 -0800299 // Register a receive statistics callback which will be called to deliver
300 // information
301 // about the video stream received by the receiving side of the VCM, for
302 // instance the
303 // average frame rate and bit rate.
304 //
305 // Input:
306 // - receiveStats : The callback object to register.
307 //
308 // Return value : VCM_OK, on success.
309 // < 0, on error.
310 virtual int32_t RegisterReceiveStatisticsCallback(
311 VCMReceiveStatisticsCallback* receiveStats) = 0;
Henrik Kjellander2557b862015-11-18 22:00:21 +0100312
philipel9d3ab612015-12-21 04:12:39 -0800313 // Register a decoder timing callback which will be called to deliver
314 // information about the timing of the decoder in the receiving side of the
315 // VCM, for instance the current and maximum frame decode latency.
316 //
317 // Input:
318 // - decoderTiming : The callback object to register.
319 //
320 // Return value : VCM_OK, on success.
321 // < 0, on error.
322 virtual int32_t RegisterDecoderTimingCallback(
323 VCMDecoderTimingCallback* decoderTiming) = 0;
Henrik Kjellander2557b862015-11-18 22:00:21 +0100324
philipel9d3ab612015-12-21 04:12:39 -0800325 // Register a frame type request callback. This callback will be called when
326 // the
327 // module needs to request specific frame types from the send side.
328 //
329 // Input:
330 // - frameTypeCallback : The callback object to be used by the
331 // module when
332 // requesting a specific type of frame from
333 // the send side.
334 // De-register with a NULL pointer.
335 //
336 // Return value : VCM_OK, on success.
337 // < 0, on error.
338 virtual int32_t RegisterFrameTypeCallback(
339 VCMFrameTypeCallback* frameTypeCallback) = 0;
Henrik Kjellander2557b862015-11-18 22:00:21 +0100340
philipel9d3ab612015-12-21 04:12:39 -0800341 // Registers a callback which is called whenever the receive side of the VCM
342 // encounters holes in the packet sequence and needs packets to be
343 // retransmitted.
344 //
345 // Input:
346 // - callback : The callback to be registered in the VCM.
347 //
348 // Return value : VCM_OK, on success.
349 // <0, on error.
350 virtual int32_t RegisterPacketRequestCallback(
351 VCMPacketRequestCallback* callback) = 0;
Henrik Kjellander2557b862015-11-18 22:00:21 +0100352
philipel9d3ab612015-12-21 04:12:39 -0800353 // Waits for the next frame in the jitter buffer to become complete
354 // (waits no longer than maxWaitTimeMs), then passes it to the decoder for
355 // decoding.
356 // Should be called as often as possible to get the most out of the decoder.
357 //
358 // Return value : VCM_OK, on success.
359 // < 0, on error.
360 virtual int32_t Decode(uint16_t maxWaitTimeMs = 200) = 0;
Henrik Kjellander2557b862015-11-18 22:00:21 +0100361
philipel9d3ab612015-12-21 04:12:39 -0800362 // API to get the codec which is currently used for decoding by the module.
363 //
364 // Input:
365 // - currentReceiveCodec : Settings for the codec to be registered.
366 //
367 // Return value : VCM_OK, on success.
368 // < 0, on error.
369 virtual int32_t ReceiveCodec(VideoCodec* currentReceiveCodec) const = 0;
Henrik Kjellander2557b862015-11-18 22:00:21 +0100370
philipel9d3ab612015-12-21 04:12:39 -0800371 // API to get the codec type currently used for decoding by the module.
372 //
373 // Return value : codecy type, on success.
374 // kVideoCodecUnknown, on error or if no receive codec is
375 // registered
376 virtual VideoCodecType ReceiveCodec() const = 0;
Henrik Kjellander2557b862015-11-18 22:00:21 +0100377
philipel9d3ab612015-12-21 04:12:39 -0800378 // Insert a parsed packet into the receiver side of the module. Will be placed
379 // in the
380 // jitter buffer waiting for the frame to become complete. Returns as soon as
381 // the packet
382 // has been placed in the jitter buffer.
383 //
384 // Input:
385 // - incomingPayload : Payload of the packet.
386 // - payloadLength : Length of the payload.
387 // - rtpInfo : The parsed header.
388 //
389 // Return value : VCM_OK, on success.
390 // < 0, on error.
391 virtual int32_t IncomingPacket(const uint8_t* incomingPayload,
392 size_t payloadLength,
393 const WebRtcRTPHeader& rtpInfo) = 0;
Henrik Kjellander2557b862015-11-18 22:00:21 +0100394
philipel9d3ab612015-12-21 04:12:39 -0800395 // Minimum playout delay (Used for lip-sync). This is the minimum delay
396 // required
397 // to sync with audio. Not included in VideoCodingModule::Delay()
398 // Defaults to 0 ms.
399 //
400 // Input:
401 // - minPlayoutDelayMs : Additional delay in ms.
402 //
403 // Return value : VCM_OK, on success.
404 // < 0, on error.
405 virtual int32_t SetMinimumPlayoutDelay(uint32_t minPlayoutDelayMs) = 0;
Henrik Kjellander2557b862015-11-18 22:00:21 +0100406
philipel9d3ab612015-12-21 04:12:39 -0800407 // Set the time required by the renderer to render a frame.
408 //
409 // Input:
410 // - timeMS : The time in ms required by the renderer to render a
411 // frame.
412 //
413 // Return value : VCM_OK, on success.
414 // < 0, on error.
415 virtual int32_t SetRenderDelay(uint32_t timeMS) = 0;
Henrik Kjellander2557b862015-11-18 22:00:21 +0100416
philipel9d3ab612015-12-21 04:12:39 -0800417 // The total delay desired by the VCM. Can be less than the minimum
418 // delay set with SetMinimumPlayoutDelay.
419 //
420 // Return value : Total delay in ms, on success.
421 // < 0, on error.
422 virtual int32_t Delay() const = 0;
Henrik Kjellander2557b862015-11-18 22:00:21 +0100423
philipel9d3ab612015-12-21 04:12:39 -0800424 // Returns the number of packets discarded by the jitter buffer due to being
425 // too late. This can include duplicated packets which arrived after the
426 // frame was sent to the decoder. Therefore packets which were prematurely
427 // NACKed will be counted.
428 virtual uint32_t DiscardedPackets() const = 0;
Henrik Kjellander2557b862015-11-18 22:00:21 +0100429
philipel9d3ab612015-12-21 04:12:39 -0800430 // Robustness APIs
Henrik Kjellander2557b862015-11-18 22:00:21 +0100431
philipel9d3ab612015-12-21 04:12:39 -0800432 // Set the receiver robustness mode. The mode decides how the receiver
433 // responds to losses in the stream. The type of counter-measure (soft or
434 // hard NACK, dual decoder, RPS, etc.) is selected through the
435 // robustnessMode parameter. The errorMode parameter decides if it is
436 // allowed to display frames corrupted by losses. Note that not all
437 // combinations of the two parameters are feasible. An error will be
438 // returned for invalid combinations.
439 // Input:
440 // - robustnessMode : selected robustness mode.
441 // - errorMode : selected error mode.
442 //
443 // Return value : VCM_OK, on success;
444 // < 0, on error.
445 virtual int SetReceiverRobustnessMode(ReceiverRobustness robustnessMode,
446 VCMDecodeErrorMode errorMode) = 0;
Henrik Kjellander2557b862015-11-18 22:00:21 +0100447
philipel9d3ab612015-12-21 04:12:39 -0800448 // Set the decode error mode. The mode decides which errors (if any) are
449 // allowed in decodable frames. Note that setting decode_error_mode to
450 // anything other than kWithErrors without enabling nack will cause
451 // long-term freezes (resulting from frequent key frame requests) if
452 // packet loss occurs.
453 virtual void SetDecodeErrorMode(VCMDecodeErrorMode decode_error_mode) = 0;
Henrik Kjellander2557b862015-11-18 22:00:21 +0100454
philipel9d3ab612015-12-21 04:12:39 -0800455 // Sets the maximum number of sequence numbers that we are allowed to NACK
456 // and the oldest sequence number that we will consider to NACK. If a
457 // sequence number older than |max_packet_age_to_nack| is missing
458 // a key frame will be requested. A key frame will also be requested if the
459 // time of incomplete or non-continuous frames in the jitter buffer is above
460 // |max_incomplete_time_ms|.
461 virtual void SetNackSettings(size_t max_nack_list_size,
462 int max_packet_age_to_nack,
463 int max_incomplete_time_ms) = 0;
Henrik Kjellander2557b862015-11-18 22:00:21 +0100464
philipel9d3ab612015-12-21 04:12:39 -0800465 // Setting a desired delay to the VCM receiver. Video rendering will be
466 // delayed by at least desired_delay_ms.
467 virtual int SetMinReceiverDelay(int desired_delay_ms) = 0;
Henrik Kjellander2557b862015-11-18 22:00:21 +0100468
philipel9d3ab612015-12-21 04:12:39 -0800469 virtual void RegisterPostEncodeImageCallback(
470 EncodedImageCallback* post_encode_callback) = 0;
471 // Releases pending decode calls, permitting faster thread shutdown.
472 virtual void TriggerDecoderShutdown() = 0;
Henrik Kjellander2557b862015-11-18 22:00:21 +0100473};
474
475} // namespace webrtc
476
philipel9d3ab612015-12-21 04:12:39 -0800477#endif // WEBRTC_MODULES_VIDEO_CODING_INCLUDE_VIDEO_CODING_H_