blob: 2addc59a17c3e34cb6f47bed5d063315155be62d [file] [log] [blame]
philipel02447bc2016-05-13 06:01:03 -07001/*
2 * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020011#ifndef MODULES_VIDEO_CODING_RTP_FRAME_REFERENCE_FINDER_H_
12#define MODULES_VIDEO_CODING_RTP_FRAME_REFERENCE_FINDER_H_
philipel02447bc2016-05-13 06:01:03 -070013
14#include <array>
15#include <map>
kwibergfd8be342016-05-14 19:44:11 -070016#include <memory>
philipel463d3012016-09-09 03:32:44 -070017#include <deque>
philipel02447bc2016-05-13 06:01:03 -070018#include <set>
19#include <utility>
20
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020021#include "modules/include/module_common_types.h"
22#include "modules/video_coding/sequence_number_util.h"
23#include "rtc_base/criticalsection.h"
24#include "rtc_base/thread_annotations.h"
philipel02447bc2016-05-13 06:01:03 -070025
26namespace webrtc {
27namespace video_coding {
28
philipel17deeb42016-08-11 15:09:26 +020029class FrameObject;
philipel02447bc2016-05-13 06:01:03 -070030class RtpFrameObject;
philipel17deeb42016-08-11 15:09:26 +020031
32// A complete frame is a frame which has received all its packets and all its
33// references are known.
34class OnCompleteFrameCallback {
35 public:
36 virtual ~OnCompleteFrameCallback() {}
37 virtual void OnCompleteFrame(std::unique_ptr<FrameObject> frame) = 0;
38};
philipel02447bc2016-05-13 06:01:03 -070039
40class RtpFrameReferenceFinder {
41 public:
42 explicit RtpFrameReferenceFinder(OnCompleteFrameCallback* frame_callback);
philipel463d3012016-09-09 03:32:44 -070043
44 // Manage this frame until:
45 // - We have all information needed to determine its references, after
46 // which |frame_callback_| is called with the completed frame, or
philipelafcf7f52017-04-26 08:17:35 -070047 // - We have too many stashed frames (determined by |kMaxStashedFrames|)
philipel463d3012016-09-09 03:32:44 -070048 // so we drop this frame, or
49 // - It gets cleared by ClearTo, which also means we drop it.
philipel02447bc2016-05-13 06:01:03 -070050 void ManageFrame(std::unique_ptr<RtpFrameObject> frame);
philipel463d3012016-09-09 03:32:44 -070051
52 // Notifies that padding has been received, which the reference finder
53 // might need to calculate the references of a frame.
philipel9b2ce6b2016-07-05 05:04:46 -070054 void PaddingReceived(uint16_t seq_num);
philipel02447bc2016-05-13 06:01:03 -070055
philipel463d3012016-09-09 03:32:44 -070056 // Clear all stashed frames that include packets older than |seq_num|.
57 void ClearTo(uint16_t seq_num);
58
philipel02447bc2016-05-13 06:01:03 -070059 private:
philipelfd5a20f2016-11-15 00:57:57 -080060 static const uint16_t kPicIdLength = 1 << 15;
philipel02447bc2016-05-13 06:01:03 -070061 static const uint8_t kMaxTemporalLayers = 5;
philipelfd5a20f2016-11-15 00:57:57 -080062 static const int kMaxLayerInfo = 50;
63 static const int kMaxStashedFrames = 50;
64 static const int kMaxNotYetReceivedFrames = 100;
65 static const int kMaxGofSaved = 50;
philipel9b2ce6b2016-07-05 05:04:46 -070066 static const int kMaxPaddingAge = 100;
philipel02447bc2016-05-13 06:01:03 -070067
philipelafcf7f52017-04-26 08:17:35 -070068 enum FrameDecision { kStash, kHandOff, kDrop };
philipelc9b27d52016-07-15 06:50:27 -070069
70 struct GofInfo {
71 GofInfo(GofInfoVP9* gof, uint16_t last_picture_id)
72 : gof(gof), last_picture_id(last_picture_id) {}
73 GofInfoVP9* gof;
74 uint16_t last_picture_id;
75 };
76
philipel02447bc2016-05-13 06:01:03 -070077 rtc::CriticalSection crit_;
78
philipel9b2ce6b2016-07-05 05:04:46 -070079 // Find the relevant group of pictures and update its "last-picture-id-with
80 // padding" sequence number.
81 void UpdateLastPictureIdWithPadding(uint16_t seq_num)
danilchap56359be2017-09-07 07:53:45 -070082 RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_);
philipel9b2ce6b2016-07-05 05:04:46 -070083
philipelafcf7f52017-04-26 08:17:35 -070084 // Retry stashed frames until no more complete frames are found.
danilchap56359be2017-09-07 07:53:45 -070085 void RetryStashedFrames() RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_);
philipel02447bc2016-05-13 06:01:03 -070086
philipelafcf7f52017-04-26 08:17:35 -070087 FrameDecision ManageFrameInternal(RtpFrameObject* frame)
danilchap56359be2017-09-07 07:53:45 -070088 RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_);
philipelafcf7f52017-04-26 08:17:35 -070089
philipel647998c2016-06-03 09:40:16 -070090 // Find references for generic frames. If |picture_id| is unspecified
91 // then packet sequence numbers will be used to determine the references
92 // of the frames.
philipelafcf7f52017-04-26 08:17:35 -070093 FrameDecision ManageFrameGeneric(RtpFrameObject* frame, int picture_id)
danilchap56359be2017-09-07 07:53:45 -070094 RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_);
philipel02447bc2016-05-13 06:01:03 -070095
philipelafcf7f52017-04-26 08:17:35 -070096 // Find references for Vp8 frames
97 FrameDecision ManageFrameVp8(RtpFrameObject* frame)
danilchap56359be2017-09-07 07:53:45 -070098 RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_);
philipelafcf7f52017-04-26 08:17:35 -070099
100 // Updates necessary layer info state used to determine frame references for
101 // Vp8.
philipeld4fac692017-09-04 07:03:46 -0700102 void UpdateLayerInfoVp8(RtpFrameObject* frame)
danilchap56359be2017-09-07 07:53:45 -0700103 RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_);
philipel02447bc2016-05-13 06:01:03 -0700104
105 // Find references for Vp9 frames
philipelafcf7f52017-04-26 08:17:35 -0700106 FrameDecision ManageFrameVp9(RtpFrameObject* frame)
danilchap56359be2017-09-07 07:53:45 -0700107 RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_);
philipel02447bc2016-05-13 06:01:03 -0700108
109 // Check if we are missing a frame necessary to determine the references
110 // for this frame.
philipelc9b27d52016-07-15 06:50:27 -0700111 bool MissingRequiredFrameVp9(uint16_t picture_id, const GofInfo& info)
danilchap56359be2017-09-07 07:53:45 -0700112 RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_);
philipel02447bc2016-05-13 06:01:03 -0700113
114 // Updates which frames that have been received. If there is a gap,
115 // missing frames will be added to |missing_frames_for_layer_| or
116 // if this is an already missing frame then it will be removed.
philipelc9b27d52016-07-15 06:50:27 -0700117 void FrameReceivedVp9(uint16_t picture_id, GofInfo* info)
danilchap56359be2017-09-07 07:53:45 -0700118 RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_);
philipel02447bc2016-05-13 06:01:03 -0700119
120 // Check if there is a frame with the up-switch flag set in the interval
121 // (|pid_ref|, |picture_id|) with temporal layer smaller than |temporal_idx|.
122 bool UpSwitchInIntervalVp9(uint16_t picture_id,
123 uint8_t temporal_idx,
danilchap56359be2017-09-07 07:53:45 -0700124 uint16_t pid_ref)
125 RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_);
philipel02447bc2016-05-13 06:01:03 -0700126
philipelafcf7f52017-04-26 08:17:35 -0700127 // Unwrap |frame|s picture id and its references to 16 bits.
danilchap56359be2017-09-07 07:53:45 -0700128 void UnwrapPictureIds(RtpFrameObject* frame)
129 RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_);
philipel02447bc2016-05-13 06:01:03 -0700130
philipelfd5a20f2016-11-15 00:57:57 -0800131 // Returns true if the frame is old and should be dropped.
132 // TODO(philipel): Remove when VP9 PID/TL0 does not jump mid-stream (should be
133 // around M59).
134 bool Vp9PidTl0Fix(const RtpFrameObject& frame,
135 int16_t* picture_id,
danilchap56359be2017-09-07 07:53:45 -0700136 int16_t* tl0_pic_idx) RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_);
philipelfd5a20f2016-11-15 00:57:57 -0800137
138 // TODO(philipel): Remove when VP9 PID/TL0 does not jump mid-stream (should be
139 // around M59).
140 bool DetectVp9PicIdJump(int fixed_pid,
141 int fixed_tl0,
142 uint32_t timestamp) const
danilchap56359be2017-09-07 07:53:45 -0700143 RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_);
philipelfd5a20f2016-11-15 00:57:57 -0800144
145 // TODO(philipel): Remove when VP9 PID/TL0 does not jump mid-stream (should be
146 // around M59).
147 bool DetectVp9Tl0PicIdxJump(int fixed_tl0, uint32_t timestamp) const
danilchap56359be2017-09-07 07:53:45 -0700148 RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_);
philipel9b2ce6b2016-07-05 05:04:46 -0700149
150 // For every group of pictures, hold two sequence numbers. The first being
151 // the sequence number of the last packet of the last completed frame, and
152 // the second being the sequence number of the last packet of the last
153 // completed frame advanced by any potential continuous packets of padding.
154 std::map<uint16_t,
155 std::pair<uint16_t, uint16_t>,
156 DescendingSeqNumComp<uint16_t>>
danilchap56359be2017-09-07 07:53:45 -0700157 last_seq_num_gop_ RTC_GUARDED_BY(crit_);
philipel02447bc2016-05-13 06:01:03 -0700158
159 // Save the last picture id in order to detect when there is a gap in frames
160 // that have not yet been fully received.
danilchap56359be2017-09-07 07:53:45 -0700161 int last_picture_id_ RTC_GUARDED_BY(crit_);
philipel02447bc2016-05-13 06:01:03 -0700162
philipel9b2ce6b2016-07-05 05:04:46 -0700163 // Padding packets that have been received but that are not yet continuous
164 // with any group of pictures.
165 std::set<uint16_t, DescendingSeqNumComp<uint16_t>> stashed_padding_
danilchap56359be2017-09-07 07:53:45 -0700166 RTC_GUARDED_BY(crit_);
philipel9b2ce6b2016-07-05 05:04:46 -0700167
philipel02447bc2016-05-13 06:01:03 -0700168 // The last unwrapped picture id. Used to unwrap the picture id from a length
169 // of |kPicIdLength| to 16 bits.
danilchap56359be2017-09-07 07:53:45 -0700170 int last_unwrap_ RTC_GUARDED_BY(crit_);
philipel02447bc2016-05-13 06:01:03 -0700171
172 // Frames earlier than the last received frame that have not yet been
173 // fully received.
174 std::set<uint16_t, DescendingSeqNumComp<uint16_t, kPicIdLength>>
danilchap56359be2017-09-07 07:53:45 -0700175 not_yet_received_frames_ RTC_GUARDED_BY(crit_);
philipel02447bc2016-05-13 06:01:03 -0700176
177 // Frames that have been fully received but didn't have all the information
178 // needed to determine their references.
danilchap56359be2017-09-07 07:53:45 -0700179 std::deque<std::unique_ptr<RtpFrameObject>> stashed_frames_
180 RTC_GUARDED_BY(crit_);
philipel02447bc2016-05-13 06:01:03 -0700181
182 // Holds the information about the last completed frame for a given temporal
183 // layer given a Tl0 picture index.
184 std::map<uint8_t,
185 std::array<int16_t, kMaxTemporalLayers>,
186 DescendingSeqNumComp<uint8_t>>
danilchap56359be2017-09-07 07:53:45 -0700187 layer_info_ RTC_GUARDED_BY(crit_);
philipel02447bc2016-05-13 06:01:03 -0700188
189 // Where the current scalability structure is in the
190 // |scalability_structures_| array.
191 uint8_t current_ss_idx_;
192
193 // Holds received scalability structures.
194 std::array<GofInfoVP9, kMaxGofSaved> scalability_structures_
danilchap56359be2017-09-07 07:53:45 -0700195 RTC_GUARDED_BY(crit_);
philipel02447bc2016-05-13 06:01:03 -0700196
philipelc9b27d52016-07-15 06:50:27 -0700197 // Holds the the Gof information for a given TL0 picture index.
198 std::map<uint8_t, GofInfo, DescendingSeqNumComp<uint8_t>> gof_info_
danilchap56359be2017-09-07 07:53:45 -0700199 RTC_GUARDED_BY(crit_);
philipel02447bc2016-05-13 06:01:03 -0700200
201 // Keep track of which picture id and which temporal layer that had the
202 // up switch flag set.
philipelc9b27d52016-07-15 06:50:27 -0700203 std::map<uint16_t, uint8_t, DescendingSeqNumComp<uint16_t, kPicIdLength>>
danilchap56359be2017-09-07 07:53:45 -0700204 up_switch_ RTC_GUARDED_BY(crit_);
philipel02447bc2016-05-13 06:01:03 -0700205
206 // For every temporal layer, keep a set of which frames that are missing.
207 std::array<std::set<uint16_t, DescendingSeqNumComp<uint16_t, kPicIdLength>>,
208 kMaxTemporalLayers>
danilchap56359be2017-09-07 07:53:45 -0700209 missing_frames_for_layer_ RTC_GUARDED_BY(crit_);
philipel02447bc2016-05-13 06:01:03 -0700210
philipel463d3012016-09-09 03:32:44 -0700211 // How far frames have been cleared by sequence number. A frame will be
212 // cleared if it contains a packet with a sequence number older than
213 // |cleared_to_seq_num_|.
danilchap56359be2017-09-07 07:53:45 -0700214 int cleared_to_seq_num_ RTC_GUARDED_BY(crit_);
philipel463d3012016-09-09 03:32:44 -0700215
philipel02447bc2016-05-13 06:01:03 -0700216 OnCompleteFrameCallback* frame_callback_;
philipeld4fac692017-09-04 07:03:46 -0700217
218 // Unwrapper used to unwrap generic RTP streams. In a generic stream we derive
219 // a picture id from the packet sequence number.
danilchap56359be2017-09-07 07:53:45 -0700220 SeqNumUnwrapper<uint16_t> generic_unwrapper_ RTC_GUARDED_BY(crit_);
philipeld4fac692017-09-04 07:03:46 -0700221
222 // Unwrapper used to unwrap VP8/VP9 streams which have their picture id
223 // specified.
danilchap56359be2017-09-07 07:53:45 -0700224 SeqNumUnwrapper<uint16_t, kPicIdLength> unwrapper_ RTC_GUARDED_BY(crit_);
philipel02447bc2016-05-13 06:01:03 -0700225};
226
227} // namespace video_coding
228} // namespace webrtc
229
Mirko Bonadei92ea95e2017-09-15 06:47:31 +0200230#endif // MODULES_VIDEO_CODING_RTP_FRAME_REFERENCE_FINDER_H_