philipel | 02447bc | 2016-05-13 06:01:03 -0700 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. |
| 3 | * |
| 4 | * Use of this source code is governed by a BSD-style license |
| 5 | * that can be found in the LICENSE file in the root of the source |
| 6 | * tree. An additional intellectual property rights grant can be found |
| 7 | * in the file PATENTS. All contributing project authors may |
| 8 | * be found in the AUTHORS file in the root of the source tree. |
| 9 | */ |
| 10 | |
Mirko Bonadei | 92ea95e | 2017-09-15 06:47:31 +0200 | [diff] [blame] | 11 | #ifndef MODULES_VIDEO_CODING_RTP_FRAME_REFERENCE_FINDER_H_ |
| 12 | #define MODULES_VIDEO_CODING_RTP_FRAME_REFERENCE_FINDER_H_ |
philipel | 02447bc | 2016-05-13 06:01:03 -0700 | [diff] [blame] | 13 | |
| 14 | #include <array> |
| 15 | #include <map> |
kwiberg | fd8be34 | 2016-05-14 19:44:11 -0700 | [diff] [blame] | 16 | #include <memory> |
philipel | 463d301 | 2016-09-09 03:32:44 -0700 | [diff] [blame] | 17 | #include <deque> |
philipel | 02447bc | 2016-05-13 06:01:03 -0700 | [diff] [blame] | 18 | #include <set> |
| 19 | #include <utility> |
| 20 | |
Mirko Bonadei | 92ea95e | 2017-09-15 06:47:31 +0200 | [diff] [blame] | 21 | #include "modules/include/module_common_types.h" |
| 22 | #include "modules/video_coding/sequence_number_util.h" |
| 23 | #include "rtc_base/criticalsection.h" |
| 24 | #include "rtc_base/thread_annotations.h" |
philipel | 02447bc | 2016-05-13 06:01:03 -0700 | [diff] [blame] | 25 | |
| 26 | namespace webrtc { |
| 27 | namespace video_coding { |
| 28 | |
philipel | 17deeb4 | 2016-08-11 15:09:26 +0200 | [diff] [blame] | 29 | class FrameObject; |
philipel | 02447bc | 2016-05-13 06:01:03 -0700 | [diff] [blame] | 30 | class RtpFrameObject; |
philipel | 17deeb4 | 2016-08-11 15:09:26 +0200 | [diff] [blame] | 31 | |
| 32 | // A complete frame is a frame which has received all its packets and all its |
| 33 | // references are known. |
| 34 | class OnCompleteFrameCallback { |
| 35 | public: |
| 36 | virtual ~OnCompleteFrameCallback() {} |
| 37 | virtual void OnCompleteFrame(std::unique_ptr<FrameObject> frame) = 0; |
| 38 | }; |
philipel | 02447bc | 2016-05-13 06:01:03 -0700 | [diff] [blame] | 39 | |
| 40 | class RtpFrameReferenceFinder { |
| 41 | public: |
| 42 | explicit RtpFrameReferenceFinder(OnCompleteFrameCallback* frame_callback); |
philipel | 463d301 | 2016-09-09 03:32:44 -0700 | [diff] [blame] | 43 | |
| 44 | // Manage this frame until: |
| 45 | // - We have all information needed to determine its references, after |
| 46 | // which |frame_callback_| is called with the completed frame, or |
philipel | afcf7f5 | 2017-04-26 08:17:35 -0700 | [diff] [blame] | 47 | // - We have too many stashed frames (determined by |kMaxStashedFrames|) |
philipel | 463d301 | 2016-09-09 03:32:44 -0700 | [diff] [blame] | 48 | // so we drop this frame, or |
| 49 | // - It gets cleared by ClearTo, which also means we drop it. |
philipel | 02447bc | 2016-05-13 06:01:03 -0700 | [diff] [blame] | 50 | void ManageFrame(std::unique_ptr<RtpFrameObject> frame); |
philipel | 463d301 | 2016-09-09 03:32:44 -0700 | [diff] [blame] | 51 | |
| 52 | // Notifies that padding has been received, which the reference finder |
| 53 | // might need to calculate the references of a frame. |
philipel | 9b2ce6b | 2016-07-05 05:04:46 -0700 | [diff] [blame] | 54 | void PaddingReceived(uint16_t seq_num); |
philipel | 02447bc | 2016-05-13 06:01:03 -0700 | [diff] [blame] | 55 | |
philipel | 463d301 | 2016-09-09 03:32:44 -0700 | [diff] [blame] | 56 | // Clear all stashed frames that include packets older than |seq_num|. |
| 57 | void ClearTo(uint16_t seq_num); |
| 58 | |
philipel | 02447bc | 2016-05-13 06:01:03 -0700 | [diff] [blame] | 59 | private: |
philipel | fd5a20f | 2016-11-15 00:57:57 -0800 | [diff] [blame] | 60 | static const uint16_t kPicIdLength = 1 << 15; |
philipel | 02447bc | 2016-05-13 06:01:03 -0700 | [diff] [blame] | 61 | static const uint8_t kMaxTemporalLayers = 5; |
philipel | fd5a20f | 2016-11-15 00:57:57 -0800 | [diff] [blame] | 62 | static const int kMaxLayerInfo = 50; |
| 63 | static const int kMaxStashedFrames = 50; |
| 64 | static const int kMaxNotYetReceivedFrames = 100; |
| 65 | static const int kMaxGofSaved = 50; |
philipel | 9b2ce6b | 2016-07-05 05:04:46 -0700 | [diff] [blame] | 66 | static const int kMaxPaddingAge = 100; |
philipel | 02447bc | 2016-05-13 06:01:03 -0700 | [diff] [blame] | 67 | |
philipel | afcf7f5 | 2017-04-26 08:17:35 -0700 | [diff] [blame] | 68 | enum FrameDecision { kStash, kHandOff, kDrop }; |
philipel | c9b27d5 | 2016-07-15 06:50:27 -0700 | [diff] [blame] | 69 | |
| 70 | struct GofInfo { |
| 71 | GofInfo(GofInfoVP9* gof, uint16_t last_picture_id) |
| 72 | : gof(gof), last_picture_id(last_picture_id) {} |
| 73 | GofInfoVP9* gof; |
| 74 | uint16_t last_picture_id; |
| 75 | }; |
| 76 | |
philipel | 02447bc | 2016-05-13 06:01:03 -0700 | [diff] [blame] | 77 | rtc::CriticalSection crit_; |
| 78 | |
philipel | 9b2ce6b | 2016-07-05 05:04:46 -0700 | [diff] [blame] | 79 | // Find the relevant group of pictures and update its "last-picture-id-with |
| 80 | // padding" sequence number. |
| 81 | void UpdateLastPictureIdWithPadding(uint16_t seq_num) |
danilchap | 56359be | 2017-09-07 07:53:45 -0700 | [diff] [blame] | 82 | RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_); |
philipel | 9b2ce6b | 2016-07-05 05:04:46 -0700 | [diff] [blame] | 83 | |
philipel | afcf7f5 | 2017-04-26 08:17:35 -0700 | [diff] [blame] | 84 | // Retry stashed frames until no more complete frames are found. |
danilchap | 56359be | 2017-09-07 07:53:45 -0700 | [diff] [blame] | 85 | void RetryStashedFrames() RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_); |
philipel | 02447bc | 2016-05-13 06:01:03 -0700 | [diff] [blame] | 86 | |
philipel | afcf7f5 | 2017-04-26 08:17:35 -0700 | [diff] [blame] | 87 | FrameDecision ManageFrameInternal(RtpFrameObject* frame) |
danilchap | 56359be | 2017-09-07 07:53:45 -0700 | [diff] [blame] | 88 | RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_); |
philipel | afcf7f5 | 2017-04-26 08:17:35 -0700 | [diff] [blame] | 89 | |
philipel | 647998c | 2016-06-03 09:40:16 -0700 | [diff] [blame] | 90 | // Find references for generic frames. If |picture_id| is unspecified |
| 91 | // then packet sequence numbers will be used to determine the references |
| 92 | // of the frames. |
philipel | afcf7f5 | 2017-04-26 08:17:35 -0700 | [diff] [blame] | 93 | FrameDecision ManageFrameGeneric(RtpFrameObject* frame, int picture_id) |
danilchap | 56359be | 2017-09-07 07:53:45 -0700 | [diff] [blame] | 94 | RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_); |
philipel | 02447bc | 2016-05-13 06:01:03 -0700 | [diff] [blame] | 95 | |
philipel | afcf7f5 | 2017-04-26 08:17:35 -0700 | [diff] [blame] | 96 | // Find references for Vp8 frames |
| 97 | FrameDecision ManageFrameVp8(RtpFrameObject* frame) |
danilchap | 56359be | 2017-09-07 07:53:45 -0700 | [diff] [blame] | 98 | RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_); |
philipel | afcf7f5 | 2017-04-26 08:17:35 -0700 | [diff] [blame] | 99 | |
| 100 | // Updates necessary layer info state used to determine frame references for |
| 101 | // Vp8. |
philipel | d4fac69 | 2017-09-04 07:03:46 -0700 | [diff] [blame] | 102 | void UpdateLayerInfoVp8(RtpFrameObject* frame) |
danilchap | 56359be | 2017-09-07 07:53:45 -0700 | [diff] [blame] | 103 | RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_); |
philipel | 02447bc | 2016-05-13 06:01:03 -0700 | [diff] [blame] | 104 | |
| 105 | // Find references for Vp9 frames |
philipel | afcf7f5 | 2017-04-26 08:17:35 -0700 | [diff] [blame] | 106 | FrameDecision ManageFrameVp9(RtpFrameObject* frame) |
danilchap | 56359be | 2017-09-07 07:53:45 -0700 | [diff] [blame] | 107 | RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_); |
philipel | 02447bc | 2016-05-13 06:01:03 -0700 | [diff] [blame] | 108 | |
| 109 | // Check if we are missing a frame necessary to determine the references |
| 110 | // for this frame. |
philipel | c9b27d5 | 2016-07-15 06:50:27 -0700 | [diff] [blame] | 111 | bool MissingRequiredFrameVp9(uint16_t picture_id, const GofInfo& info) |
danilchap | 56359be | 2017-09-07 07:53:45 -0700 | [diff] [blame] | 112 | RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_); |
philipel | 02447bc | 2016-05-13 06:01:03 -0700 | [diff] [blame] | 113 | |
| 114 | // Updates which frames that have been received. If there is a gap, |
| 115 | // missing frames will be added to |missing_frames_for_layer_| or |
| 116 | // if this is an already missing frame then it will be removed. |
philipel | c9b27d5 | 2016-07-15 06:50:27 -0700 | [diff] [blame] | 117 | void FrameReceivedVp9(uint16_t picture_id, GofInfo* info) |
danilchap | 56359be | 2017-09-07 07:53:45 -0700 | [diff] [blame] | 118 | RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_); |
philipel | 02447bc | 2016-05-13 06:01:03 -0700 | [diff] [blame] | 119 | |
| 120 | // Check if there is a frame with the up-switch flag set in the interval |
| 121 | // (|pid_ref|, |picture_id|) with temporal layer smaller than |temporal_idx|. |
| 122 | bool UpSwitchInIntervalVp9(uint16_t picture_id, |
| 123 | uint8_t temporal_idx, |
danilchap | 56359be | 2017-09-07 07:53:45 -0700 | [diff] [blame] | 124 | uint16_t pid_ref) |
| 125 | RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_); |
philipel | 02447bc | 2016-05-13 06:01:03 -0700 | [diff] [blame] | 126 | |
philipel | afcf7f5 | 2017-04-26 08:17:35 -0700 | [diff] [blame] | 127 | // Unwrap |frame|s picture id and its references to 16 bits. |
danilchap | 56359be | 2017-09-07 07:53:45 -0700 | [diff] [blame] | 128 | void UnwrapPictureIds(RtpFrameObject* frame) |
| 129 | RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_); |
philipel | 02447bc | 2016-05-13 06:01:03 -0700 | [diff] [blame] | 130 | |
philipel | fd5a20f | 2016-11-15 00:57:57 -0800 | [diff] [blame] | 131 | // Returns true if the frame is old and should be dropped. |
| 132 | // TODO(philipel): Remove when VP9 PID/TL0 does not jump mid-stream (should be |
| 133 | // around M59). |
| 134 | bool Vp9PidTl0Fix(const RtpFrameObject& frame, |
| 135 | int16_t* picture_id, |
danilchap | 56359be | 2017-09-07 07:53:45 -0700 | [diff] [blame] | 136 | int16_t* tl0_pic_idx) RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_); |
philipel | fd5a20f | 2016-11-15 00:57:57 -0800 | [diff] [blame] | 137 | |
| 138 | // TODO(philipel): Remove when VP9 PID/TL0 does not jump mid-stream (should be |
| 139 | // around M59). |
| 140 | bool DetectVp9PicIdJump(int fixed_pid, |
| 141 | int fixed_tl0, |
| 142 | uint32_t timestamp) const |
danilchap | 56359be | 2017-09-07 07:53:45 -0700 | [diff] [blame] | 143 | RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_); |
philipel | fd5a20f | 2016-11-15 00:57:57 -0800 | [diff] [blame] | 144 | |
| 145 | // TODO(philipel): Remove when VP9 PID/TL0 does not jump mid-stream (should be |
| 146 | // around M59). |
| 147 | bool DetectVp9Tl0PicIdxJump(int fixed_tl0, uint32_t timestamp) const |
danilchap | 56359be | 2017-09-07 07:53:45 -0700 | [diff] [blame] | 148 | RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_); |
philipel | 9b2ce6b | 2016-07-05 05:04:46 -0700 | [diff] [blame] | 149 | |
| 150 | // For every group of pictures, hold two sequence numbers. The first being |
| 151 | // the sequence number of the last packet of the last completed frame, and |
| 152 | // the second being the sequence number of the last packet of the last |
| 153 | // completed frame advanced by any potential continuous packets of padding. |
| 154 | std::map<uint16_t, |
| 155 | std::pair<uint16_t, uint16_t>, |
| 156 | DescendingSeqNumComp<uint16_t>> |
danilchap | 56359be | 2017-09-07 07:53:45 -0700 | [diff] [blame] | 157 | last_seq_num_gop_ RTC_GUARDED_BY(crit_); |
philipel | 02447bc | 2016-05-13 06:01:03 -0700 | [diff] [blame] | 158 | |
| 159 | // Save the last picture id in order to detect when there is a gap in frames |
| 160 | // that have not yet been fully received. |
danilchap | 56359be | 2017-09-07 07:53:45 -0700 | [diff] [blame] | 161 | int last_picture_id_ RTC_GUARDED_BY(crit_); |
philipel | 02447bc | 2016-05-13 06:01:03 -0700 | [diff] [blame] | 162 | |
philipel | 9b2ce6b | 2016-07-05 05:04:46 -0700 | [diff] [blame] | 163 | // Padding packets that have been received but that are not yet continuous |
| 164 | // with any group of pictures. |
| 165 | std::set<uint16_t, DescendingSeqNumComp<uint16_t>> stashed_padding_ |
danilchap | 56359be | 2017-09-07 07:53:45 -0700 | [diff] [blame] | 166 | RTC_GUARDED_BY(crit_); |
philipel | 9b2ce6b | 2016-07-05 05:04:46 -0700 | [diff] [blame] | 167 | |
philipel | 02447bc | 2016-05-13 06:01:03 -0700 | [diff] [blame] | 168 | // The last unwrapped picture id. Used to unwrap the picture id from a length |
| 169 | // of |kPicIdLength| to 16 bits. |
danilchap | 56359be | 2017-09-07 07:53:45 -0700 | [diff] [blame] | 170 | int last_unwrap_ RTC_GUARDED_BY(crit_); |
philipel | 02447bc | 2016-05-13 06:01:03 -0700 | [diff] [blame] | 171 | |
| 172 | // Frames earlier than the last received frame that have not yet been |
| 173 | // fully received. |
| 174 | std::set<uint16_t, DescendingSeqNumComp<uint16_t, kPicIdLength>> |
danilchap | 56359be | 2017-09-07 07:53:45 -0700 | [diff] [blame] | 175 | not_yet_received_frames_ RTC_GUARDED_BY(crit_); |
philipel | 02447bc | 2016-05-13 06:01:03 -0700 | [diff] [blame] | 176 | |
| 177 | // Frames that have been fully received but didn't have all the information |
| 178 | // needed to determine their references. |
danilchap | 56359be | 2017-09-07 07:53:45 -0700 | [diff] [blame] | 179 | std::deque<std::unique_ptr<RtpFrameObject>> stashed_frames_ |
| 180 | RTC_GUARDED_BY(crit_); |
philipel | 02447bc | 2016-05-13 06:01:03 -0700 | [diff] [blame] | 181 | |
| 182 | // Holds the information about the last completed frame for a given temporal |
| 183 | // layer given a Tl0 picture index. |
| 184 | std::map<uint8_t, |
| 185 | std::array<int16_t, kMaxTemporalLayers>, |
| 186 | DescendingSeqNumComp<uint8_t>> |
danilchap | 56359be | 2017-09-07 07:53:45 -0700 | [diff] [blame] | 187 | layer_info_ RTC_GUARDED_BY(crit_); |
philipel | 02447bc | 2016-05-13 06:01:03 -0700 | [diff] [blame] | 188 | |
| 189 | // Where the current scalability structure is in the |
| 190 | // |scalability_structures_| array. |
| 191 | uint8_t current_ss_idx_; |
| 192 | |
| 193 | // Holds received scalability structures. |
| 194 | std::array<GofInfoVP9, kMaxGofSaved> scalability_structures_ |
danilchap | 56359be | 2017-09-07 07:53:45 -0700 | [diff] [blame] | 195 | RTC_GUARDED_BY(crit_); |
philipel | 02447bc | 2016-05-13 06:01:03 -0700 | [diff] [blame] | 196 | |
philipel | c9b27d5 | 2016-07-15 06:50:27 -0700 | [diff] [blame] | 197 | // Holds the the Gof information for a given TL0 picture index. |
| 198 | std::map<uint8_t, GofInfo, DescendingSeqNumComp<uint8_t>> gof_info_ |
danilchap | 56359be | 2017-09-07 07:53:45 -0700 | [diff] [blame] | 199 | RTC_GUARDED_BY(crit_); |
philipel | 02447bc | 2016-05-13 06:01:03 -0700 | [diff] [blame] | 200 | |
| 201 | // Keep track of which picture id and which temporal layer that had the |
| 202 | // up switch flag set. |
philipel | c9b27d5 | 2016-07-15 06:50:27 -0700 | [diff] [blame] | 203 | std::map<uint16_t, uint8_t, DescendingSeqNumComp<uint16_t, kPicIdLength>> |
danilchap | 56359be | 2017-09-07 07:53:45 -0700 | [diff] [blame] | 204 | up_switch_ RTC_GUARDED_BY(crit_); |
philipel | 02447bc | 2016-05-13 06:01:03 -0700 | [diff] [blame] | 205 | |
| 206 | // For every temporal layer, keep a set of which frames that are missing. |
| 207 | std::array<std::set<uint16_t, DescendingSeqNumComp<uint16_t, kPicIdLength>>, |
| 208 | kMaxTemporalLayers> |
danilchap | 56359be | 2017-09-07 07:53:45 -0700 | [diff] [blame] | 209 | missing_frames_for_layer_ RTC_GUARDED_BY(crit_); |
philipel | 02447bc | 2016-05-13 06:01:03 -0700 | [diff] [blame] | 210 | |
philipel | 463d301 | 2016-09-09 03:32:44 -0700 | [diff] [blame] | 211 | // How far frames have been cleared by sequence number. A frame will be |
| 212 | // cleared if it contains a packet with a sequence number older than |
| 213 | // |cleared_to_seq_num_|. |
danilchap | 56359be | 2017-09-07 07:53:45 -0700 | [diff] [blame] | 214 | int cleared_to_seq_num_ RTC_GUARDED_BY(crit_); |
philipel | 463d301 | 2016-09-09 03:32:44 -0700 | [diff] [blame] | 215 | |
philipel | 02447bc | 2016-05-13 06:01:03 -0700 | [diff] [blame] | 216 | OnCompleteFrameCallback* frame_callback_; |
philipel | d4fac69 | 2017-09-04 07:03:46 -0700 | [diff] [blame] | 217 | |
| 218 | // Unwrapper used to unwrap generic RTP streams. In a generic stream we derive |
| 219 | // a picture id from the packet sequence number. |
danilchap | 56359be | 2017-09-07 07:53:45 -0700 | [diff] [blame] | 220 | SeqNumUnwrapper<uint16_t> generic_unwrapper_ RTC_GUARDED_BY(crit_); |
philipel | d4fac69 | 2017-09-04 07:03:46 -0700 | [diff] [blame] | 221 | |
| 222 | // Unwrapper used to unwrap VP8/VP9 streams which have their picture id |
| 223 | // specified. |
danilchap | 56359be | 2017-09-07 07:53:45 -0700 | [diff] [blame] | 224 | SeqNumUnwrapper<uint16_t, kPicIdLength> unwrapper_ RTC_GUARDED_BY(crit_); |
philipel | 02447bc | 2016-05-13 06:01:03 -0700 | [diff] [blame] | 225 | }; |
| 226 | |
| 227 | } // namespace video_coding |
| 228 | } // namespace webrtc |
| 229 | |
Mirko Bonadei | 92ea95e | 2017-09-15 06:47:31 +0200 | [diff] [blame] | 230 | #endif // MODULES_VIDEO_CODING_RTP_FRAME_REFERENCE_FINDER_H_ |