philipel | c707ab7 | 2016-04-01 02:01:54 -0700 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. |
| 3 | * |
| 4 | * Use of this source code is governed by a BSD-style license |
| 5 | * that can be found in the LICENSE file in the root of the source |
| 6 | * tree. An additional intellectual property rights grant can be found |
| 7 | * in the file PATENTS. All contributing project authors may |
| 8 | * be found in the AUTHORS file in the root of the source tree. |
| 9 | */ |
| 10 | |
Mirko Bonadei | 92ea95e | 2017-09-15 06:47:31 +0200 | [diff] [blame] | 11 | #ifndef MODULES_VIDEO_CODING_PACKET_BUFFER_H_ |
| 12 | #define MODULES_VIDEO_CODING_PACKET_BUFFER_H_ |
philipel | c707ab7 | 2016-04-01 02:01:54 -0700 | [diff] [blame] | 13 | |
philipel | 17deeb4 | 2016-08-11 15:09:26 +0200 | [diff] [blame] | 14 | #include <memory> |
Ilya Nikolaevskiy | d397a0d | 2018-02-21 15:57:09 +0100 | [diff] [blame] | 15 | #include <queue> |
philipel | 2c9f9f2 | 2017-06-13 02:47:28 -0700 | [diff] [blame] | 16 | #include <set> |
| 17 | #include <vector> |
philipel | c707ab7 | 2016-04-01 02:01:54 -0700 | [diff] [blame] | 18 | |
philipel | b5e4785 | 2019-09-20 11:30:12 +0200 | [diff] [blame] | 19 | #include "api/video/encoded_image.h" |
Mirko Bonadei | 92ea95e | 2017-09-15 06:47:31 +0200 | [diff] [blame] | 20 | #include "modules/video_coding/packet.h" |
Steve Anton | 10542f2 | 2019-01-11 09:11:00 -0800 | [diff] [blame] | 21 | #include "rtc_base/critical_section.h" |
Bjorn Terelius | a194e58 | 2017-10-25 13:07:09 +0200 | [diff] [blame] | 22 | #include "rtc_base/numerics/sequence_number_util.h" |
Mirko Bonadei | 92ea95e | 2017-09-15 06:47:31 +0200 | [diff] [blame] | 23 | #include "rtc_base/thread_annotations.h" |
philipel | c707ab7 | 2016-04-01 02:01:54 -0700 | [diff] [blame] | 24 | |
| 25 | namespace webrtc { |
philipel | b4d3108 | 2016-07-11 08:46:29 -0700 | [diff] [blame] | 26 | |
| 27 | class Clock; |
| 28 | |
philipel | c707ab7 | 2016-04-01 02:01:54 -0700 | [diff] [blame] | 29 | namespace video_coding { |
| 30 | |
philipel | c707ab7 | 2016-04-01 02:01:54 -0700 | [diff] [blame] | 31 | class RtpFrameObject; |
| 32 | |
Elad Alon | b4643ad | 2019-02-22 11:19:50 +0100 | [diff] [blame] | 33 | // A frame is assembled when all of its packets have been received. |
| 34 | class OnAssembledFrameCallback { |
philipel | c707ab7 | 2016-04-01 02:01:54 -0700 | [diff] [blame] | 35 | public: |
Elad Alon | b4643ad | 2019-02-22 11:19:50 +0100 | [diff] [blame] | 36 | virtual ~OnAssembledFrameCallback() {} |
| 37 | virtual void OnAssembledFrame(std::unique_ptr<RtpFrameObject> frame) = 0; |
philipel | c707ab7 | 2016-04-01 02:01:54 -0700 | [diff] [blame] | 38 | }; |
| 39 | |
| 40 | class PacketBuffer { |
| 41 | public: |
Danil Chapovalov | f7457e5 | 2019-09-20 17:57:15 +0200 | [diff] [blame] | 42 | // Both |start_buffer_size| and |max_buffer_size| must be a power of 2. |
| 43 | PacketBuffer(Clock* clock, |
| 44 | size_t start_buffer_size, |
| 45 | size_t max_buffer_size, |
| 46 | OnAssembledFrameCallback* frame_callback); |
philipel | 17deeb4 | 2016-08-11 15:09:26 +0200 | [diff] [blame] | 47 | virtual ~PacketBuffer(); |
| 48 | |
Johannes Kron | bd3f305 | 2019-08-01 15:45:54 +0200 | [diff] [blame] | 49 | // Returns true unless the packet buffer is cleared, which means that a key |
| 50 | // frame request should be sent. The PacketBuffer will always take ownership |
| 51 | // of the |packet.dataPtr| when this function is called. Made virtual for |
| 52 | // testing. |
philipel | 759e0b7 | 2016-11-30 01:32:05 -0800 | [diff] [blame] | 53 | virtual bool InsertPacket(VCMPacket* packet); |
philipel | 17deeb4 | 2016-08-11 15:09:26 +0200 | [diff] [blame] | 54 | void ClearTo(uint16_t seq_num); |
| 55 | void Clear(); |
philipel | 2c9f9f2 | 2017-06-13 02:47:28 -0700 | [diff] [blame] | 56 | void PaddingReceived(uint16_t seq_num); |
philipel | 17deeb4 | 2016-08-11 15:09:26 +0200 | [diff] [blame] | 57 | |
philipel | 3184f8e | 2017-05-18 08:08:53 -0700 | [diff] [blame] | 58 | // Timestamp (not RTP timestamp) of the last received packet/keyframe packet. |
Danil Chapovalov | 0040b66 | 2018-06-18 10:48:16 +0200 | [diff] [blame] | 59 | absl::optional<int64_t> LastReceivedPacketMs() const; |
| 60 | absl::optional<int64_t> LastReceivedKeyframePacketMs() const; |
philipel | 3184f8e | 2017-05-18 08:08:53 -0700 | [diff] [blame] | 61 | |
Ilya Nikolaevskiy | d397a0d | 2018-02-21 15:57:09 +0100 | [diff] [blame] | 62 | // Returns number of different frames seen in the packet buffer |
| 63 | int GetUniqueFramesSeen() const; |
| 64 | |
philipel | c707ab7 | 2016-04-01 02:01:54 -0700 | [diff] [blame] | 65 | private: |
| 66 | friend RtpFrameObject; |
| 67 | // Since we want the packet buffer to be as packet type agnostic |
| 68 | // as possible we extract only the information needed in order |
philipel | f413933 | 2016-04-20 10:26:34 +0200 | [diff] [blame] | 69 | // to determine whether a sequence of packets is continuous or not. |
philipel | c707ab7 | 2016-04-01 02:01:54 -0700 | [diff] [blame] | 70 | struct ContinuityInfo { |
philipel | f413933 | 2016-04-20 10:26:34 +0200 | [diff] [blame] | 71 | // The sequence number of the packet. |
philipel | c707ab7 | 2016-04-01 02:01:54 -0700 | [diff] [blame] | 72 | uint16_t seq_num = 0; |
philipel | f413933 | 2016-04-20 10:26:34 +0200 | [diff] [blame] | 73 | |
| 74 | // If this is the first packet of the frame. |
philipel | c707ab7 | 2016-04-01 02:01:54 -0700 | [diff] [blame] | 75 | bool frame_begin = false; |
philipel | f413933 | 2016-04-20 10:26:34 +0200 | [diff] [blame] | 76 | |
| 77 | // If this is the last packet of the frame. |
philipel | c707ab7 | 2016-04-01 02:01:54 -0700 | [diff] [blame] | 78 | bool frame_end = false; |
philipel | f413933 | 2016-04-20 10:26:34 +0200 | [diff] [blame] | 79 | |
| 80 | // If this slot is currently used. |
philipel | c707ab7 | 2016-04-01 02:01:54 -0700 | [diff] [blame] | 81 | bool used = false; |
philipel | f413933 | 2016-04-20 10:26:34 +0200 | [diff] [blame] | 82 | |
| 83 | // If all its previous packets have been inserted into the packet buffer. |
philipel | c707ab7 | 2016-04-01 02:01:54 -0700 | [diff] [blame] | 84 | bool continuous = false; |
philipel | f413933 | 2016-04-20 10:26:34 +0200 | [diff] [blame] | 85 | |
| 86 | // If this packet has been used to create a frame already. |
| 87 | bool frame_created = false; |
philipel | c707ab7 | 2016-04-01 02:01:54 -0700 | [diff] [blame] | 88 | }; |
| 89 | |
philipel | b4d3108 | 2016-07-11 08:46:29 -0700 | [diff] [blame] | 90 | Clock* const clock_; |
| 91 | |
philipel | 02447bc | 2016-05-13 06:01:03 -0700 | [diff] [blame] | 92 | // Tries to expand the buffer. |
danilchap | 56359be | 2017-09-07 07:53:45 -0700 | [diff] [blame] | 93 | bool ExpandBufferSize() RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_); |
philipel | f413933 | 2016-04-20 10:26:34 +0200 | [diff] [blame] | 94 | |
| 95 | // Test if all previous packets has arrived for the given sequence number. |
philipel | aee3e0e | 2016-11-01 11:45:34 +0100 | [diff] [blame] | 96 | bool PotentialNewFrame(uint16_t seq_num) const |
danilchap | 56359be | 2017-09-07 07:53:45 -0700 | [diff] [blame] | 97 | RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_); |
philipel | f413933 | 2016-04-20 10:26:34 +0200 | [diff] [blame] | 98 | |
| 99 | // Test if all packets of a frame has arrived, and if so, creates a frame. |
philipel | fd5a20f | 2016-11-15 00:57:57 -0800 | [diff] [blame] | 100 | // Returns a vector of received frames. |
| 101 | std::vector<std::unique_ptr<RtpFrameObject>> FindFrames(uint16_t seq_num) |
danilchap | 56359be | 2017-09-07 07:53:45 -0700 | [diff] [blame] | 102 | RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_); |
philipel | f413933 | 2016-04-20 10:26:34 +0200 | [diff] [blame] | 103 | |
philipel | b5e4785 | 2019-09-20 11:30:12 +0200 | [diff] [blame] | 104 | rtc::scoped_refptr<EncodedImageBuffer> GetEncodedImageBuffer( |
| 105 | size_t frame_size, |
| 106 | uint16_t first_seq_num, |
| 107 | uint16_t last_seq_num) RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_); |
philipel | f413933 | 2016-04-20 10:26:34 +0200 | [diff] [blame] | 108 | |
philipel | 02447bc | 2016-05-13 06:01:03 -0700 | [diff] [blame] | 109 | // Get the packet with sequence number |seq_num|. |
philipel | 17deeb4 | 2016-08-11 15:09:26 +0200 | [diff] [blame] | 110 | // Virtual for testing. |
philipel | fd5a20f | 2016-11-15 00:57:57 -0800 | [diff] [blame] | 111 | virtual VCMPacket* GetPacket(uint16_t seq_num) |
danilchap | 56359be | 2017-09-07 07:53:45 -0700 | [diff] [blame] | 112 | RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_); |
philipel | 02447bc | 2016-05-13 06:01:03 -0700 | [diff] [blame] | 113 | |
Johannes Kron | a370556 | 2019-08-26 16:37:11 +0200 | [diff] [blame] | 114 | // Clears the packet buffer from |start_seq_num| to |stop_seq_num| where the |
| 115 | // endpoints are inclusive. |
| 116 | void ClearInterval(uint16_t start_seq_num, uint16_t stop_seq_num) |
| 117 | RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_); |
philipel | c707ab7 | 2016-04-01 02:01:54 -0700 | [diff] [blame] | 118 | |
danilchap | 56359be | 2017-09-07 07:53:45 -0700 | [diff] [blame] | 119 | void UpdateMissingPackets(uint16_t seq_num) |
| 120 | RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_); |
philipel | 2c9f9f2 | 2017-06-13 02:47:28 -0700 | [diff] [blame] | 121 | |
Ilya Nikolaevskiy | d397a0d | 2018-02-21 15:57:09 +0100 | [diff] [blame] | 122 | // Counts unique received timestamps and updates |unique_frames_seen_|. |
| 123 | void OnTimestampReceived(uint32_t rtp_timestamp) |
| 124 | RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_); |
| 125 | |
philipel | c707ab7 | 2016-04-01 02:01:54 -0700 | [diff] [blame] | 126 | rtc::CriticalSection crit_; |
| 127 | |
| 128 | // Buffer size_ and max_size_ must always be a power of two. |
danilchap | 56359be | 2017-09-07 07:53:45 -0700 | [diff] [blame] | 129 | size_t size_ RTC_GUARDED_BY(crit_); |
philipel | c707ab7 | 2016-04-01 02:01:54 -0700 | [diff] [blame] | 130 | const size_t max_size_; |
| 131 | |
philipel | f413933 | 2016-04-20 10:26:34 +0200 | [diff] [blame] | 132 | // The fist sequence number currently in the buffer. |
danilchap | 56359be | 2017-09-07 07:53:45 -0700 | [diff] [blame] | 133 | uint16_t first_seq_num_ RTC_GUARDED_BY(crit_); |
philipel | f413933 | 2016-04-20 10:26:34 +0200 | [diff] [blame] | 134 | |
philipel | f413933 | 2016-04-20 10:26:34 +0200 | [diff] [blame] | 135 | // If the packet buffer has received its first packet. |
danilchap | 56359be | 2017-09-07 07:53:45 -0700 | [diff] [blame] | 136 | bool first_packet_received_ RTC_GUARDED_BY(crit_); |
philipel | f413933 | 2016-04-20 10:26:34 +0200 | [diff] [blame] | 137 | |
philipel | aee3e0e | 2016-11-01 11:45:34 +0100 | [diff] [blame] | 138 | // If the buffer is cleared to |first_seq_num_|. |
danilchap | 56359be | 2017-09-07 07:53:45 -0700 | [diff] [blame] | 139 | bool is_cleared_to_first_seq_num_ RTC_GUARDED_BY(crit_); |
philipel | aee3e0e | 2016-11-01 11:45:34 +0100 | [diff] [blame] | 140 | |
philipel | f413933 | 2016-04-20 10:26:34 +0200 | [diff] [blame] | 141 | // Buffer that holds the inserted packets. |
danilchap | 56359be | 2017-09-07 07:53:45 -0700 | [diff] [blame] | 142 | std::vector<VCMPacket> data_buffer_ RTC_GUARDED_BY(crit_); |
philipel | f413933 | 2016-04-20 10:26:34 +0200 | [diff] [blame] | 143 | |
| 144 | // Buffer that holds the information about which slot that is currently in use |
| 145 | // and information needed to determine the continuity between packets. |
danilchap | 56359be | 2017-09-07 07:53:45 -0700 | [diff] [blame] | 146 | std::vector<ContinuityInfo> sequence_buffer_ RTC_GUARDED_BY(crit_); |
philipel | c707ab7 | 2016-04-01 02:01:54 -0700 | [diff] [blame] | 147 | |
Elad Alon | b4643ad | 2019-02-22 11:19:50 +0100 | [diff] [blame] | 148 | // Called when all packets in a frame are received, allowing the frame |
| 149 | // to be assembled. |
| 150 | OnAssembledFrameCallback* const assembled_frame_callback_; |
philipel | 17deeb4 | 2016-08-11 15:09:26 +0200 | [diff] [blame] | 151 | |
philipel | 3184f8e | 2017-05-18 08:08:53 -0700 | [diff] [blame] | 152 | // Timestamp (not RTP timestamp) of the last received packet/keyframe packet. |
Danil Chapovalov | 0040b66 | 2018-06-18 10:48:16 +0200 | [diff] [blame] | 153 | absl::optional<int64_t> last_received_packet_ms_ RTC_GUARDED_BY(crit_); |
| 154 | absl::optional<int64_t> last_received_keyframe_packet_ms_ |
danilchap | 56359be | 2017-09-07 07:53:45 -0700 | [diff] [blame] | 155 | RTC_GUARDED_BY(crit_); |
philipel | 3184f8e | 2017-05-18 08:08:53 -0700 | [diff] [blame] | 156 | |
Ilya Nikolaevskiy | d397a0d | 2018-02-21 15:57:09 +0100 | [diff] [blame] | 157 | int unique_frames_seen_ RTC_GUARDED_BY(crit_); |
| 158 | |
Danil Chapovalov | 0040b66 | 2018-06-18 10:48:16 +0200 | [diff] [blame] | 159 | absl::optional<uint16_t> newest_inserted_seq_num_ RTC_GUARDED_BY(crit_); |
philipel | 2c9f9f2 | 2017-06-13 02:47:28 -0700 | [diff] [blame] | 160 | std::set<uint16_t, DescendingSeqNumComp<uint16_t>> missing_packets_ |
danilchap | 56359be | 2017-09-07 07:53:45 -0700 | [diff] [blame] | 161 | RTC_GUARDED_BY(crit_); |
philipel | 2c9f9f2 | 2017-06-13 02:47:28 -0700 | [diff] [blame] | 162 | |
Rasmus Brandt | 88f080a | 2017-11-02 14:28:06 +0100 | [diff] [blame] | 163 | // Indicates if we should require SPS, PPS, and IDR for a particular |
| 164 | // RTP timestamp to treat the corresponding frame as a keyframe. |
| 165 | const bool sps_pps_idr_is_h264_keyframe_; |
| 166 | |
Ilya Nikolaevskiy | d397a0d | 2018-02-21 15:57:09 +0100 | [diff] [blame] | 167 | // Stores several last seen unique timestamps for quick search. |
| 168 | std::set<uint32_t> rtp_timestamps_history_set_ RTC_GUARDED_BY(crit_); |
| 169 | // Stores the same unique timestamps in the order of insertion. |
| 170 | std::queue<uint32_t> rtp_timestamps_history_queue_ RTC_GUARDED_BY(crit_); |
philipel | c707ab7 | 2016-04-01 02:01:54 -0700 | [diff] [blame] | 171 | }; |
| 172 | |
| 173 | } // namespace video_coding |
| 174 | } // namespace webrtc |
| 175 | |
Mirko Bonadei | 92ea95e | 2017-09-15 06:47:31 +0200 | [diff] [blame] | 176 | #endif // MODULES_VIDEO_CODING_PACKET_BUFFER_H_ |