philipel | c707ab7 | 2016-04-01 02:01:54 -0700 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. |
| 3 | * |
| 4 | * Use of this source code is governed by a BSD-style license |
| 5 | * that can be found in the LICENSE file in the root of the source |
| 6 | * tree. An additional intellectual property rights grant can be found |
| 7 | * in the file PATENTS. All contributing project authors may |
| 8 | * be found in the AUTHORS file in the root of the source tree. |
| 9 | */ |
| 10 | |
Mirko Bonadei | 92ea95e | 2017-09-15 06:47:31 +0200 | [diff] [blame] | 11 | #include "modules/video_coding/packet_buffer.h" |
philipel | c707ab7 | 2016-04-01 02:01:54 -0700 | [diff] [blame] | 12 | |
Yves Gerey | 3e70781 | 2018-11-28 16:47:49 +0100 | [diff] [blame] | 13 | #include <string.h> |
Jonas Olsson | a4d8737 | 2019-07-05 19:08:33 +0200 | [diff] [blame] | 14 | |
philipel | c707ab7 | 2016-04-01 02:01:54 -0700 | [diff] [blame] | 15 | #include <algorithm> |
Yves Gerey | 3e70781 | 2018-11-28 16:47:49 +0100 | [diff] [blame] | 16 | #include <cstdint> |
philipel | 17deeb4 | 2016-08-11 15:09:26 +0200 | [diff] [blame] | 17 | #include <utility> |
philipel | c707ab7 | 2016-04-01 02:01:54 -0700 | [diff] [blame] | 18 | |
Yves Gerey | 3e70781 | 2018-11-28 16:47:49 +0100 | [diff] [blame] | 19 | #include "absl/types/variant.h" |
| 20 | #include "api/video/encoded_frame.h" |
Mirko Bonadei | 92ea95e | 2017-09-15 06:47:31 +0200 | [diff] [blame] | 21 | #include "common_video/h264/h264_common.h" |
Yves Gerey | 3e70781 | 2018-11-28 16:47:49 +0100 | [diff] [blame] | 22 | #include "modules/rtp_rtcp/source/rtp_video_header.h" |
| 23 | #include "modules/video_coding/codecs/h264/include/h264_globals.h" |
Mirko Bonadei | 92ea95e | 2017-09-15 06:47:31 +0200 | [diff] [blame] | 24 | #include "modules/video_coding/frame_object.h" |
Mirko Bonadei | 92ea95e | 2017-09-15 06:47:31 +0200 | [diff] [blame] | 25 | #include "rtc_base/checks.h" |
| 26 | #include "rtc_base/logging.h" |
Yves Gerey | 3e70781 | 2018-11-28 16:47:49 +0100 | [diff] [blame] | 27 | #include "rtc_base/numerics/mod_ops.h" |
Mirko Bonadei | 92ea95e | 2017-09-15 06:47:31 +0200 | [diff] [blame] | 28 | #include "system_wrappers/include/clock.h" |
Rasmus Brandt | 88f080a | 2017-11-02 14:28:06 +0100 | [diff] [blame] | 29 | #include "system_wrappers/include/field_trial.h" |
philipel | c707ab7 | 2016-04-01 02:01:54 -0700 | [diff] [blame] | 30 | |
| 31 | namespace webrtc { |
| 32 | namespace video_coding { |
| 33 | |
philipel | b4d3108 | 2016-07-11 08:46:29 -0700 | [diff] [blame] | 34 | PacketBuffer::PacketBuffer(Clock* clock, |
| 35 | size_t start_buffer_size, |
philipel | c707ab7 | 2016-04-01 02:01:54 -0700 | [diff] [blame] | 36 | size_t max_buffer_size, |
Elad Alon | b4643ad | 2019-02-22 11:19:50 +0100 | [diff] [blame] | 37 | OnAssembledFrameCallback* assembled_frame_callback) |
philipel | b4d3108 | 2016-07-11 08:46:29 -0700 | [diff] [blame] | 38 | : clock_(clock), |
| 39 | size_(start_buffer_size), |
philipel | c707ab7 | 2016-04-01 02:01:54 -0700 | [diff] [blame] | 40 | max_size_(max_buffer_size), |
philipel | c707ab7 | 2016-04-01 02:01:54 -0700 | [diff] [blame] | 41 | first_seq_num_(0), |
philipel | f413933 | 2016-04-20 10:26:34 +0200 | [diff] [blame] | 42 | first_packet_received_(false), |
philipel | aee3e0e | 2016-11-01 11:45:34 +0100 | [diff] [blame] | 43 | is_cleared_to_first_seq_num_(false), |
philipel | c707ab7 | 2016-04-01 02:01:54 -0700 | [diff] [blame] | 44 | data_buffer_(start_buffer_size), |
| 45 | sequence_buffer_(start_buffer_size), |
Elad Alon | b4643ad | 2019-02-22 11:19:50 +0100 | [diff] [blame] | 46 | assembled_frame_callback_(assembled_frame_callback), |
Ilya Nikolaevskiy | d397a0d | 2018-02-21 15:57:09 +0100 | [diff] [blame] | 47 | unique_frames_seen_(0), |
Rasmus Brandt | 88f080a | 2017-11-02 14:28:06 +0100 | [diff] [blame] | 48 | sps_pps_idr_is_h264_keyframe_( |
| 49 | field_trial::IsEnabled("WebRTC-SpsPpsIdrIsH264Keyframe")) { |
philipel | c707ab7 | 2016-04-01 02:01:54 -0700 | [diff] [blame] | 50 | RTC_DCHECK_LE(start_buffer_size, max_buffer_size); |
| 51 | // Buffer size must always be a power of 2. |
| 52 | RTC_DCHECK((start_buffer_size & (start_buffer_size - 1)) == 0); |
| 53 | RTC_DCHECK((max_buffer_size & (max_buffer_size - 1)) == 0); |
| 54 | } |
| 55 | |
philipel | aee3e0e | 2016-11-01 11:45:34 +0100 | [diff] [blame] | 56 | PacketBuffer::~PacketBuffer() { |
| 57 | Clear(); |
| 58 | } |
philipel | 17deeb4 | 2016-08-11 15:09:26 +0200 | [diff] [blame] | 59 | |
philipel | 759e0b7 | 2016-11-30 01:32:05 -0800 | [diff] [blame] | 60 | bool PacketBuffer::InsertPacket(VCMPacket* packet) { |
philipel | fd5a20f | 2016-11-15 00:57:57 -0800 | [diff] [blame] | 61 | std::vector<std::unique_ptr<RtpFrameObject>> found_frames; |
| 62 | { |
| 63 | rtc::CritScope lock(&crit_); |
philipel | 3184f8e | 2017-05-18 08:08:53 -0700 | [diff] [blame] | 64 | |
Ilya Nikolaevskiy | d397a0d | 2018-02-21 15:57:09 +0100 | [diff] [blame] | 65 | OnTimestampReceived(packet->timestamp); |
| 66 | |
philipel | 759e0b7 | 2016-11-30 01:32:05 -0800 | [diff] [blame] | 67 | uint16_t seq_num = packet->seqNum; |
philipel | fd5a20f | 2016-11-15 00:57:57 -0800 | [diff] [blame] | 68 | size_t index = seq_num % size_; |
philipel | c707ab7 | 2016-04-01 02:01:54 -0700 | [diff] [blame] | 69 | |
philipel | fd5a20f | 2016-11-15 00:57:57 -0800 | [diff] [blame] | 70 | if (!first_packet_received_) { |
| 71 | first_seq_num_ = seq_num; |
philipel | fd5a20f | 2016-11-15 00:57:57 -0800 | [diff] [blame] | 72 | first_packet_received_ = true; |
| 73 | } else if (AheadOf(first_seq_num_, seq_num)) { |
| 74 | // If we have explicitly cleared past this packet then it's old, |
Johannes Kron | bd3f305 | 2019-08-01 15:45:54 +0200 | [diff] [blame] | 75 | // don't insert it, just silently ignore it. |
philipel | 759e0b7 | 2016-11-30 01:32:05 -0800 | [diff] [blame] | 76 | if (is_cleared_to_first_seq_num_) { |
| 77 | delete[] packet->dataPtr; |
| 78 | packet->dataPtr = nullptr; |
Johannes Kron | bd3f305 | 2019-08-01 15:45:54 +0200 | [diff] [blame] | 79 | return true; |
philipel | 759e0b7 | 2016-11-30 01:32:05 -0800 | [diff] [blame] | 80 | } |
philipel | aee3e0e | 2016-11-01 11:45:34 +0100 | [diff] [blame] | 81 | |
philipel | fd5a20f | 2016-11-15 00:57:57 -0800 | [diff] [blame] | 82 | first_seq_num_ = seq_num; |
philipel | c707ab7 | 2016-04-01 02:01:54 -0700 | [diff] [blame] | 83 | } |
philipel | c707ab7 | 2016-04-01 02:01:54 -0700 | [diff] [blame] | 84 | |
philipel | fd5a20f | 2016-11-15 00:57:57 -0800 | [diff] [blame] | 85 | if (sequence_buffer_[index].used) { |
philipel | 759e0b7 | 2016-11-30 01:32:05 -0800 | [diff] [blame] | 86 | // Duplicate packet, just delete the payload. |
| 87 | if (data_buffer_[index].seqNum == packet->seqNum) { |
| 88 | delete[] packet->dataPtr; |
| 89 | packet->dataPtr = nullptr; |
philipel | fd5a20f | 2016-11-15 00:57:57 -0800 | [diff] [blame] | 90 | return true; |
philipel | 759e0b7 | 2016-11-30 01:32:05 -0800 | [diff] [blame] | 91 | } |
philipel | fd5a20f | 2016-11-15 00:57:57 -0800 | [diff] [blame] | 92 | |
| 93 | // The packet buffer is full, try to expand the buffer. |
| 94 | while (ExpandBufferSize() && sequence_buffer_[seq_num % size_].used) { |
| 95 | } |
| 96 | index = seq_num % size_; |
| 97 | |
Johannes Kron | bd3f305 | 2019-08-01 15:45:54 +0200 | [diff] [blame] | 98 | // Packet buffer is still full since we were unable to expand the buffer. |
philipel | 759e0b7 | 2016-11-30 01:32:05 -0800 | [diff] [blame] | 99 | if (sequence_buffer_[index].used) { |
Johannes Kron | bd3f305 | 2019-08-01 15:45:54 +0200 | [diff] [blame] | 100 | // Clear the buffer, delete payload, and return false to signal that a |
| 101 | // new keyframe is needed. |
| 102 | RTC_LOG(LS_WARNING) << "Clear PacketBuffer and request key frame."; |
| 103 | Clear(); |
philipel | 759e0b7 | 2016-11-30 01:32:05 -0800 | [diff] [blame] | 104 | delete[] packet->dataPtr; |
| 105 | packet->dataPtr = nullptr; |
philipel | fd5a20f | 2016-11-15 00:57:57 -0800 | [diff] [blame] | 106 | return false; |
philipel | 759e0b7 | 2016-11-30 01:32:05 -0800 | [diff] [blame] | 107 | } |
philipel | fd5a20f | 2016-11-15 00:57:57 -0800 | [diff] [blame] | 108 | } |
| 109 | |
Niels Möller | d5e02f0 | 2019-02-20 13:12:21 +0100 | [diff] [blame] | 110 | sequence_buffer_[index].frame_begin = packet->is_first_packet_in_frame(); |
| 111 | sequence_buffer_[index].frame_end = packet->is_last_packet_in_frame(); |
philipel | 759e0b7 | 2016-11-30 01:32:05 -0800 | [diff] [blame] | 112 | sequence_buffer_[index].seq_num = packet->seqNum; |
philipel | fd5a20f | 2016-11-15 00:57:57 -0800 | [diff] [blame] | 113 | sequence_buffer_[index].continuous = false; |
| 114 | sequence_buffer_[index].frame_created = false; |
| 115 | sequence_buffer_[index].used = true; |
philipel | 759e0b7 | 2016-11-30 01:32:05 -0800 | [diff] [blame] | 116 | data_buffer_[index] = *packet; |
| 117 | packet->dataPtr = nullptr; |
philipel | fd5a20f | 2016-11-15 00:57:57 -0800 | [diff] [blame] | 118 | |
philipel | 2c9f9f2 | 2017-06-13 02:47:28 -0700 | [diff] [blame] | 119 | UpdateMissingPackets(packet->seqNum); |
| 120 | |
philipel | 3184f8e | 2017-05-18 08:08:53 -0700 | [diff] [blame] | 121 | int64_t now_ms = clock_->TimeInMilliseconds(); |
Oskar Sundbom | 6bd3902 | 2017-11-16 10:54:49 +0100 | [diff] [blame] | 122 | last_received_packet_ms_ = now_ms; |
Niels Möller | abbc50e | 2019-04-24 09:41:16 +0200 | [diff] [blame] | 123 | if (packet->video_header.frame_type == VideoFrameType::kVideoFrameKey) |
Oskar Sundbom | 6bd3902 | 2017-11-16 10:54:49 +0100 | [diff] [blame] | 124 | last_received_keyframe_packet_ms_ = now_ms; |
philipel | 3184f8e | 2017-05-18 08:08:53 -0700 | [diff] [blame] | 125 | |
philipel | fd5a20f | 2016-11-15 00:57:57 -0800 | [diff] [blame] | 126 | found_frames = FindFrames(seq_num); |
philipel | c707ab7 | 2016-04-01 02:01:54 -0700 | [diff] [blame] | 127 | } |
| 128 | |
philipel | fd5a20f | 2016-11-15 00:57:57 -0800 | [diff] [blame] | 129 | for (std::unique_ptr<RtpFrameObject>& frame : found_frames) |
Elad Alon | b4643ad | 2019-02-22 11:19:50 +0100 | [diff] [blame] | 130 | assembled_frame_callback_->OnAssembledFrame(std::move(frame)); |
philipel | c707ab7 | 2016-04-01 02:01:54 -0700 | [diff] [blame] | 131 | |
philipel | c707ab7 | 2016-04-01 02:01:54 -0700 | [diff] [blame] | 132 | return true; |
| 133 | } |
| 134 | |
| 135 | void PacketBuffer::ClearTo(uint16_t seq_num) { |
| 136 | rtc::CritScope lock(&crit_); |
philipel | c5fb468 | 2017-08-02 04:28:57 -0700 | [diff] [blame] | 137 | // We have already cleared past this sequence number, no need to do anything. |
| 138 | if (is_cleared_to_first_seq_num_ && |
| 139 | AheadOf<uint16_t>(first_seq_num_, seq_num)) { |
| 140 | return; |
| 141 | } |
philipel | aee3e0e | 2016-11-01 11:45:34 +0100 | [diff] [blame] | 142 | |
| 143 | // If the packet buffer was cleared between a frame was created and returned. |
| 144 | if (!first_packet_received_) |
| 145 | return; |
| 146 | |
philipel | c5fb468 | 2017-08-02 04:28:57 -0700 | [diff] [blame] | 147 | // Avoid iterating over the buffer more than once by capping the number of |
| 148 | // iterations to the |size_| of the buffer. |
| 149 | ++seq_num; |
| 150 | size_t diff = ForwardDiff<uint16_t>(first_seq_num_, seq_num); |
| 151 | size_t iterations = std::min(diff, size_); |
| 152 | for (size_t i = 0; i < iterations; ++i) { |
philipel | aee3e0e | 2016-11-01 11:45:34 +0100 | [diff] [blame] | 153 | size_t index = first_seq_num_ % size_; |
philipel | c5fb468 | 2017-08-02 04:28:57 -0700 | [diff] [blame] | 154 | RTC_DCHECK_EQ(data_buffer_[index].seqNum, sequence_buffer_[index].seq_num); |
| 155 | if (AheadOf<uint16_t>(seq_num, sequence_buffer_[index].seq_num)) { |
| 156 | delete[] data_buffer_[index].dataPtr; |
| 157 | data_buffer_[index].dataPtr = nullptr; |
| 158 | sequence_buffer_[index].used = false; |
| 159 | } |
philipel | aee3e0e | 2016-11-01 11:45:34 +0100 | [diff] [blame] | 160 | ++first_seq_num_; |
philipel | c707ab7 | 2016-04-01 02:01:54 -0700 | [diff] [blame] | 161 | } |
philipel | 2c9f9f2 | 2017-06-13 02:47:28 -0700 | [diff] [blame] | 162 | |
philipel | c5fb468 | 2017-08-02 04:28:57 -0700 | [diff] [blame] | 163 | // If |diff| is larger than |iterations| it means that we don't increment |
| 164 | // |first_seq_num_| until we reach |seq_num|, so we set it here. |
| 165 | first_seq_num_ = seq_num; |
| 166 | |
| 167 | is_cleared_to_first_seq_num_ = true; |
philipel | bc5a408 | 2017-12-06 10:41:08 +0100 | [diff] [blame] | 168 | auto clear_to_it = missing_packets_.upper_bound(seq_num); |
| 169 | if (clear_to_it != missing_packets_.begin()) { |
| 170 | --clear_to_it; |
| 171 | missing_packets_.erase(missing_packets_.begin(), clear_to_it); |
| 172 | } |
philipel | c707ab7 | 2016-04-01 02:01:54 -0700 | [diff] [blame] | 173 | } |
| 174 | |
Johannes Kron | a370556 | 2019-08-26 16:37:11 +0200 | [diff] [blame] | 175 | void PacketBuffer::ClearInterval(uint16_t start_seq_num, |
| 176 | uint16_t stop_seq_num) { |
| 177 | size_t iterations = ForwardDiff<uint16_t>(start_seq_num, stop_seq_num + 1); |
| 178 | RTC_DCHECK_LE(iterations, size_); |
| 179 | uint16_t seq_num = start_seq_num; |
| 180 | for (size_t i = 0; i < iterations; ++i) { |
| 181 | size_t index = seq_num % size_; |
| 182 | RTC_DCHECK_EQ(sequence_buffer_[index].seq_num, seq_num); |
| 183 | RTC_DCHECK_EQ(sequence_buffer_[index].seq_num, data_buffer_[index].seqNum); |
| 184 | delete[] data_buffer_[index].dataPtr; |
| 185 | data_buffer_[index].dataPtr = nullptr; |
| 186 | sequence_buffer_[index].used = false; |
| 187 | |
| 188 | ++seq_num; |
| 189 | } |
| 190 | } |
| 191 | |
philipel | aee3e0e | 2016-11-01 11:45:34 +0100 | [diff] [blame] | 192 | void PacketBuffer::Clear() { |
| 193 | rtc::CritScope lock(&crit_); |
| 194 | for (size_t i = 0; i < size_; ++i) { |
| 195 | delete[] data_buffer_[i].dataPtr; |
| 196 | data_buffer_[i].dataPtr = nullptr; |
| 197 | sequence_buffer_[i].used = false; |
| 198 | } |
| 199 | |
| 200 | first_packet_received_ = false; |
| 201 | is_cleared_to_first_seq_num_ = false; |
philipel | 2c9f9f2 | 2017-06-13 02:47:28 -0700 | [diff] [blame] | 202 | last_received_packet_ms_.reset(); |
| 203 | last_received_keyframe_packet_ms_.reset(); |
| 204 | newest_inserted_seq_num_.reset(); |
| 205 | missing_packets_.clear(); |
| 206 | } |
| 207 | |
| 208 | void PacketBuffer::PaddingReceived(uint16_t seq_num) { |
| 209 | std::vector<std::unique_ptr<RtpFrameObject>> found_frames; |
| 210 | { |
| 211 | rtc::CritScope lock(&crit_); |
| 212 | UpdateMissingPackets(seq_num); |
| 213 | found_frames = FindFrames(static_cast<uint16_t>(seq_num + 1)); |
| 214 | } |
| 215 | |
| 216 | for (std::unique_ptr<RtpFrameObject>& frame : found_frames) |
Elad Alon | b4643ad | 2019-02-22 11:19:50 +0100 | [diff] [blame] | 217 | assembled_frame_callback_->OnAssembledFrame(std::move(frame)); |
philipel | 3184f8e | 2017-05-18 08:08:53 -0700 | [diff] [blame] | 218 | } |
| 219 | |
Danil Chapovalov | 0040b66 | 2018-06-18 10:48:16 +0200 | [diff] [blame] | 220 | absl::optional<int64_t> PacketBuffer::LastReceivedPacketMs() const { |
philipel | 3184f8e | 2017-05-18 08:08:53 -0700 | [diff] [blame] | 221 | rtc::CritScope lock(&crit_); |
| 222 | return last_received_packet_ms_; |
| 223 | } |
| 224 | |
Danil Chapovalov | 0040b66 | 2018-06-18 10:48:16 +0200 | [diff] [blame] | 225 | absl::optional<int64_t> PacketBuffer::LastReceivedKeyframePacketMs() const { |
philipel | 3184f8e | 2017-05-18 08:08:53 -0700 | [diff] [blame] | 226 | rtc::CritScope lock(&crit_); |
| 227 | return last_received_keyframe_packet_ms_; |
philipel | aee3e0e | 2016-11-01 11:45:34 +0100 | [diff] [blame] | 228 | } |
| 229 | |
Ilya Nikolaevskiy | d397a0d | 2018-02-21 15:57:09 +0100 | [diff] [blame] | 230 | int PacketBuffer::GetUniqueFramesSeen() const { |
| 231 | rtc::CritScope lock(&crit_); |
| 232 | return unique_frames_seen_; |
| 233 | } |
| 234 | |
philipel | c707ab7 | 2016-04-01 02:01:54 -0700 | [diff] [blame] | 235 | bool PacketBuffer::ExpandBufferSize() { |
philipel | aee3e0e | 2016-11-01 11:45:34 +0100 | [diff] [blame] | 236 | if (size_ == max_size_) { |
Mirko Bonadei | 675513b | 2017-11-09 11:09:25 +0100 | [diff] [blame] | 237 | RTC_LOG(LS_WARNING) << "PacketBuffer is already at max size (" << max_size_ |
Johannes Kron | bd3f305 | 2019-08-01 15:45:54 +0200 | [diff] [blame] | 238 | << "), failed to increase size."; |
philipel | c707ab7 | 2016-04-01 02:01:54 -0700 | [diff] [blame] | 239 | return false; |
philipel | aee3e0e | 2016-11-01 11:45:34 +0100 | [diff] [blame] | 240 | } |
philipel | c707ab7 | 2016-04-01 02:01:54 -0700 | [diff] [blame] | 241 | |
| 242 | size_t new_size = std::min(max_size_, 2 * size_); |
| 243 | std::vector<VCMPacket> new_data_buffer(new_size); |
| 244 | std::vector<ContinuityInfo> new_sequence_buffer(new_size); |
| 245 | for (size_t i = 0; i < size_; ++i) { |
| 246 | if (sequence_buffer_[i].used) { |
philipel | f413933 | 2016-04-20 10:26:34 +0200 | [diff] [blame] | 247 | size_t index = sequence_buffer_[i].seq_num % new_size; |
philipel | c707ab7 | 2016-04-01 02:01:54 -0700 | [diff] [blame] | 248 | new_sequence_buffer[index] = sequence_buffer_[i]; |
| 249 | new_data_buffer[index] = data_buffer_[i]; |
| 250 | } |
| 251 | } |
| 252 | size_ = new_size; |
| 253 | sequence_buffer_ = std::move(new_sequence_buffer); |
| 254 | data_buffer_ = std::move(new_data_buffer); |
Mirko Bonadei | 675513b | 2017-11-09 11:09:25 +0100 | [diff] [blame] | 255 | RTC_LOG(LS_INFO) << "PacketBuffer size expanded to " << new_size; |
philipel | c707ab7 | 2016-04-01 02:01:54 -0700 | [diff] [blame] | 256 | return true; |
| 257 | } |
| 258 | |
philipel | aee3e0e | 2016-11-01 11:45:34 +0100 | [diff] [blame] | 259 | bool PacketBuffer::PotentialNewFrame(uint16_t seq_num) const { |
philipel | f413933 | 2016-04-20 10:26:34 +0200 | [diff] [blame] | 260 | size_t index = seq_num % size_; |
philipel | c707ab7 | 2016-04-01 02:01:54 -0700 | [diff] [blame] | 261 | int prev_index = index > 0 ? index - 1 : size_ - 1; |
philipel | f413933 | 2016-04-20 10:26:34 +0200 | [diff] [blame] | 262 | |
philipel | c707ab7 | 2016-04-01 02:01:54 -0700 | [diff] [blame] | 263 | if (!sequence_buffer_[index].used) |
| 264 | return false; |
philipel | 2c9f9f2 | 2017-06-13 02:47:28 -0700 | [diff] [blame] | 265 | if (sequence_buffer_[index].seq_num != seq_num) |
| 266 | return false; |
philipel | f413933 | 2016-04-20 10:26:34 +0200 | [diff] [blame] | 267 | if (sequence_buffer_[index].frame_created) |
| 268 | return false; |
philipel | 20dce34 | 2016-11-28 16:14:57 +0100 | [diff] [blame] | 269 | if (sequence_buffer_[index].frame_begin) |
philipel | c707ab7 | 2016-04-01 02:01:54 -0700 | [diff] [blame] | 270 | return true; |
| 271 | if (!sequence_buffer_[prev_index].used) |
| 272 | return false; |
philipel | ea142f8 | 2017-01-11 02:01:56 -0800 | [diff] [blame] | 273 | if (sequence_buffer_[prev_index].frame_created) |
| 274 | return false; |
philipel | f413933 | 2016-04-20 10:26:34 +0200 | [diff] [blame] | 275 | if (sequence_buffer_[prev_index].seq_num != |
philipel | 2c2f34c | 2017-01-03 05:55:34 -0800 | [diff] [blame] | 276 | static_cast<uint16_t>(sequence_buffer_[index].seq_num - 1)) { |
philipel | f413933 | 2016-04-20 10:26:34 +0200 | [diff] [blame] | 277 | return false; |
philipel | aee3e0e | 2016-11-01 11:45:34 +0100 | [diff] [blame] | 278 | } |
philipel | 8b6995b | 2019-01-09 12:39:18 +0100 | [diff] [blame] | 279 | if (data_buffer_[prev_index].timestamp != data_buffer_[index].timestamp) |
| 280 | return false; |
philipel | c707ab7 | 2016-04-01 02:01:54 -0700 | [diff] [blame] | 281 | if (sequence_buffer_[prev_index].continuous) |
| 282 | return true; |
| 283 | |
| 284 | return false; |
| 285 | } |
| 286 | |
philipel | fd5a20f | 2016-11-15 00:57:57 -0800 | [diff] [blame] | 287 | std::vector<std::unique_ptr<RtpFrameObject>> PacketBuffer::FindFrames( |
| 288 | uint16_t seq_num) { |
| 289 | std::vector<std::unique_ptr<RtpFrameObject>> found_frames; |
philipel | 5391071 | 2017-05-18 02:24:40 -0700 | [diff] [blame] | 290 | for (size_t i = 0; i < size_ && PotentialNewFrame(seq_num); ++i) { |
philipel | aee3e0e | 2016-11-01 11:45:34 +0100 | [diff] [blame] | 291 | size_t index = seq_num % size_; |
philipel | c707ab7 | 2016-04-01 02:01:54 -0700 | [diff] [blame] | 292 | sequence_buffer_[index].continuous = true; |
| 293 | |
philipel | f413933 | 2016-04-20 10:26:34 +0200 | [diff] [blame] | 294 | // If all packets of the frame is continuous, find the first packet of the |
| 295 | // frame and create an RtpFrameObject. |
philipel | c707ab7 | 2016-04-01 02:01:54 -0700 | [diff] [blame] | 296 | if (sequence_buffer_[index].frame_end) { |
philipel | 5ceaaae | 2016-05-24 10:20:47 +0200 | [diff] [blame] | 297 | size_t frame_size = 0; |
| 298 | int max_nack_count = -1; |
philipel | c707ab7 | 2016-04-01 02:01:54 -0700 | [diff] [blame] | 299 | uint16_t start_seq_num = seq_num; |
Chen Xing | f00bf42 | 2019-06-20 10:05:55 +0200 | [diff] [blame] | 300 | int64_t min_recv_time = data_buffer_[index].packet_info.receive_time_ms(); |
| 301 | int64_t max_recv_time = data_buffer_[index].packet_info.receive_time_ms(); |
| 302 | RtpPacketInfos::vector_type packet_infos; |
philipel | f413933 | 2016-04-20 10:26:34 +0200 | [diff] [blame] | 303 | |
philipel | 5ceaaae | 2016-05-24 10:20:47 +0200 | [diff] [blame] | 304 | // Find the start index by searching backward until the packet with |
| 305 | // the |frame_begin| flag is set. |
| 306 | int start_index = index; |
philipel | 227f8b9 | 2017-08-04 06:39:31 -0700 | [diff] [blame] | 307 | size_t tested_packets = 0; |
philipel | 8c61924 | 2017-02-02 08:51:29 -0800 | [diff] [blame] | 308 | int64_t frame_timestamp = data_buffer_[start_index].timestamp; |
philipel | 5391071 | 2017-05-18 02:24:40 -0700 | [diff] [blame] | 309 | |
Rasmus Brandt | 88f080a | 2017-11-02 14:28:06 +0100 | [diff] [blame] | 310 | // Identify H.264 keyframes by means of SPS, PPS, and IDR. |
Niels Möller | d5e02f0 | 2019-02-20 13:12:21 +0100 | [diff] [blame] | 311 | bool is_h264 = data_buffer_[start_index].codec() == kVideoCodecH264; |
Rasmus Brandt | 88f080a | 2017-11-02 14:28:06 +0100 | [diff] [blame] | 312 | bool has_h264_sps = false; |
| 313 | bool has_h264_pps = false; |
| 314 | bool has_h264_idr = false; |
| 315 | bool is_h264_keyframe = false; |
| 316 | |
philipel | 227f8b9 | 2017-08-04 06:39:31 -0700 | [diff] [blame] | 317 | while (true) { |
| 318 | ++tested_packets; |
philipel | 5ceaaae | 2016-05-24 10:20:47 +0200 | [diff] [blame] | 319 | frame_size += data_buffer_[start_index].sizeBytes; |
philipel | fd5a20f | 2016-11-15 00:57:57 -0800 | [diff] [blame] | 320 | max_nack_count = |
| 321 | std::max(max_nack_count, data_buffer_[start_index].timesNacked); |
philipel | f413933 | 2016-04-20 10:26:34 +0200 | [diff] [blame] | 322 | sequence_buffer_[start_index].frame_created = true; |
philipel | 5ceaaae | 2016-05-24 10:20:47 +0200 | [diff] [blame] | 323 | |
Ilya Nikolaevskiy | 4348ce2 | 2018-12-07 16:26:56 +0100 | [diff] [blame] | 324 | min_recv_time = |
Chen Xing | f00bf42 | 2019-06-20 10:05:55 +0200 | [diff] [blame] | 325 | std::min(min_recv_time, |
| 326 | data_buffer_[start_index].packet_info.receive_time_ms()); |
Ilya Nikolaevskiy | 4348ce2 | 2018-12-07 16:26:56 +0100 | [diff] [blame] | 327 | max_recv_time = |
Chen Xing | f00bf42 | 2019-06-20 10:05:55 +0200 | [diff] [blame] | 328 | std::max(max_recv_time, |
| 329 | data_buffer_[start_index].packet_info.receive_time_ms()); |
| 330 | |
| 331 | // Should use |push_front()| since the loop traverses backwards. But |
| 332 | // it's too inefficient to do so on a vector so we'll instead fix the |
| 333 | // order afterwards. |
| 334 | packet_infos.push_back(data_buffer_[start_index].packet_info); |
Ilya Nikolaevskiy | 4348ce2 | 2018-12-07 16:26:56 +0100 | [diff] [blame] | 335 | |
philipel | 8c61924 | 2017-02-02 08:51:29 -0800 | [diff] [blame] | 336 | if (!is_h264 && sequence_buffer_[start_index].frame_begin) |
philipel | 5ceaaae | 2016-05-24 10:20:47 +0200 | [diff] [blame] | 337 | break; |
| 338 | |
philipel | 2c9f9f2 | 2017-06-13 02:47:28 -0700 | [diff] [blame] | 339 | if (is_h264 && !is_h264_keyframe) { |
philipel | 7d745e5 | 2018-08-02 14:03:53 +0200 | [diff] [blame] | 340 | const auto* h264_header = absl::get_if<RTPVideoHeaderH264>( |
| 341 | &data_buffer_[start_index].video_header.video_type_header); |
| 342 | if (!h264_header || h264_header->nalus_length >= kMaxNalusPerPacket) |
philipel | 09133af | 2018-05-17 14:11:09 +0200 | [diff] [blame] | 343 | return found_frames; |
| 344 | |
philipel | 7d745e5 | 2018-08-02 14:03:53 +0200 | [diff] [blame] | 345 | for (size_t j = 0; j < h264_header->nalus_length; ++j) { |
| 346 | if (h264_header->nalus[j].type == H264::NaluType::kSps) { |
Rasmus Brandt | 88f080a | 2017-11-02 14:28:06 +0100 | [diff] [blame] | 347 | has_h264_sps = true; |
philipel | 7d745e5 | 2018-08-02 14:03:53 +0200 | [diff] [blame] | 348 | } else if (h264_header->nalus[j].type == H264::NaluType::kPps) { |
Rasmus Brandt | 88f080a | 2017-11-02 14:28:06 +0100 | [diff] [blame] | 349 | has_h264_pps = true; |
philipel | 7d745e5 | 2018-08-02 14:03:53 +0200 | [diff] [blame] | 350 | } else if (h264_header->nalus[j].type == H264::NaluType::kIdr) { |
Rasmus Brandt | 88f080a | 2017-11-02 14:28:06 +0100 | [diff] [blame] | 351 | has_h264_idr = true; |
philipel | 2c9f9f2 | 2017-06-13 02:47:28 -0700 | [diff] [blame] | 352 | } |
| 353 | } |
Rasmus Brandt | 88f080a | 2017-11-02 14:28:06 +0100 | [diff] [blame] | 354 | if ((sps_pps_idr_is_h264_keyframe_ && has_h264_idr && has_h264_sps && |
| 355 | has_h264_pps) || |
| 356 | (!sps_pps_idr_is_h264_keyframe_ && has_h264_idr)) { |
| 357 | is_h264_keyframe = true; |
| 358 | } |
philipel | 2c9f9f2 | 2017-06-13 02:47:28 -0700 | [diff] [blame] | 359 | } |
| 360 | |
philipel | 227f8b9 | 2017-08-04 06:39:31 -0700 | [diff] [blame] | 361 | if (tested_packets == size_) |
| 362 | break; |
| 363 | |
philipel | f413933 | 2016-04-20 10:26:34 +0200 | [diff] [blame] | 364 | start_index = start_index > 0 ? start_index - 1 : size_ - 1; |
philipel | 8c61924 | 2017-02-02 08:51:29 -0800 | [diff] [blame] | 365 | |
| 366 | // In the case of H264 we don't have a frame_begin bit (yes, |
| 367 | // |frame_begin| might be set to true but that is a lie). So instead |
| 368 | // we traverese backwards as long as we have a previous packet and |
| 369 | // the timestamp of that packet is the same as this one. This may cause |
| 370 | // the PacketBuffer to hand out incomplete frames. |
| 371 | // See: https://bugs.chromium.org/p/webrtc/issues/detail?id=7106 |
philipel | 5391071 | 2017-05-18 02:24:40 -0700 | [diff] [blame] | 372 | if (is_h264 && |
philipel | 8c61924 | 2017-02-02 08:51:29 -0800 | [diff] [blame] | 373 | (!sequence_buffer_[start_index].used || |
| 374 | data_buffer_[start_index].timestamp != frame_timestamp)) { |
| 375 | break; |
| 376 | } |
| 377 | |
| 378 | --start_seq_num; |
philipel | c707ab7 | 2016-04-01 02:01:54 -0700 | [diff] [blame] | 379 | } |
| 380 | |
Chen Xing | f00bf42 | 2019-06-20 10:05:55 +0200 | [diff] [blame] | 381 | // Fix the order since the packet-finding loop traverses backwards. |
| 382 | std::reverse(packet_infos.begin(), packet_infos.end()); |
| 383 | |
Rasmus Brandt | 88f080a | 2017-11-02 14:28:06 +0100 | [diff] [blame] | 384 | if (is_h264) { |
| 385 | // Warn if this is an unsafe frame. |
| 386 | if (has_h264_idr && (!has_h264_sps || !has_h264_pps)) { |
Jonas Olsson | fc50110 | 2018-06-15 14:24:10 +0200 | [diff] [blame] | 387 | RTC_LOG(LS_WARNING) |
| 388 | << "Received H.264-IDR frame " |
| 389 | << "(SPS: " << has_h264_sps << ", PPS: " << has_h264_pps |
| 390 | << "). Treating as " |
| 391 | << (sps_pps_idr_is_h264_keyframe_ ? "delta" : "key") |
| 392 | << " frame since WebRTC-SpsPpsIdrIsH264Keyframe is " |
| 393 | << (sps_pps_idr_is_h264_keyframe_ ? "enabled." : "disabled"); |
philipel | 2c9f9f2 | 2017-06-13 02:47:28 -0700 | [diff] [blame] | 394 | } |
| 395 | |
Rasmus Brandt | 88f080a | 2017-11-02 14:28:06 +0100 | [diff] [blame] | 396 | // Now that we have decided whether to treat this frame as a key frame |
| 397 | // or delta frame in the frame buffer, we update the field that |
| 398 | // determines if the RtpFrameObject is a key frame or delta frame. |
| 399 | const size_t first_packet_index = start_seq_num % size_; |
| 400 | RTC_CHECK_LT(first_packet_index, size_); |
| 401 | if (is_h264_keyframe) { |
Niels Möller | abbc50e | 2019-04-24 09:41:16 +0200 | [diff] [blame] | 402 | data_buffer_[first_packet_index].video_header.frame_type = |
Niels Möller | 8f7ce22 | 2019-03-21 15:43:58 +0100 | [diff] [blame] | 403 | VideoFrameType::kVideoFrameKey; |
Rasmus Brandt | 88f080a | 2017-11-02 14:28:06 +0100 | [diff] [blame] | 404 | } else { |
Niels Möller | abbc50e | 2019-04-24 09:41:16 +0200 | [diff] [blame] | 405 | data_buffer_[first_packet_index].video_header.frame_type = |
Niels Möller | 8f7ce22 | 2019-03-21 15:43:58 +0100 | [diff] [blame] | 406 | VideoFrameType::kVideoFrameDelta; |
Rasmus Brandt | 88f080a | 2017-11-02 14:28:06 +0100 | [diff] [blame] | 407 | } |
| 408 | |
Johnny Lee | bc7f41b | 2019-05-01 14:41:32 -0400 | [diff] [blame] | 409 | // With IPPP, if this is not a keyframe, make sure there are no gaps |
| 410 | // in the packet sequence numbers up until this point. |
| 411 | const uint8_t h264tid = |
| 412 | data_buffer_[start_index].video_header.frame_marking.temporal_id; |
Jonas Olsson | a4d8737 | 2019-07-05 19:08:33 +0200 | [diff] [blame] | 413 | if (h264tid == kNoTemporalIdx && !is_h264_keyframe && |
| 414 | missing_packets_.upper_bound(start_seq_num) != |
| 415 | missing_packets_.begin()) { |
Rasmus Brandt | 88f080a | 2017-11-02 14:28:06 +0100 | [diff] [blame] | 416 | uint16_t stop_index = (index + 1) % size_; |
| 417 | while (start_index != stop_index) { |
| 418 | sequence_buffer_[start_index].frame_created = false; |
| 419 | start_index = (start_index + 1) % size_; |
| 420 | } |
| 421 | |
| 422 | return found_frames; |
| 423 | } |
philipel | 2c9f9f2 | 2017-06-13 02:47:28 -0700 | [diff] [blame] | 424 | } |
| 425 | |
| 426 | missing_packets_.erase(missing_packets_.begin(), |
| 427 | missing_packets_.upper_bound(seq_num)); |
| 428 | |
philipel | 85d5c19 | 2019-09-25 17:15:37 +0200 | [diff] [blame] | 429 | const VCMPacket* first_packet = GetPacket(start_seq_num); |
| 430 | const VCMPacket* last_packet = GetPacket(seq_num); |
philipel | 76161f7 | 2019-09-19 11:22:22 +0200 | [diff] [blame] | 431 | auto frame = std::make_unique<RtpFrameObject>( |
philipel | 85d5c19 | 2019-09-25 17:15:37 +0200 | [diff] [blame] | 432 | start_seq_num, seq_num, last_packet->markerBit, max_nack_count, |
| 433 | min_recv_time, max_recv_time, first_packet->timestamp, |
| 434 | first_packet->ntp_time_ms_, last_packet->video_header.video_timing, |
| 435 | first_packet->payloadType, first_packet->codec(), |
| 436 | last_packet->video_header.rotation, |
| 437 | last_packet->video_header.content_type, first_packet->video_header, |
| 438 | last_packet->video_header.color_space, |
| 439 | first_packet->generic_descriptor, |
| 440 | RtpPacketInfos(std::move(packet_infos)), |
philipel | b5e4785 | 2019-09-20 11:30:12 +0200 | [diff] [blame] | 441 | GetEncodedImageBuffer(frame_size, start_seq_num, seq_num)); |
| 442 | |
philipel | 76161f7 | 2019-09-19 11:22:22 +0200 | [diff] [blame] | 443 | found_frames.emplace_back(std::move(frame)); |
| 444 | |
Johannes Kron | a370556 | 2019-08-26 16:37:11 +0200 | [diff] [blame] | 445 | ClearInterval(start_seq_num, seq_num); |
philipel | c707ab7 | 2016-04-01 02:01:54 -0700 | [diff] [blame] | 446 | } |
philipel | c707ab7 | 2016-04-01 02:01:54 -0700 | [diff] [blame] | 447 | ++seq_num; |
| 448 | } |
philipel | fd5a20f | 2016-11-15 00:57:57 -0800 | [diff] [blame] | 449 | return found_frames; |
philipel | c707ab7 | 2016-04-01 02:01:54 -0700 | [diff] [blame] | 450 | } |
| 451 | |
philipel | b5e4785 | 2019-09-20 11:30:12 +0200 | [diff] [blame] | 452 | rtc::scoped_refptr<EncodedImageBuffer> PacketBuffer::GetEncodedImageBuffer( |
| 453 | size_t frame_size, |
| 454 | uint16_t first_seq_num, |
| 455 | uint16_t last_seq_num) { |
| 456 | size_t index = first_seq_num % size_; |
| 457 | size_t end = (last_seq_num + 1) % size_; |
philipel | c707ab7 | 2016-04-01 02:01:54 -0700 | [diff] [blame] | 458 | |
philipel | b5e4785 | 2019-09-20 11:30:12 +0200 | [diff] [blame] | 459 | auto buffer = EncodedImageBuffer::Create(frame_size); |
| 460 | size_t offset = 0; |
philipel | 227f8b9 | 2017-08-04 06:39:31 -0700 | [diff] [blame] | 461 | |
| 462 | do { |
philipel | b5e4785 | 2019-09-20 11:30:12 +0200 | [diff] [blame] | 463 | RTC_DCHECK(sequence_buffer_[index].used); |
philipel | c707ab7 | 2016-04-01 02:01:54 -0700 | [diff] [blame] | 464 | |
philipel | c18f1d7 | 2017-08-02 04:18:02 -0700 | [diff] [blame] | 465 | size_t length = data_buffer_[index].sizeBytes; |
philipel | b5e4785 | 2019-09-20 11:30:12 +0200 | [diff] [blame] | 466 | RTC_CHECK_LE(offset + length, buffer->size()); |
| 467 | memcpy(buffer->data() + offset, data_buffer_[index].dataPtr, length); |
| 468 | offset += length; |
philipel | 227f8b9 | 2017-08-04 06:39:31 -0700 | [diff] [blame] | 469 | |
philipel | c707ab7 | 2016-04-01 02:01:54 -0700 | [diff] [blame] | 470 | index = (index + 1) % size_; |
philipel | 227f8b9 | 2017-08-04 06:39:31 -0700 | [diff] [blame] | 471 | } while (index != end); |
| 472 | |
philipel | b5e4785 | 2019-09-20 11:30:12 +0200 | [diff] [blame] | 473 | return buffer; |
philipel | c707ab7 | 2016-04-01 02:01:54 -0700 | [diff] [blame] | 474 | } |
| 475 | |
philipel | 02447bc | 2016-05-13 06:01:03 -0700 | [diff] [blame] | 476 | VCMPacket* PacketBuffer::GetPacket(uint16_t seq_num) { |
philipel | 02447bc | 2016-05-13 06:01:03 -0700 | [diff] [blame] | 477 | size_t index = seq_num % size_; |
| 478 | if (!sequence_buffer_[index].used || |
| 479 | seq_num != sequence_buffer_[index].seq_num) { |
| 480 | return nullptr; |
philipel | f413933 | 2016-04-20 10:26:34 +0200 | [diff] [blame] | 481 | } |
philipel | 02447bc | 2016-05-13 06:01:03 -0700 | [diff] [blame] | 482 | return &data_buffer_[index]; |
philipel | f413933 | 2016-04-20 10:26:34 +0200 | [diff] [blame] | 483 | } |
| 484 | |
philipel | 2c9f9f2 | 2017-06-13 02:47:28 -0700 | [diff] [blame] | 485 | void PacketBuffer::UpdateMissingPackets(uint16_t seq_num) { |
| 486 | if (!newest_inserted_seq_num_) |
Oskar Sundbom | 6bd3902 | 2017-11-16 10:54:49 +0100 | [diff] [blame] | 487 | newest_inserted_seq_num_ = seq_num; |
philipel | 2c9f9f2 | 2017-06-13 02:47:28 -0700 | [diff] [blame] | 488 | |
| 489 | const int kMaxPaddingAge = 1000; |
| 490 | if (AheadOf(seq_num, *newest_inserted_seq_num_)) { |
| 491 | uint16_t old_seq_num = seq_num - kMaxPaddingAge; |
| 492 | auto erase_to = missing_packets_.lower_bound(old_seq_num); |
| 493 | missing_packets_.erase(missing_packets_.begin(), erase_to); |
| 494 | |
| 495 | // Guard against inserting a large amount of missing packets if there is a |
| 496 | // jump in the sequence number. |
| 497 | if (AheadOf(old_seq_num, *newest_inserted_seq_num_)) |
| 498 | *newest_inserted_seq_num_ = old_seq_num; |
| 499 | |
| 500 | ++*newest_inserted_seq_num_; |
| 501 | while (AheadOf(seq_num, *newest_inserted_seq_num_)) { |
| 502 | missing_packets_.insert(*newest_inserted_seq_num_); |
| 503 | ++*newest_inserted_seq_num_; |
| 504 | } |
| 505 | } else { |
| 506 | missing_packets_.erase(seq_num); |
| 507 | } |
| 508 | } |
| 509 | |
Ilya Nikolaevskiy | d397a0d | 2018-02-21 15:57:09 +0100 | [diff] [blame] | 510 | void PacketBuffer::OnTimestampReceived(uint32_t rtp_timestamp) { |
| 511 | const size_t kMaxTimestampsHistory = 1000; |
| 512 | if (rtp_timestamps_history_set_.insert(rtp_timestamp).second) { |
| 513 | rtp_timestamps_history_queue_.push(rtp_timestamp); |
| 514 | ++unique_frames_seen_; |
| 515 | if (rtp_timestamps_history_set_.size() > kMaxTimestampsHistory) { |
| 516 | uint32_t discarded_timestamp = rtp_timestamps_history_queue_.front(); |
| 517 | rtp_timestamps_history_set_.erase(discarded_timestamp); |
| 518 | rtp_timestamps_history_queue_.pop(); |
| 519 | } |
| 520 | } |
| 521 | } |
| 522 | |
philipel | c707ab7 | 2016-04-01 02:01:54 -0700 | [diff] [blame] | 523 | } // namespace video_coding |
| 524 | } // namespace webrtc |