philipel | c707ab7 | 2016-04-01 02:01:54 -0700 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. |
| 3 | * |
| 4 | * Use of this source code is governed by a BSD-style license |
| 5 | * that can be found in the LICENSE file in the root of the source |
| 6 | * tree. An additional intellectual property rights grant can be found |
| 7 | * in the file PATENTS. All contributing project authors may |
| 8 | * be found in the AUTHORS file in the root of the source tree. |
| 9 | */ |
| 10 | |
Mirko Bonadei | 92ea95e | 2017-09-15 06:47:31 +0200 | [diff] [blame] | 11 | #include "modules/video_coding/frame_object.h" |
Rasmus Brandt | 58b7291 | 2017-10-25 11:09:23 +0200 | [diff] [blame] | 12 | |
| 13 | #include <sstream> |
| 14 | |
Mirko Bonadei | 92ea95e | 2017-09-15 06:47:31 +0200 | [diff] [blame] | 15 | #include "common_video/h264/h264_common.h" |
| 16 | #include "modules/video_coding/packet_buffer.h" |
| 17 | #include "rtc_base/checks.h" |
Rasmus Brandt | 58b7291 | 2017-10-25 11:09:23 +0200 | [diff] [blame] | 18 | #include "rtc_base/logging.h" |
Rasmus Brandt | edf4ff7 | 2017-10-24 10:07:48 +0200 | [diff] [blame] | 19 | #include "system_wrappers/include/field_trial.h" |
philipel | c707ab7 | 2016-04-01 02:01:54 -0700 | [diff] [blame] | 20 | |
| 21 | namespace webrtc { |
| 22 | namespace video_coding { |
| 23 | |
philipel | a105987 | 2016-05-09 11:41:48 +0200 | [diff] [blame] | 24 | FrameObject::FrameObject() |
| 25 | : picture_id(0), |
| 26 | spatial_layer(0), |
philipel | be7a9e5 | 2016-05-19 12:19:35 +0200 | [diff] [blame] | 27 | timestamp(0), |
philipel | a105987 | 2016-05-09 11:41:48 +0200 | [diff] [blame] | 28 | num_references(0), |
| 29 | inter_layer_predicted(false) {} |
| 30 | |
philipel | c707ab7 | 2016-04-01 02:01:54 -0700 | [diff] [blame] | 31 | RtpFrameObject::RtpFrameObject(PacketBuffer* packet_buffer, |
philipel | 02447bc | 2016-05-13 06:01:03 -0700 | [diff] [blame] | 32 | uint16_t first_seq_num, |
philipel | 5ceaaae | 2016-05-24 10:20:47 +0200 | [diff] [blame] | 33 | uint16_t last_seq_num, |
| 34 | size_t frame_size, |
philipel | b4d3108 | 2016-07-11 08:46:29 -0700 | [diff] [blame] | 35 | int times_nacked, |
| 36 | int64_t received_time) |
philipel | c707ab7 | 2016-04-01 02:01:54 -0700 | [diff] [blame] | 37 | : packet_buffer_(packet_buffer), |
philipel | 02447bc | 2016-05-13 06:01:03 -0700 | [diff] [blame] | 38 | first_seq_num_(first_seq_num), |
philipel | 5ceaaae | 2016-05-24 10:20:47 +0200 | [diff] [blame] | 39 | last_seq_num_(last_seq_num), |
sprang | ba050a6 | 2017-08-18 02:51:12 -0700 | [diff] [blame] | 40 | timestamp_(0), |
philipel | b4d3108 | 2016-07-11 08:46:29 -0700 | [diff] [blame] | 41 | received_time_(received_time), |
philipel | 5ceaaae | 2016-05-24 10:20:47 +0200 | [diff] [blame] | 42 | times_nacked_(times_nacked) { |
philipel | 266f0a4 | 2016-11-28 08:49:07 -0800 | [diff] [blame] | 43 | VCMPacket* first_packet = packet_buffer_->GetPacket(first_seq_num); |
philipel | 7d79e63 | 2017-05-23 08:19:11 -0700 | [diff] [blame] | 44 | RTC_CHECK(first_packet); |
philipel | 3692845 | 2016-11-07 10:42:36 +0100 | [diff] [blame] | 45 | |
philipel | 266f0a4 | 2016-11-28 08:49:07 -0800 | [diff] [blame] | 46 | // RtpFrameObject members |
| 47 | frame_type_ = first_packet->frameType; |
| 48 | codec_type_ = first_packet->codec; |
philipel | 3692845 | 2016-11-07 10:42:36 +0100 | [diff] [blame] | 49 | |
philipel | 266f0a4 | 2016-11-28 08:49:07 -0800 | [diff] [blame] | 50 | // TODO(philipel): Remove when encoded image is replaced by FrameObject. |
| 51 | // VCMEncodedFrame members |
| 52 | CopyCodecSpecific(&first_packet->video_header); |
| 53 | _completeFrame = true; |
| 54 | _payloadType = first_packet->payloadType; |
| 55 | _timeStamp = first_packet->timestamp; |
| 56 | ntp_time_ms_ = first_packet->ntp_time_ms_; |
philipel | 3692845 | 2016-11-07 10:42:36 +0100 | [diff] [blame] | 57 | |
gnish | b2a318b | 2017-05-10 09:21:33 -0700 | [diff] [blame] | 58 | // Setting frame's playout delays to the same values |
| 59 | // as of the first packet's. |
| 60 | SetPlayoutDelay(first_packet->video_header.playout_delay); |
| 61 | |
philipel | 266f0a4 | 2016-11-28 08:49:07 -0800 | [diff] [blame] | 62 | // Since FFmpeg use an optimized bitstream reader that reads in chunks of |
| 63 | // 32/64 bits we have to add at least that much padding to the buffer |
| 64 | // to make sure the decoder doesn't read out of bounds. |
| 65 | // NOTE! EncodedImage::_size is the size of the buffer (think capacity of |
| 66 | // an std::vector) and EncodedImage::_length is the actual size of |
| 67 | // the bitstream (think size of an std::vector). |
| 68 | if (codec_type_ == kVideoCodecH264) |
| 69 | _size = frame_size + EncodedImage::kBufferPaddingBytesH264; |
| 70 | else |
| 71 | _size = frame_size; |
philipel | 552866c | 2016-07-04 16:18:55 +0200 | [diff] [blame] | 72 | |
philipel | 266f0a4 | 2016-11-28 08:49:07 -0800 | [diff] [blame] | 73 | _buffer = new uint8_t[_size]; |
| 74 | _length = frame_size; |
philipel | e87c876 | 2017-05-19 06:38:50 -0700 | [diff] [blame] | 75 | |
| 76 | // For H264 frames we can't determine the frame type by just looking at the |
Rasmus Brandt | edf4ff7 | 2017-10-24 10:07:48 +0200 | [diff] [blame] | 77 | // first packet. Instead we consider the frame to be a keyframe if it contains |
| 78 | // an IDR, and SPS/PPS if the field trial is set. |
philipel | e87c876 | 2017-05-19 06:38:50 -0700 | [diff] [blame] | 79 | if (codec_type_ == kVideoCodecH264) { |
| 80 | _frameType = kVideoFrameDelta; |
| 81 | frame_type_ = kVideoFrameDelta; |
Rasmus Brandt | edf4ff7 | 2017-10-24 10:07:48 +0200 | [diff] [blame] | 82 | bool contains_sps = false; |
| 83 | bool contains_pps = false; |
| 84 | bool contains_idr = false; |
philipel | e87c876 | 2017-05-19 06:38:50 -0700 | [diff] [blame] | 85 | for (uint16_t seq_num = first_seq_num; |
philipel | 7d79e63 | 2017-05-23 08:19:11 -0700 | [diff] [blame] | 86 | seq_num != static_cast<uint16_t>(last_seq_num + 1) && |
| 87 | _frameType == kVideoFrameDelta; |
philipel | e87c876 | 2017-05-19 06:38:50 -0700 | [diff] [blame] | 88 | ++seq_num) { |
| 89 | VCMPacket* packet = packet_buffer_->GetPacket(seq_num); |
philipel | 7d79e63 | 2017-05-23 08:19:11 -0700 | [diff] [blame] | 90 | RTC_CHECK(packet); |
philipel | e87c876 | 2017-05-19 06:38:50 -0700 | [diff] [blame] | 91 | const RTPVideoHeaderH264& header = packet->video_header.codecHeader.H264; |
| 92 | for (size_t i = 0; i < header.nalus_length; ++i) { |
Rasmus Brandt | edf4ff7 | 2017-10-24 10:07:48 +0200 | [diff] [blame] | 93 | if (header.nalus[i].type == H264::NaluType::kSps) { |
| 94 | contains_sps = true; |
| 95 | } else if (header.nalus[i].type == H264::NaluType::kPps) { |
| 96 | contains_pps = true; |
| 97 | } else if (header.nalus[i].type == H264::NaluType::kIdr) { |
| 98 | contains_idr = true; |
philipel | e87c876 | 2017-05-19 06:38:50 -0700 | [diff] [blame] | 99 | } |
| 100 | } |
| 101 | } |
Rasmus Brandt | edf4ff7 | 2017-10-24 10:07:48 +0200 | [diff] [blame] | 102 | const bool sps_pps_idr_is_keyframe = |
| 103 | field_trial::IsEnabled("WebRTC-SpsPpsIdrIsH264Keyframe"); |
| 104 | if ((sps_pps_idr_is_keyframe && contains_idr && contains_sps && |
| 105 | contains_pps) || |
| 106 | (!sps_pps_idr_is_keyframe && contains_idr)) { |
| 107 | _frameType = kVideoFrameKey; |
| 108 | frame_type_ = kVideoFrameKey; |
| 109 | } |
Rasmus Brandt | 58b7291 | 2017-10-25 11:09:23 +0200 | [diff] [blame] | 110 | if (contains_idr && (!contains_sps || !contains_pps)) { |
| 111 | std::stringstream ss; |
| 112 | ss << "Received H.264-IDR frame " |
| 113 | << "(SPS: " << contains_sps << ", PPS: " << contains_pps << "). "; |
| 114 | if (sps_pps_idr_is_keyframe) { |
| 115 | ss << "Treating as delta frame since WebRTC-SpsPpsIdrIsH264Keyframe is " |
| 116 | "enabled."; |
| 117 | } else { |
| 118 | ss << "Treating as key frame since WebRTC-SpsPpsIdrIsH264Keyframe is " |
| 119 | "disabled."; |
| 120 | } |
| 121 | LOG(LS_WARNING) << ss.str(); |
| 122 | } |
philipel | e87c876 | 2017-05-19 06:38:50 -0700 | [diff] [blame] | 123 | } else { |
| 124 | _frameType = first_packet->frameType; |
| 125 | frame_type_ = first_packet->frameType; |
| 126 | } |
| 127 | |
philipel | 227f8b9 | 2017-08-04 06:39:31 -0700 | [diff] [blame] | 128 | bool bitstream_copied = GetBitstream(_buffer); |
| 129 | RTC_DCHECK(bitstream_copied); |
philipel | 59e99b7 | 2017-01-12 03:26:04 -0800 | [diff] [blame] | 130 | _encodedWidth = first_packet->width; |
| 131 | _encodedHeight = first_packet->height; |
philipel | 266f0a4 | 2016-11-28 08:49:07 -0800 | [diff] [blame] | 132 | |
| 133 | // FrameObject members |
| 134 | timestamp = first_packet->timestamp; |
| 135 | |
| 136 | VCMPacket* last_packet = packet_buffer_->GetPacket(last_seq_num); |
kwiberg | ee89e78 | 2017-08-09 17:22:01 -0700 | [diff] [blame] | 137 | RTC_CHECK(last_packet); |
| 138 | RTC_CHECK(last_packet->markerBit); |
philipel | 266f0a4 | 2016-11-28 08:49:07 -0800 | [diff] [blame] | 139 | // http://www.etsi.org/deliver/etsi_ts/126100_126199/126114/12.07.00_60/ |
| 140 | // ts_126114v120700p.pdf Section 7.4.5. |
| 141 | // The MTSI client shall add the payload bytes as defined in this clause |
| 142 | // onto the last RTP packet in each group of packets which make up a key |
| 143 | // frame (I-frame or IDR frame in H.264 (AVC), or an IRAP picture in H.265 |
| 144 | // (HEVC)). |
| 145 | rotation_ = last_packet->video_header.rotation; |
| 146 | _rotation_set = true; |
ilnik | 00d802b | 2017-04-11 10:34:31 -0700 | [diff] [blame] | 147 | content_type_ = last_packet->video_header.content_type; |
sprang | ba050a6 | 2017-08-18 02:51:12 -0700 | [diff] [blame] | 148 | if (last_packet->video_header.video_timing.flags != |
| 149 | TimingFrameFlags::kInvalid) { |
ilnik | 04f4d12 | 2017-06-19 07:18:55 -0700 | [diff] [blame] | 150 | // ntp_time_ms_ may be -1 if not estimated yet. This is not a problem, |
| 151 | // as this will be dealt with at the time of reporting. |
ilnik | 04f4d12 | 2017-06-19 07:18:55 -0700 | [diff] [blame] | 152 | timing_.encode_start_ms = |
| 153 | ntp_time_ms_ + |
| 154 | last_packet->video_header.video_timing.encode_start_delta_ms; |
| 155 | timing_.encode_finish_ms = |
| 156 | ntp_time_ms_ + |
| 157 | last_packet->video_header.video_timing.encode_finish_delta_ms; |
| 158 | timing_.packetization_finish_ms = |
| 159 | ntp_time_ms_ + |
| 160 | last_packet->video_header.video_timing.packetization_finish_delta_ms; |
| 161 | timing_.pacer_exit_ms = |
| 162 | ntp_time_ms_ + |
| 163 | last_packet->video_header.video_timing.pacer_exit_delta_ms; |
| 164 | timing_.network_timestamp_ms = |
| 165 | ntp_time_ms_ + |
Danil Chapovalov | 996eb9e | 2017-10-30 17:14:41 +0100 | [diff] [blame] | 166 | last_packet->video_header.video_timing.network_timestamp_delta_ms; |
ilnik | 04f4d12 | 2017-06-19 07:18:55 -0700 | [diff] [blame] | 167 | timing_.network2_timestamp_ms = |
| 168 | ntp_time_ms_ + |
Danil Chapovalov | 996eb9e | 2017-10-30 17:14:41 +0100 | [diff] [blame] | 169 | last_packet->video_header.video_timing.network2_timestamp_delta_ms; |
ilnik | 04f4d12 | 2017-06-19 07:18:55 -0700 | [diff] [blame] | 170 | |
| 171 | timing_.receive_start_ms = first_packet->receive_time_ms; |
| 172 | timing_.receive_finish_ms = last_packet->receive_time_ms; |
ilnik | 04f4d12 | 2017-06-19 07:18:55 -0700 | [diff] [blame] | 173 | } |
sprang | ba050a6 | 2017-08-18 02:51:12 -0700 | [diff] [blame] | 174 | timing_.flags = last_packet->video_header.video_timing.flags; |
philipel | 02447bc | 2016-05-13 06:01:03 -0700 | [diff] [blame] | 175 | } |
philipel | c707ab7 | 2016-04-01 02:01:54 -0700 | [diff] [blame] | 176 | |
| 177 | RtpFrameObject::~RtpFrameObject() { |
| 178 | packet_buffer_->ReturnFrame(this); |
| 179 | } |
| 180 | |
philipel | f413933 | 2016-04-20 10:26:34 +0200 | [diff] [blame] | 181 | uint16_t RtpFrameObject::first_seq_num() const { |
philipel | 02447bc | 2016-05-13 06:01:03 -0700 | [diff] [blame] | 182 | return first_seq_num_; |
philipel | c707ab7 | 2016-04-01 02:01:54 -0700 | [diff] [blame] | 183 | } |
| 184 | |
philipel | f413933 | 2016-04-20 10:26:34 +0200 | [diff] [blame] | 185 | uint16_t RtpFrameObject::last_seq_num() const { |
philipel | 02447bc | 2016-05-13 06:01:03 -0700 | [diff] [blame] | 186 | return last_seq_num_; |
| 187 | } |
| 188 | |
philipel | 5ceaaae | 2016-05-24 10:20:47 +0200 | [diff] [blame] | 189 | int RtpFrameObject::times_nacked() const { |
| 190 | return times_nacked_; |
| 191 | } |
| 192 | |
philipel | 02447bc | 2016-05-13 06:01:03 -0700 | [diff] [blame] | 193 | FrameType RtpFrameObject::frame_type() const { |
| 194 | return frame_type_; |
| 195 | } |
| 196 | |
| 197 | VideoCodecType RtpFrameObject::codec_type() const { |
| 198 | return codec_type_; |
philipel | c707ab7 | 2016-04-01 02:01:54 -0700 | [diff] [blame] | 199 | } |
| 200 | |
philipel | c707ab7 | 2016-04-01 02:01:54 -0700 | [diff] [blame] | 201 | bool RtpFrameObject::GetBitstream(uint8_t* destination) const { |
| 202 | return packet_buffer_->GetBitstream(*this, destination); |
| 203 | } |
| 204 | |
philipel | b4d3108 | 2016-07-11 08:46:29 -0700 | [diff] [blame] | 205 | uint32_t RtpFrameObject::Timestamp() const { |
| 206 | return timestamp_; |
| 207 | } |
| 208 | |
| 209 | int64_t RtpFrameObject::ReceivedTime() const { |
| 210 | return received_time_; |
| 211 | } |
| 212 | |
| 213 | int64_t RtpFrameObject::RenderTime() const { |
| 214 | return _renderTimeMs; |
| 215 | } |
| 216 | |
philipel | e075430 | 2017-01-25 08:56:23 -0800 | [diff] [blame] | 217 | bool RtpFrameObject::delayed_by_retransmission() const { |
| 218 | return times_nacked() > 0; |
| 219 | } |
| 220 | |
philipel | 8848828 | 2016-11-03 08:56:54 -0700 | [diff] [blame] | 221 | rtc::Optional<RTPVideoTypeHeader> RtpFrameObject::GetCodecHeader() const { |
| 222 | rtc::CritScope lock(&packet_buffer_->crit_); |
philipel | 02447bc | 2016-05-13 06:01:03 -0700 | [diff] [blame] | 223 | VCMPacket* packet = packet_buffer_->GetPacket(first_seq_num_); |
| 224 | if (!packet) |
philipel | 8848828 | 2016-11-03 08:56:54 -0700 | [diff] [blame] | 225 | return rtc::Optional<RTPVideoTypeHeader>(); |
| 226 | return rtc::Optional<RTPVideoTypeHeader>(packet->video_header.codecHeader); |
philipel | 02447bc | 2016-05-13 06:01:03 -0700 | [diff] [blame] | 227 | } |
| 228 | |
philipel | c707ab7 | 2016-04-01 02:01:54 -0700 | [diff] [blame] | 229 | } // namespace video_coding |
| 230 | } // namespace webrtc |