blob: 648af011f0e7a176bc295b452ae026ab1075a2d8 [file] [log] [blame]
philipelc707ab72016-04-01 02:01:54 -07001/*
2 * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020011#include "modules/video_coding/frame_object.h"
Rasmus Brandt58b72912017-10-25 11:09:23 +020012
13#include <sstream>
14
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020015#include "common_video/h264/h264_common.h"
16#include "modules/video_coding/packet_buffer.h"
17#include "rtc_base/checks.h"
Rasmus Brandt58b72912017-10-25 11:09:23 +020018#include "rtc_base/logging.h"
Rasmus Brandtedf4ff72017-10-24 10:07:48 +020019#include "system_wrappers/include/field_trial.h"
philipelc707ab72016-04-01 02:01:54 -070020
21namespace webrtc {
22namespace video_coding {
23
philipela1059872016-05-09 11:41:48 +020024FrameObject::FrameObject()
25 : picture_id(0),
26 spatial_layer(0),
philipelbe7a9e52016-05-19 12:19:35 +020027 timestamp(0),
philipela1059872016-05-09 11:41:48 +020028 num_references(0),
29 inter_layer_predicted(false) {}
30
philipelc707ab72016-04-01 02:01:54 -070031RtpFrameObject::RtpFrameObject(PacketBuffer* packet_buffer,
philipel02447bc2016-05-13 06:01:03 -070032 uint16_t first_seq_num,
philipel5ceaaae2016-05-24 10:20:47 +020033 uint16_t last_seq_num,
34 size_t frame_size,
philipelb4d31082016-07-11 08:46:29 -070035 int times_nacked,
36 int64_t received_time)
philipelc707ab72016-04-01 02:01:54 -070037 : packet_buffer_(packet_buffer),
philipel02447bc2016-05-13 06:01:03 -070038 first_seq_num_(first_seq_num),
philipel5ceaaae2016-05-24 10:20:47 +020039 last_seq_num_(last_seq_num),
sprangba050a62017-08-18 02:51:12 -070040 timestamp_(0),
philipelb4d31082016-07-11 08:46:29 -070041 received_time_(received_time),
philipel5ceaaae2016-05-24 10:20:47 +020042 times_nacked_(times_nacked) {
philipel266f0a42016-11-28 08:49:07 -080043 VCMPacket* first_packet = packet_buffer_->GetPacket(first_seq_num);
philipel7d79e632017-05-23 08:19:11 -070044 RTC_CHECK(first_packet);
philipel36928452016-11-07 10:42:36 +010045
philipel266f0a42016-11-28 08:49:07 -080046 // RtpFrameObject members
47 frame_type_ = first_packet->frameType;
48 codec_type_ = first_packet->codec;
philipel36928452016-11-07 10:42:36 +010049
philipel266f0a42016-11-28 08:49:07 -080050 // TODO(philipel): Remove when encoded image is replaced by FrameObject.
51 // VCMEncodedFrame members
52 CopyCodecSpecific(&first_packet->video_header);
53 _completeFrame = true;
54 _payloadType = first_packet->payloadType;
55 _timeStamp = first_packet->timestamp;
56 ntp_time_ms_ = first_packet->ntp_time_ms_;
philipel36928452016-11-07 10:42:36 +010057
gnishb2a318b2017-05-10 09:21:33 -070058 // Setting frame's playout delays to the same values
59 // as of the first packet's.
60 SetPlayoutDelay(first_packet->video_header.playout_delay);
61
philipel266f0a42016-11-28 08:49:07 -080062 // Since FFmpeg use an optimized bitstream reader that reads in chunks of
63 // 32/64 bits we have to add at least that much padding to the buffer
64 // to make sure the decoder doesn't read out of bounds.
65 // NOTE! EncodedImage::_size is the size of the buffer (think capacity of
66 // an std::vector) and EncodedImage::_length is the actual size of
67 // the bitstream (think size of an std::vector).
68 if (codec_type_ == kVideoCodecH264)
69 _size = frame_size + EncodedImage::kBufferPaddingBytesH264;
70 else
71 _size = frame_size;
philipel552866c2016-07-04 16:18:55 +020072
philipel266f0a42016-11-28 08:49:07 -080073 _buffer = new uint8_t[_size];
74 _length = frame_size;
philipele87c8762017-05-19 06:38:50 -070075
76 // For H264 frames we can't determine the frame type by just looking at the
Rasmus Brandtedf4ff72017-10-24 10:07:48 +020077 // first packet. Instead we consider the frame to be a keyframe if it contains
78 // an IDR, and SPS/PPS if the field trial is set.
philipele87c8762017-05-19 06:38:50 -070079 if (codec_type_ == kVideoCodecH264) {
80 _frameType = kVideoFrameDelta;
81 frame_type_ = kVideoFrameDelta;
Rasmus Brandtedf4ff72017-10-24 10:07:48 +020082 bool contains_sps = false;
83 bool contains_pps = false;
84 bool contains_idr = false;
philipele87c8762017-05-19 06:38:50 -070085 for (uint16_t seq_num = first_seq_num;
philipel7d79e632017-05-23 08:19:11 -070086 seq_num != static_cast<uint16_t>(last_seq_num + 1) &&
87 _frameType == kVideoFrameDelta;
philipele87c8762017-05-19 06:38:50 -070088 ++seq_num) {
89 VCMPacket* packet = packet_buffer_->GetPacket(seq_num);
philipel7d79e632017-05-23 08:19:11 -070090 RTC_CHECK(packet);
philipele87c8762017-05-19 06:38:50 -070091 const RTPVideoHeaderH264& header = packet->video_header.codecHeader.H264;
92 for (size_t i = 0; i < header.nalus_length; ++i) {
Rasmus Brandtedf4ff72017-10-24 10:07:48 +020093 if (header.nalus[i].type == H264::NaluType::kSps) {
94 contains_sps = true;
95 } else if (header.nalus[i].type == H264::NaluType::kPps) {
96 contains_pps = true;
97 } else if (header.nalus[i].type == H264::NaluType::kIdr) {
98 contains_idr = true;
philipele87c8762017-05-19 06:38:50 -070099 }
100 }
101 }
Rasmus Brandtedf4ff72017-10-24 10:07:48 +0200102 const bool sps_pps_idr_is_keyframe =
103 field_trial::IsEnabled("WebRTC-SpsPpsIdrIsH264Keyframe");
104 if ((sps_pps_idr_is_keyframe && contains_idr && contains_sps &&
105 contains_pps) ||
106 (!sps_pps_idr_is_keyframe && contains_idr)) {
107 _frameType = kVideoFrameKey;
108 frame_type_ = kVideoFrameKey;
109 }
Rasmus Brandt58b72912017-10-25 11:09:23 +0200110 if (contains_idr && (!contains_sps || !contains_pps)) {
111 std::stringstream ss;
112 ss << "Received H.264-IDR frame "
113 << "(SPS: " << contains_sps << ", PPS: " << contains_pps << "). ";
114 if (sps_pps_idr_is_keyframe) {
115 ss << "Treating as delta frame since WebRTC-SpsPpsIdrIsH264Keyframe is "
116 "enabled.";
117 } else {
118 ss << "Treating as key frame since WebRTC-SpsPpsIdrIsH264Keyframe is "
119 "disabled.";
120 }
121 LOG(LS_WARNING) << ss.str();
122 }
philipele87c8762017-05-19 06:38:50 -0700123 } else {
124 _frameType = first_packet->frameType;
125 frame_type_ = first_packet->frameType;
126 }
127
philipel227f8b92017-08-04 06:39:31 -0700128 bool bitstream_copied = GetBitstream(_buffer);
129 RTC_DCHECK(bitstream_copied);
philipel59e99b72017-01-12 03:26:04 -0800130 _encodedWidth = first_packet->width;
131 _encodedHeight = first_packet->height;
philipel266f0a42016-11-28 08:49:07 -0800132
133 // FrameObject members
134 timestamp = first_packet->timestamp;
135
136 VCMPacket* last_packet = packet_buffer_->GetPacket(last_seq_num);
kwibergee89e782017-08-09 17:22:01 -0700137 RTC_CHECK(last_packet);
138 RTC_CHECK(last_packet->markerBit);
philipel266f0a42016-11-28 08:49:07 -0800139 // http://www.etsi.org/deliver/etsi_ts/126100_126199/126114/12.07.00_60/
140 // ts_126114v120700p.pdf Section 7.4.5.
141 // The MTSI client shall add the payload bytes as defined in this clause
142 // onto the last RTP packet in each group of packets which make up a key
143 // frame (I-frame or IDR frame in H.264 (AVC), or an IRAP picture in H.265
144 // (HEVC)).
145 rotation_ = last_packet->video_header.rotation;
146 _rotation_set = true;
ilnik00d802b2017-04-11 10:34:31 -0700147 content_type_ = last_packet->video_header.content_type;
sprangba050a62017-08-18 02:51:12 -0700148 if (last_packet->video_header.video_timing.flags !=
149 TimingFrameFlags::kInvalid) {
ilnik04f4d122017-06-19 07:18:55 -0700150 // ntp_time_ms_ may be -1 if not estimated yet. This is not a problem,
151 // as this will be dealt with at the time of reporting.
ilnik04f4d122017-06-19 07:18:55 -0700152 timing_.encode_start_ms =
153 ntp_time_ms_ +
154 last_packet->video_header.video_timing.encode_start_delta_ms;
155 timing_.encode_finish_ms =
156 ntp_time_ms_ +
157 last_packet->video_header.video_timing.encode_finish_delta_ms;
158 timing_.packetization_finish_ms =
159 ntp_time_ms_ +
160 last_packet->video_header.video_timing.packetization_finish_delta_ms;
161 timing_.pacer_exit_ms =
162 ntp_time_ms_ +
163 last_packet->video_header.video_timing.pacer_exit_delta_ms;
164 timing_.network_timestamp_ms =
165 ntp_time_ms_ +
Danil Chapovalov996eb9e2017-10-30 17:14:41 +0100166 last_packet->video_header.video_timing.network_timestamp_delta_ms;
ilnik04f4d122017-06-19 07:18:55 -0700167 timing_.network2_timestamp_ms =
168 ntp_time_ms_ +
Danil Chapovalov996eb9e2017-10-30 17:14:41 +0100169 last_packet->video_header.video_timing.network2_timestamp_delta_ms;
ilnik04f4d122017-06-19 07:18:55 -0700170
171 timing_.receive_start_ms = first_packet->receive_time_ms;
172 timing_.receive_finish_ms = last_packet->receive_time_ms;
ilnik04f4d122017-06-19 07:18:55 -0700173 }
sprangba050a62017-08-18 02:51:12 -0700174 timing_.flags = last_packet->video_header.video_timing.flags;
philipel02447bc2016-05-13 06:01:03 -0700175}
philipelc707ab72016-04-01 02:01:54 -0700176
177RtpFrameObject::~RtpFrameObject() {
178 packet_buffer_->ReturnFrame(this);
179}
180
philipelf4139332016-04-20 10:26:34 +0200181uint16_t RtpFrameObject::first_seq_num() const {
philipel02447bc2016-05-13 06:01:03 -0700182 return first_seq_num_;
philipelc707ab72016-04-01 02:01:54 -0700183}
184
philipelf4139332016-04-20 10:26:34 +0200185uint16_t RtpFrameObject::last_seq_num() const {
philipel02447bc2016-05-13 06:01:03 -0700186 return last_seq_num_;
187}
188
philipel5ceaaae2016-05-24 10:20:47 +0200189int RtpFrameObject::times_nacked() const {
190 return times_nacked_;
191}
192
philipel02447bc2016-05-13 06:01:03 -0700193FrameType RtpFrameObject::frame_type() const {
194 return frame_type_;
195}
196
197VideoCodecType RtpFrameObject::codec_type() const {
198 return codec_type_;
philipelc707ab72016-04-01 02:01:54 -0700199}
200
philipelc707ab72016-04-01 02:01:54 -0700201bool RtpFrameObject::GetBitstream(uint8_t* destination) const {
202 return packet_buffer_->GetBitstream(*this, destination);
203}
204
philipelb4d31082016-07-11 08:46:29 -0700205uint32_t RtpFrameObject::Timestamp() const {
206 return timestamp_;
207}
208
209int64_t RtpFrameObject::ReceivedTime() const {
210 return received_time_;
211}
212
213int64_t RtpFrameObject::RenderTime() const {
214 return _renderTimeMs;
215}
216
philipele0754302017-01-25 08:56:23 -0800217bool RtpFrameObject::delayed_by_retransmission() const {
218 return times_nacked() > 0;
219}
220
philipel88488282016-11-03 08:56:54 -0700221rtc::Optional<RTPVideoTypeHeader> RtpFrameObject::GetCodecHeader() const {
222 rtc::CritScope lock(&packet_buffer_->crit_);
philipel02447bc2016-05-13 06:01:03 -0700223 VCMPacket* packet = packet_buffer_->GetPacket(first_seq_num_);
224 if (!packet)
philipel88488282016-11-03 08:56:54 -0700225 return rtc::Optional<RTPVideoTypeHeader>();
226 return rtc::Optional<RTPVideoTypeHeader>(packet->video_header.codecHeader);
philipel02447bc2016-05-13 06:01:03 -0700227}
228
philipelc707ab72016-04-01 02:01:54 -0700229} // namespace video_coding
230} // namespace webrtc