blob: ddd44709e5bb1d32f992e51690c74efd89d8e1fb [file] [log] [blame]
Stefan Holmerf7044682018-07-17 10:16:41 +02001/*
2 * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11#include "call/rtp_payload_params.h"
12
Yves Gerey3e707812018-11-28 16:47:49 +010013#include <stddef.h>
Jonas Olssona4d87372019-07-05 19:08:33 +020014
Elad Alonf5b216a2019-01-28 14:25:17 +010015#include <algorithm>
Yves Gerey3e707812018-11-28 16:47:49 +010016
17#include "absl/container/inlined_vector.h"
Erik Språngcbc0cba2020-04-18 14:36:59 +020018#include "absl/strings/match.h"
Yves Gerey3e707812018-11-28 16:47:49 +010019#include "absl/types/variant.h"
20#include "api/video/video_timing.h"
Yves Gerey3e707812018-11-28 16:47:49 +010021#include "modules/video_coding/codecs/h264/include/h264_globals.h"
22#include "modules/video_coding/codecs/interface/common_constants.h"
23#include "modules/video_coding/codecs/vp8/include/vp8_globals.h"
24#include "modules/video_coding/codecs/vp9/include/vp9_globals.h"
Danil Chapovalov02d71fb2020-02-10 16:22:57 +010025#include "modules/video_coding/frame_dependencies_calculator.h"
Elad Alonf5b216a2019-01-28 14:25:17 +010026#include "rtc_base/arraysize.h"
Stefan Holmerf7044682018-07-17 10:16:41 +020027#include "rtc_base/checks.h"
philipelbf2b6202018-08-27 14:33:18 +020028#include "rtc_base/logging.h"
Stefan Holmerf7044682018-07-17 10:16:41 +020029#include "rtc_base/random.h"
Steve Anton10542f22019-01-11 09:11:00 -080030#include "rtc_base/time_utils.h"
Stefan Holmerf7044682018-07-17 10:16:41 +020031
32namespace webrtc {
33
34namespace {
35void PopulateRtpWithCodecSpecifics(const CodecSpecificInfo& info,
Niels Möllerd3b8c632018-08-27 15:33:42 +020036 absl::optional<int> spatial_index,
Stefan Holmerf7044682018-07-17 10:16:41 +020037 RTPVideoHeader* rtp) {
38 rtp->codec = info.codecType;
Danil Chapovalov62a9a322020-11-11 16:15:07 +010039 rtp->is_last_frame_in_picture = info.end_of_picture;
Stefan Holmerf7044682018-07-17 10:16:41 +020040 switch (info.codecType) {
41 case kVideoCodecVP8: {
Philip Eliassond52a1a62018-09-07 13:03:55 +000042 auto& vp8_header = rtp->video_type_header.emplace<RTPVideoHeaderVP8>();
43 vp8_header.InitRTPVideoHeaderVP8();
44 vp8_header.nonReference = info.codecSpecific.VP8.nonReference;
45 vp8_header.temporalIdx = info.codecSpecific.VP8.temporalIdx;
46 vp8_header.layerSync = info.codecSpecific.VP8.layerSync;
47 vp8_header.keyIdx = info.codecSpecific.VP8.keyIdx;
Niels Möllerd3b8c632018-08-27 15:33:42 +020048 rtp->simulcastIdx = spatial_index.value_or(0);
Stefan Holmerf7044682018-07-17 10:16:41 +020049 return;
50 }
51 case kVideoCodecVP9: {
philipel29d88462018-08-08 14:26:00 +020052 auto& vp9_header = rtp->video_type_header.emplace<RTPVideoHeaderVP9>();
53 vp9_header.InitRTPVideoHeaderVP9();
54 vp9_header.inter_pic_predicted =
Stefan Holmerf7044682018-07-17 10:16:41 +020055 info.codecSpecific.VP9.inter_pic_predicted;
philipel29d88462018-08-08 14:26:00 +020056 vp9_header.flexible_mode = info.codecSpecific.VP9.flexible_mode;
57 vp9_header.ss_data_available = info.codecSpecific.VP9.ss_data_available;
58 vp9_header.non_ref_for_inter_layer_pred =
Stefan Holmerf7044682018-07-17 10:16:41 +020059 info.codecSpecific.VP9.non_ref_for_inter_layer_pred;
philipel29d88462018-08-08 14:26:00 +020060 vp9_header.temporal_idx = info.codecSpecific.VP9.temporal_idx;
philipel29d88462018-08-08 14:26:00 +020061 vp9_header.temporal_up_switch = info.codecSpecific.VP9.temporal_up_switch;
62 vp9_header.inter_layer_predicted =
Stefan Holmerf7044682018-07-17 10:16:41 +020063 info.codecSpecific.VP9.inter_layer_predicted;
philipel29d88462018-08-08 14:26:00 +020064 vp9_header.gof_idx = info.codecSpecific.VP9.gof_idx;
65 vp9_header.num_spatial_layers = info.codecSpecific.VP9.num_spatial_layers;
Ilya Nikolaevskiyf5d87782020-02-04 10:06:33 +000066 vp9_header.first_active_layer = info.codecSpecific.VP9.first_active_layer;
Niels Möllerd3b8c632018-08-27 15:33:42 +020067 if (vp9_header.num_spatial_layers > 1) {
68 vp9_header.spatial_idx = spatial_index.value_or(kNoSpatialIdx);
69 } else {
70 vp9_header.spatial_idx = kNoSpatialIdx;
71 }
Stefan Holmerf7044682018-07-17 10:16:41 +020072 if (info.codecSpecific.VP9.ss_data_available) {
philipel29d88462018-08-08 14:26:00 +020073 vp9_header.spatial_layer_resolution_present =
Stefan Holmerf7044682018-07-17 10:16:41 +020074 info.codecSpecific.VP9.spatial_layer_resolution_present;
75 if (info.codecSpecific.VP9.spatial_layer_resolution_present) {
76 for (size_t i = 0; i < info.codecSpecific.VP9.num_spatial_layers;
77 ++i) {
philipel29d88462018-08-08 14:26:00 +020078 vp9_header.width[i] = info.codecSpecific.VP9.width[i];
79 vp9_header.height[i] = info.codecSpecific.VP9.height[i];
Stefan Holmerf7044682018-07-17 10:16:41 +020080 }
81 }
philipel29d88462018-08-08 14:26:00 +020082 vp9_header.gof.CopyGofInfoVP9(info.codecSpecific.VP9.gof);
Stefan Holmerf7044682018-07-17 10:16:41 +020083 }
84
philipel29d88462018-08-08 14:26:00 +020085 vp9_header.num_ref_pics = info.codecSpecific.VP9.num_ref_pics;
Stefan Holmerf7044682018-07-17 10:16:41 +020086 for (int i = 0; i < info.codecSpecific.VP9.num_ref_pics; ++i) {
philipel29d88462018-08-08 14:26:00 +020087 vp9_header.pid_diff[i] = info.codecSpecific.VP9.p_diff[i];
Stefan Holmerf7044682018-07-17 10:16:41 +020088 }
Danil Chapovalov06bbeb32020-11-11 12:42:56 +010089 vp9_header.end_of_picture = info.end_of_picture;
Stefan Holmerf7044682018-07-17 10:16:41 +020090 return;
91 }
92 case kVideoCodecH264: {
philipel7d745e52018-08-02 14:03:53 +020093 auto& h264_header = rtp->video_type_header.emplace<RTPVideoHeaderH264>();
94 h264_header.packetization_mode =
Stefan Holmerf7044682018-07-17 10:16:41 +020095 info.codecSpecific.H264.packetization_mode;
Niels Möllerd3b8c632018-08-27 15:33:42 +020096 rtp->simulcastIdx = spatial_index.value_or(0);
Stefan Holmerf7044682018-07-17 10:16:41 +020097 return;
98 }
99 case kVideoCodecMultiplex:
100 case kVideoCodecGeneric:
101 rtp->codec = kVideoCodecGeneric;
Niels Möllerd3b8c632018-08-27 15:33:42 +0200102 rtp->simulcastIdx = spatial_index.value_or(0);
Stefan Holmerf7044682018-07-17 10:16:41 +0200103 return;
104 default:
105 return;
106 }
107}
108
109void SetVideoTiming(const EncodedImage& image, VideoSendTiming* timing) {
110 if (image.timing_.flags == VideoSendTiming::TimingFrameFlags::kInvalid ||
111 image.timing_.flags == VideoSendTiming::TimingFrameFlags::kNotTriggered) {
112 timing->flags = VideoSendTiming::TimingFrameFlags::kInvalid;
113 return;
114 }
115
116 timing->encode_start_delta_ms = VideoSendTiming::GetDeltaCappedMs(
117 image.capture_time_ms_, image.timing_.encode_start_ms);
118 timing->encode_finish_delta_ms = VideoSendTiming::GetDeltaCappedMs(
119 image.capture_time_ms_, image.timing_.encode_finish_ms);
120 timing->packetization_finish_delta_ms = 0;
121 timing->pacer_exit_delta_ms = 0;
122 timing->network_timestamp_delta_ms = 0;
123 timing->network2_timestamp_delta_ms = 0;
124 timing->flags = image.timing_.flags;
125}
126} // namespace
127
128RtpPayloadParams::RtpPayloadParams(const uint32_t ssrc,
Erik Språngcbc0cba2020-04-18 14:36:59 +0200129 const RtpPayloadState* state,
Jonas Orelande62c2f22022-03-29 11:04:48 +0200130 const FieldTrialsView& trials)
philipelbf2b6202018-08-27 14:33:18 +0200131 : ssrc_(ssrc),
132 generic_picture_id_experiment_(
Erik Språngcbc0cba2020-04-18 14:36:59 +0200133 absl::StartsWith(trials.Lookup("WebRTC-GenericPictureId"),
Emil Lundmark823ba0b2021-10-18 11:27:26 +0200134 "Enabled")) {
philipelbf2b6202018-08-27 14:33:18 +0200135 for (auto& spatial_layer : last_shared_frame_id_)
136 spatial_layer.fill(-1);
137
Emil Lundmarkadfc7002021-07-30 09:45:10 +0200138 chain_last_frame_id_.fill(-1);
Elad Alonf5b216a2019-01-28 14:25:17 +0100139 buffer_id_to_frame_id_.fill(-1);
140
Stefan Holmerf7044682018-07-17 10:16:41 +0200141 Random random(rtc::TimeMicros());
142 state_.picture_id =
143 state ? state->picture_id : (random.Rand<int16_t>() & 0x7FFF);
144 state_.tl0_pic_idx = state ? state->tl0_pic_idx : (random.Rand<uint8_t>());
145}
philipelbf2b6202018-08-27 14:33:18 +0200146
147RtpPayloadParams::RtpPayloadParams(const RtpPayloadParams& other) = default;
148
Stefan Holmerf7044682018-07-17 10:16:41 +0200149RtpPayloadParams::~RtpPayloadParams() {}
150
151RTPVideoHeader RtpPayloadParams::GetRtpVideoHeader(
152 const EncodedImage& image,
philipelbf2b6202018-08-27 14:33:18 +0200153 const CodecSpecificInfo* codec_specific_info,
154 int64_t shared_frame_id) {
Stefan Holmerf7044682018-07-17 10:16:41 +0200155 RTPVideoHeader rtp_video_header;
156 if (codec_specific_info) {
Niels Möllerd3b8c632018-08-27 15:33:42 +0200157 PopulateRtpWithCodecSpecifics(*codec_specific_info, image.SpatialIndex(),
158 &rtp_video_header);
Stefan Holmerf7044682018-07-17 10:16:41 +0200159 }
Zhaoliang Maf3dc47e2021-02-05 13:19:02 +0800160 rtp_video_header.frame_type = image._frameType;
Stefan Holmerf7044682018-07-17 10:16:41 +0200161 rtp_video_header.rotation = image.rotation_;
162 rtp_video_header.content_type = image.content_type_;
163 rtp_video_header.playout_delay = image.playout_delay_;
philipelfab91292018-10-17 14:36:08 +0200164 rtp_video_header.width = image._encodedWidth;
165 rtp_video_header.height = image._encodedHeight;
Johannes Krond0b69a82018-12-03 14:18:53 +0100166 rtp_video_header.color_space = image.ColorSpace()
167 ? absl::make_optional(*image.ColorSpace())
168 : absl::nullopt;
Jeremy Leconteb258c562021-03-18 13:50:42 +0100169 rtp_video_header.video_frame_tracking_id = image.VideoFrameTrackingId();
Stefan Holmerf7044682018-07-17 10:16:41 +0200170 SetVideoTiming(image, &rtp_video_header.video_timing);
171
Niels Möller8f7ce222019-03-21 15:43:58 +0100172 const bool is_keyframe = image._frameType == VideoFrameType::kVideoFrameKey;
Stefan Holmerf7044682018-07-17 10:16:41 +0200173 const bool first_frame_in_picture =
174 (codec_specific_info && codec_specific_info->codecType == kVideoCodecVP9)
175 ? codec_specific_info->codecSpecific.VP9.first_frame_in_picture
176 : true;
philipelbf2b6202018-08-27 14:33:18 +0200177
178 SetCodecSpecific(&rtp_video_header, first_frame_in_picture);
philipel569397f2018-09-26 12:25:31 +0200179
Danil Chapovalov636865e2020-06-03 14:11:26 +0200180 SetGeneric(codec_specific_info, shared_frame_id, is_keyframe,
181 &rtp_video_header);
philipelbf2b6202018-08-27 14:33:18 +0200182
Stefan Holmerf7044682018-07-17 10:16:41 +0200183 return rtp_video_header;
184}
185
186uint32_t RtpPayloadParams::ssrc() const {
187 return ssrc_;
188}
189
190RtpPayloadState RtpPayloadParams::state() const {
191 return state_;
192}
193
philipelbf2b6202018-08-27 14:33:18 +0200194void RtpPayloadParams::SetCodecSpecific(RTPVideoHeader* rtp_video_header,
195 bool first_frame_in_picture) {
Stefan Holmerf7044682018-07-17 10:16:41 +0200196 // Always set picture id. Set tl0_pic_idx iff temporal index is set.
197 if (first_frame_in_picture) {
198 state_.picture_id = (static_cast<uint16_t>(state_.picture_id) + 1) & 0x7FFF;
199 }
200 if (rtp_video_header->codec == kVideoCodecVP8) {
Philip Eliassond52a1a62018-09-07 13:03:55 +0000201 auto& vp8_header =
202 absl::get<RTPVideoHeaderVP8>(rtp_video_header->video_type_header);
203 vp8_header.pictureId = state_.picture_id;
Stefan Holmerf7044682018-07-17 10:16:41 +0200204
Philip Eliassond52a1a62018-09-07 13:03:55 +0000205 if (vp8_header.temporalIdx != kNoTemporalIdx) {
206 if (vp8_header.temporalIdx == 0) {
Stefan Holmerf7044682018-07-17 10:16:41 +0200207 ++state_.tl0_pic_idx;
208 }
Philip Eliassond52a1a62018-09-07 13:03:55 +0000209 vp8_header.tl0PicIdx = state_.tl0_pic_idx;
Stefan Holmerf7044682018-07-17 10:16:41 +0200210 }
211 }
212 if (rtp_video_header->codec == kVideoCodecVP9) {
philipel29d88462018-08-08 14:26:00 +0200213 auto& vp9_header =
214 absl::get<RTPVideoHeaderVP9>(rtp_video_header->video_type_header);
215 vp9_header.picture_id = state_.picture_id;
Stefan Holmerf7044682018-07-17 10:16:41 +0200216
217 // Note that in the case that we have no temporal layers but we do have
218 // spatial layers, packets will carry layering info with a temporal_idx of
219 // zero, and we then have to set and increment tl0_pic_idx.
philipel29d88462018-08-08 14:26:00 +0200220 if (vp9_header.temporal_idx != kNoTemporalIdx ||
221 vp9_header.spatial_idx != kNoSpatialIdx) {
Stefan Holmerf7044682018-07-17 10:16:41 +0200222 if (first_frame_in_picture &&
philipel29d88462018-08-08 14:26:00 +0200223 (vp9_header.temporal_idx == 0 ||
224 vp9_header.temporal_idx == kNoTemporalIdx)) {
Stefan Holmerf7044682018-07-17 10:16:41 +0200225 ++state_.tl0_pic_idx;
226 }
philipel29d88462018-08-08 14:26:00 +0200227 vp9_header.tl0_pic_idx = state_.tl0_pic_idx;
Stefan Holmerf7044682018-07-17 10:16:41 +0200228 }
229 }
philipelbf2b6202018-08-27 14:33:18 +0200230 if (generic_picture_id_experiment_ &&
231 rtp_video_header->codec == kVideoCodecGeneric) {
Danil Chapovalovb6bf0b22020-01-28 18:36:57 +0100232 rtp_video_header->video_type_header.emplace<RTPVideoHeaderLegacyGeneric>()
233 .picture_id = state_.picture_id;
philipelbf2b6202018-08-27 14:33:18 +0200234 }
Stefan Holmerf7044682018-07-17 10:16:41 +0200235}
philipelbf2b6202018-08-27 14:33:18 +0200236
Danil Chapovalov02d71fb2020-02-10 16:22:57 +0100237RTPVideoHeader::GenericDescriptorInfo
238RtpPayloadParams::GenericDescriptorFromFrameInfo(
239 const GenericFrameInfo& frame_info,
Danil Chapovalovcf1308f2020-11-18 18:27:37 +0100240 int64_t frame_id) {
Danil Chapovalov02d71fb2020-02-10 16:22:57 +0100241 RTPVideoHeader::GenericDescriptorInfo generic;
242 generic.frame_id = frame_id;
243 generic.dependencies = dependencies_calculator_.FromBuffersUsage(
Danil Chapovalovcf1308f2020-11-18 18:27:37 +0100244 frame_id, frame_info.encoder_buffers);
Danil Chapovalov4b860c12020-05-19 14:48:19 +0200245 generic.chain_diffs =
246 chains_calculator_.From(frame_id, frame_info.part_of_chain);
Danil Chapovalov02d71fb2020-02-10 16:22:57 +0100247 generic.spatial_index = frame_info.spatial_id;
248 generic.temporal_index = frame_info.temporal_id;
249 generic.decode_target_indications = frame_info.decode_target_indications;
Danil Chapovalove6ac8ff2020-06-26 13:51:08 +0200250 generic.active_decode_targets = frame_info.active_decode_targets;
Danil Chapovalov02d71fb2020-02-10 16:22:57 +0100251 return generic;
252}
253
Elad Alonf5b216a2019-01-28 14:25:17 +0100254void RtpPayloadParams::SetGeneric(const CodecSpecificInfo* codec_specific_info,
255 int64_t frame_id,
philipelbf2b6202018-08-27 14:33:18 +0200256 bool is_keyframe,
257 RTPVideoHeader* rtp_video_header) {
Danil Chapovalov02d71fb2020-02-10 16:22:57 +0100258 if (codec_specific_info && codec_specific_info->generic_frame_info &&
259 !codec_specific_info->generic_frame_info->encoder_buffers.empty()) {
Danil Chapovalov4b860c12020-05-19 14:48:19 +0200260 if (is_keyframe) {
261 // Key frame resets all chains it is in.
262 chains_calculator_.Reset(
263 codec_specific_info->generic_frame_info->part_of_chain);
264 }
Danil Chapovalovcf1308f2020-11-18 18:27:37 +0100265 rtp_video_header->generic = GenericDescriptorFromFrameInfo(
266 *codec_specific_info->generic_frame_info, frame_id);
Danil Chapovalov02d71fb2020-02-10 16:22:57 +0100267 return;
268 }
269
Elad Alonf5b216a2019-01-28 14:25:17 +0100270 switch (rtp_video_header->codec) {
271 case VideoCodecType::kVideoCodecGeneric:
philipel8aba8fe2019-06-13 15:13:16 +0200272 GenericToGeneric(frame_id, is_keyframe, rtp_video_header);
Elad Alonf5b216a2019-01-28 14:25:17 +0100273 return;
274 case VideoCodecType::kVideoCodecVP8:
275 if (codec_specific_info) {
276 Vp8ToGeneric(codec_specific_info->codecSpecific.VP8, frame_id,
277 is_keyframe, rtp_video_header);
278 }
279 return;
280 case VideoCodecType::kVideoCodecVP9:
Emil Lundmark823ba0b2021-10-18 11:27:26 +0200281 if (codec_specific_info != nullptr) {
Danil Chapovalovaf366442021-04-22 15:20:28 +0200282 Vp9ToGeneric(codec_specific_info->codecSpecific.VP9, frame_id,
283 *rtp_video_header);
284 }
285 return;
Danil Chapovalovdc368292019-11-26 14:48:20 +0100286 case VideoCodecType::kVideoCodecAV1:
Danil Chapovalovaf366442021-04-22 15:20:28 +0200287 // TODO(philipel): Implement AV1 to generic descriptor.
Elad Alonf5b216a2019-01-28 14:25:17 +0100288 return;
289 case VideoCodecType::kVideoCodecH264:
philipel8aba8fe2019-06-13 15:13:16 +0200290 if (codec_specific_info) {
291 H264ToGeneric(codec_specific_info->codecSpecific.H264, frame_id,
292 is_keyframe, rtp_video_header);
293 }
294 return;
Elad Alonf5b216a2019-01-28 14:25:17 +0100295 case VideoCodecType::kVideoCodecMultiplex:
296 return;
philipelbf2b6202018-08-27 14:33:18 +0200297 }
Artem Titovd3251962021-11-15 16:57:07 +0100298 RTC_DCHECK_NOTREACHED() << "Unsupported codec.";
philipelbf2b6202018-08-27 14:33:18 +0200299}
300
philipel8aba8fe2019-06-13 15:13:16 +0200301void RtpPayloadParams::GenericToGeneric(int64_t shared_frame_id,
302 bool is_keyframe,
303 RTPVideoHeader* rtp_video_header) {
304 RTPVideoHeader::GenericDescriptorInfo& generic =
305 rtp_video_header->generic.emplace();
306
307 generic.frame_id = shared_frame_id;
philipel5b231de2021-09-01 15:21:16 +0200308 generic.decode_target_indications.push_back(DecodeTargetIndication::kSwitch);
philipel8aba8fe2019-06-13 15:13:16 +0200309
310 if (is_keyframe) {
philipel5b231de2021-09-01 15:21:16 +0200311 generic.chain_diffs.push_back(0);
philipel8aba8fe2019-06-13 15:13:16 +0200312 last_shared_frame_id_[0].fill(-1);
313 } else {
314 int64_t frame_id = last_shared_frame_id_[0][0];
315 RTC_DCHECK_NE(frame_id, -1);
316 RTC_DCHECK_LT(frame_id, shared_frame_id);
philipel5b231de2021-09-01 15:21:16 +0200317 generic.chain_diffs.push_back(shared_frame_id - frame_id);
philipel8aba8fe2019-06-13 15:13:16 +0200318 generic.dependencies.push_back(frame_id);
319 }
320
321 last_shared_frame_id_[0][0] = shared_frame_id;
322}
323
324void RtpPayloadParams::H264ToGeneric(const CodecSpecificInfoH264& h264_info,
325 int64_t shared_frame_id,
326 bool is_keyframe,
327 RTPVideoHeader* rtp_video_header) {
328 const int temporal_index =
329 h264_info.temporal_idx != kNoTemporalIdx ? h264_info.temporal_idx : 0;
330
331 if (temporal_index >= RtpGenericFrameDescriptor::kMaxTemporalLayers) {
332 RTC_LOG(LS_WARNING) << "Temporal and/or spatial index is too high to be "
333 "used with generic frame descriptor.";
334 return;
335 }
336
337 RTPVideoHeader::GenericDescriptorInfo& generic =
338 rtp_video_header->generic.emplace();
339
340 generic.frame_id = shared_frame_id;
341 generic.temporal_index = temporal_index;
342
343 if (is_keyframe) {
344 RTC_DCHECK_EQ(temporal_index, 0);
345 last_shared_frame_id_[/*spatial index*/ 0].fill(-1);
346 last_shared_frame_id_[/*spatial index*/ 0][temporal_index] =
347 shared_frame_id;
348 return;
349 }
350
351 if (h264_info.base_layer_sync) {
352 int64_t tl0_frame_id = last_shared_frame_id_[/*spatial index*/ 0][0];
353
354 for (int i = 1; i < RtpGenericFrameDescriptor::kMaxTemporalLayers; ++i) {
355 if (last_shared_frame_id_[/*spatial index*/ 0][i] < tl0_frame_id) {
356 last_shared_frame_id_[/*spatial index*/ 0][i] = -1;
357 }
358 }
359
360 RTC_DCHECK_GE(tl0_frame_id, 0);
361 RTC_DCHECK_LT(tl0_frame_id, shared_frame_id);
362 generic.dependencies.push_back(tl0_frame_id);
363 } else {
364 for (int i = 0; i <= temporal_index; ++i) {
365 int64_t frame_id = last_shared_frame_id_[/*spatial index*/ 0][i];
366
367 if (frame_id != -1) {
368 RTC_DCHECK_LT(frame_id, shared_frame_id);
369 generic.dependencies.push_back(frame_id);
370 }
371 }
372 }
373
374 last_shared_frame_id_[/*spatial_index*/ 0][temporal_index] = shared_frame_id;
375}
376
Elad Alonf5b216a2019-01-28 14:25:17 +0100377void RtpPayloadParams::Vp8ToGeneric(const CodecSpecificInfoVP8& vp8_info,
378 int64_t shared_frame_id,
philipelbf2b6202018-08-27 14:33:18 +0200379 bool is_keyframe,
380 RTPVideoHeader* rtp_video_header) {
381 const auto& vp8_header =
382 absl::get<RTPVideoHeaderVP8>(rtp_video_header->video_type_header);
383 const int spatial_index = 0;
384 const int temporal_index =
385 vp8_header.temporalIdx != kNoTemporalIdx ? vp8_header.temporalIdx : 0;
386
387 if (temporal_index >= RtpGenericFrameDescriptor::kMaxTemporalLayers ||
388 spatial_index >= RtpGenericFrameDescriptor::kMaxSpatialLayers) {
389 RTC_LOG(LS_WARNING) << "Temporal and/or spatial index is too high to be "
390 "used with generic frame descriptor.";
391 return;
392 }
393
394 RTPVideoHeader::GenericDescriptorInfo& generic =
395 rtp_video_header->generic.emplace();
396
397 generic.frame_id = shared_frame_id;
398 generic.spatial_index = spatial_index;
399 generic.temporal_index = temporal_index;
400
Qingsi Wang1c1b99e2020-01-07 19:16:33 +0000401 if (vp8_info.useExplicitDependencies) {
402 SetDependenciesVp8New(vp8_info, shared_frame_id, is_keyframe,
403 vp8_header.layerSync, &generic);
404 } else {
405 SetDependenciesVp8Deprecated(vp8_info, shared_frame_id, is_keyframe,
406 spatial_index, temporal_index,
407 vp8_header.layerSync, &generic);
408 }
409}
410
philipel5b231de2021-09-01 15:21:16 +0200411FrameDependencyStructure RtpPayloadParams::MinimalisticStructure(
412 int num_spatial_layers,
413 int num_temporal_layers) {
414 RTC_DCHECK_LE(num_spatial_layers * num_temporal_layers, 32);
Danil Chapovalov748550d2021-04-29 11:42:54 +0200415 FrameDependencyStructure structure;
416 structure.num_decode_targets = num_spatial_layers * num_temporal_layers;
417 structure.num_chains = num_spatial_layers;
418 structure.templates.reserve(num_spatial_layers * num_temporal_layers);
419 for (int sid = 0; sid < num_spatial_layers; ++sid) {
420 for (int tid = 0; tid < num_temporal_layers; ++tid) {
421 FrameDependencyTemplate a_template;
422 a_template.spatial_id = sid;
423 a_template.temporal_id = tid;
424 for (int s = 0; s < num_spatial_layers; ++s) {
425 for (int t = 0; t < num_temporal_layers; ++t) {
philipel5b231de2021-09-01 15:21:16 +0200426 // Prefer kSwitch indication for frames that is part of the decode
427 // target because dependency descriptor information generated in this
428 // class use kSwitch indications more often that kRequired, increasing
429 // the chance of a good (or complete) template match.
Danil Chapovalov748550d2021-04-29 11:42:54 +0200430 a_template.decode_target_indications.push_back(
431 sid <= s && tid <= t ? DecodeTargetIndication::kSwitch
432 : DecodeTargetIndication::kNotPresent);
433 }
434 }
435 a_template.frame_diffs.push_back(tid == 0 ? num_spatial_layers *
436 num_temporal_layers
437 : num_spatial_layers);
438 a_template.chain_diffs.assign(structure.num_chains, 1);
439 structure.templates.push_back(a_template);
440
441 structure.decode_target_protected_by_chain.push_back(sid);
442 }
Danil Chapovalov748550d2021-04-29 11:42:54 +0200443 }
444 return structure;
445}
446
Danil Chapovalovaf366442021-04-22 15:20:28 +0200447void RtpPayloadParams::Vp9ToGeneric(const CodecSpecificInfoVP9& vp9_info,
448 int64_t shared_frame_id,
449 RTPVideoHeader& rtp_video_header) {
450 const auto& vp9_header =
451 absl::get<RTPVideoHeaderVP9>(rtp_video_header.video_type_header);
452 const int num_spatial_layers = vp9_header.num_spatial_layers;
453 const int num_temporal_layers = kMaxTemporalStreams;
454
455 int spatial_index =
456 vp9_header.spatial_idx != kNoSpatialIdx ? vp9_header.spatial_idx : 0;
457 int temporal_index =
458 vp9_header.temporal_idx != kNoTemporalIdx ? vp9_header.temporal_idx : 0;
459
460 if (spatial_index >= num_spatial_layers ||
461 temporal_index >= num_temporal_layers ||
462 num_spatial_layers > RtpGenericFrameDescriptor::kMaxSpatialLayers) {
463 // Prefer to generate no generic layering than an inconsistent one.
464 return;
465 }
466
467 RTPVideoHeader::GenericDescriptorInfo& result =
468 rtp_video_header.generic.emplace();
469
470 result.frame_id = shared_frame_id;
471 result.spatial_index = spatial_index;
472 result.temporal_index = temporal_index;
473
474 result.decode_target_indications.reserve(num_spatial_layers *
475 num_temporal_layers);
476 for (int sid = 0; sid < num_spatial_layers; ++sid) {
477 for (int tid = 0; tid < num_temporal_layers; ++tid) {
478 DecodeTargetIndication dti;
479 if (sid < spatial_index || tid < temporal_index) {
480 dti = DecodeTargetIndication::kNotPresent;
481 } else if (spatial_index != sid &&
482 vp9_header.non_ref_for_inter_layer_pred) {
483 dti = DecodeTargetIndication::kNotPresent;
484 } else if (sid == spatial_index && tid == temporal_index) {
485 // Assume that if frame is decodable, all of its own layer is decodable.
486 dti = DecodeTargetIndication::kSwitch;
487 } else if (sid == spatial_index && vp9_header.temporal_up_switch) {
488 dti = DecodeTargetIndication::kSwitch;
489 } else if (!vp9_header.inter_pic_predicted) {
490 // Key frame or spatial upswitch
491 dti = DecodeTargetIndication::kSwitch;
492 } else {
493 // Make no other assumptions. That should be safe, though suboptimal.
494 // To provide more accurate dti, encoder wrapper should fill in
495 // CodecSpecificInfo::generic_frame_info
496 dti = DecodeTargetIndication::kRequired;
497 }
498 result.decode_target_indications.push_back(dti);
499 }
500 }
501
502 // Calculate frame dependencies.
503 static constexpr int kPictureDiffLimit = 128;
504 if (last_vp9_frame_id_.empty()) {
505 // Create the array only if it is ever used.
506 last_vp9_frame_id_.resize(kPictureDiffLimit);
507 }
508 if (vp9_header.inter_layer_predicted && spatial_index > 0) {
509 result.dependencies.push_back(
510 last_vp9_frame_id_[vp9_header.picture_id % kPictureDiffLimit]
511 [spatial_index - 1]);
512 }
513 if (vp9_header.inter_pic_predicted) {
514 for (size_t i = 0; i < vp9_header.num_ref_pics; ++i) {
515 // picture_id is 15 bit number that wraps around. Though undeflow may
516 // produce picture that exceeds 2^15, it is ok because in this
517 // code block only last 7 bits of the picture_id are used.
518 uint16_t depend_on = vp9_header.picture_id - vp9_header.pid_diff[i];
519 result.dependencies.push_back(
520 last_vp9_frame_id_[depend_on % kPictureDiffLimit][spatial_index]);
521 }
522 }
523 last_vp9_frame_id_[vp9_header.picture_id % kPictureDiffLimit][spatial_index] =
524 shared_frame_id;
525
526 // Calculate chains, asuming chain includes all frames with temporal_id = 0
527 if (!vp9_header.inter_pic_predicted && !vp9_header.inter_layer_predicted) {
528 // Assume frames without dependencies also reset chains.
529 for (int sid = spatial_index; sid < num_spatial_layers; ++sid) {
530 chain_last_frame_id_[sid] = -1;
531 }
532 }
533 result.chain_diffs.resize(num_spatial_layers);
534 for (int sid = 0; sid < num_spatial_layers; ++sid) {
535 if (chain_last_frame_id_[sid] == -1) {
536 result.chain_diffs[sid] = 0;
537 continue;
538 }
539 result.chain_diffs[sid] = shared_frame_id - chain_last_frame_id_[sid];
540 }
541
542 if (temporal_index == 0) {
543 chain_last_frame_id_[spatial_index] = shared_frame_id;
544 if (!vp9_header.non_ref_for_inter_layer_pred) {
545 for (int sid = spatial_index + 1; sid < num_spatial_layers; ++sid) {
546 chain_last_frame_id_[sid] = shared_frame_id;
547 }
548 }
549 }
550}
551
Qingsi Wang1c1b99e2020-01-07 19:16:33 +0000552void RtpPayloadParams::SetDependenciesVp8Deprecated(
553 const CodecSpecificInfoVP8& vp8_info,
554 int64_t shared_frame_id,
555 bool is_keyframe,
556 int spatial_index,
557 int temporal_index,
558 bool layer_sync,
559 RTPVideoHeader::GenericDescriptorInfo* generic) {
560 RTC_DCHECK(!vp8_info.useExplicitDependencies);
561 RTC_DCHECK(!new_version_used_.has_value() || !new_version_used_.value());
562 new_version_used_ = false;
563
564 if (is_keyframe) {
565 RTC_DCHECK_EQ(temporal_index, 0);
566 last_shared_frame_id_[spatial_index].fill(-1);
567 last_shared_frame_id_[spatial_index][temporal_index] = shared_frame_id;
568 return;
569 }
570
571 if (layer_sync) {
572 int64_t tl0_frame_id = last_shared_frame_id_[spatial_index][0];
573
574 for (int i = 1; i < RtpGenericFrameDescriptor::kMaxTemporalLayers; ++i) {
575 if (last_shared_frame_id_[spatial_index][i] < tl0_frame_id) {
576 last_shared_frame_id_[spatial_index][i] = -1;
577 }
578 }
579
580 RTC_DCHECK_GE(tl0_frame_id, 0);
581 RTC_DCHECK_LT(tl0_frame_id, shared_frame_id);
582 generic->dependencies.push_back(tl0_frame_id);
583 } else {
584 for (int i = 0; i <= temporal_index; ++i) {
585 int64_t frame_id = last_shared_frame_id_[spatial_index][i];
586
587 if (frame_id != -1) {
588 RTC_DCHECK_LT(frame_id, shared_frame_id);
589 generic->dependencies.push_back(frame_id);
590 }
591 }
592 }
593
594 last_shared_frame_id_[spatial_index][temporal_index] = shared_frame_id;
595}
596
597void RtpPayloadParams::SetDependenciesVp8New(
598 const CodecSpecificInfoVP8& vp8_info,
599 int64_t shared_frame_id,
600 bool is_keyframe,
601 bool layer_sync,
602 RTPVideoHeader::GenericDescriptorInfo* generic) {
603 RTC_DCHECK(vp8_info.useExplicitDependencies);
604 RTC_DCHECK(!new_version_used_.has_value() || new_version_used_.value());
605 new_version_used_ = true;
606
Elad Alonf5b216a2019-01-28 14:25:17 +0100607 if (is_keyframe) {
608 RTC_DCHECK_EQ(vp8_info.referencedBuffersCount, 0u);
609 buffer_id_to_frame_id_.fill(shared_frame_id);
610 return;
611 }
612
613 constexpr size_t kBuffersCountVp8 = CodecSpecificInfoVP8::kBuffersCount;
614
615 RTC_DCHECK_GT(vp8_info.referencedBuffersCount, 0u);
616 RTC_DCHECK_LE(vp8_info.referencedBuffersCount,
617 arraysize(vp8_info.referencedBuffers));
618
619 for (size_t i = 0; i < vp8_info.referencedBuffersCount; ++i) {
620 const size_t referenced_buffer = vp8_info.referencedBuffers[i];
621 RTC_DCHECK_LT(referenced_buffer, kBuffersCountVp8);
622 RTC_DCHECK_LT(referenced_buffer, buffer_id_to_frame_id_.size());
623
624 const int64_t dependency_frame_id =
625 buffer_id_to_frame_id_[referenced_buffer];
626 RTC_DCHECK_GE(dependency_frame_id, 0);
627 RTC_DCHECK_LT(dependency_frame_id, shared_frame_id);
628
629 const bool is_new_dependency =
Qingsi Wang1c1b99e2020-01-07 19:16:33 +0000630 std::find(generic->dependencies.begin(), generic->dependencies.end(),
631 dependency_frame_id) == generic->dependencies.end();
Elad Alonf5b216a2019-01-28 14:25:17 +0100632 if (is_new_dependency) {
Qingsi Wang1c1b99e2020-01-07 19:16:33 +0000633 generic->dependencies.push_back(dependency_frame_id);
Elad Alonf5b216a2019-01-28 14:25:17 +0100634 }
635 }
636
637 RTC_DCHECK_LE(vp8_info.updatedBuffersCount, kBuffersCountVp8);
638 for (size_t i = 0; i < vp8_info.updatedBuffersCount; ++i) {
639 const size_t updated_id = vp8_info.updatedBuffers[i];
640 buffer_id_to_frame_id_[updated_id] = shared_frame_id;
641 }
642
643 RTC_DCHECK_LE(buffer_id_to_frame_id_.size(), kBuffersCountVp8);
644}
645
Stefan Holmerf7044682018-07-17 10:16:41 +0200646} // namespace webrtc