blob: ad8c1dac00a2015afd101b6891c0fff66eeea7b3 [file] [log] [blame]
Stefan Holmerf7044682018-07-17 10:16:41 +02001/*
2 * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11#include "call/rtp_payload_params.h"
12
Yves Gerey3e707812018-11-28 16:47:49 +010013#include <stddef.h>
Jonas Olssona4d87372019-07-05 19:08:33 +020014
Elad Alonf5b216a2019-01-28 14:25:17 +010015#include <algorithm>
Yves Gerey3e707812018-11-28 16:47:49 +010016
17#include "absl/container/inlined_vector.h"
Erik Språngcbc0cba2020-04-18 14:36:59 +020018#include "absl/strings/match.h"
Yves Gerey3e707812018-11-28 16:47:49 +010019#include "absl/types/variant.h"
20#include "api/video/video_timing.h"
Yves Gerey3e707812018-11-28 16:47:49 +010021#include "modules/video_coding/codecs/h264/include/h264_globals.h"
22#include "modules/video_coding/codecs/interface/common_constants.h"
23#include "modules/video_coding/codecs/vp8/include/vp8_globals.h"
24#include "modules/video_coding/codecs/vp9/include/vp9_globals.h"
Danil Chapovalov02d71fb2020-02-10 16:22:57 +010025#include "modules/video_coding/frame_dependencies_calculator.h"
Elad Alonf5b216a2019-01-28 14:25:17 +010026#include "rtc_base/arraysize.h"
Stefan Holmerf7044682018-07-17 10:16:41 +020027#include "rtc_base/checks.h"
philipelbf2b6202018-08-27 14:33:18 +020028#include "rtc_base/logging.h"
Stefan Holmerf7044682018-07-17 10:16:41 +020029#include "rtc_base/random.h"
Steve Anton10542f22019-01-11 09:11:00 -080030#include "rtc_base/time_utils.h"
Stefan Holmerf7044682018-07-17 10:16:41 +020031
32namespace webrtc {
33
34namespace {
35void PopulateRtpWithCodecSpecifics(const CodecSpecificInfo& info,
Niels Möllerd3b8c632018-08-27 15:33:42 +020036 absl::optional<int> spatial_index,
Stefan Holmerf7044682018-07-17 10:16:41 +020037 RTPVideoHeader* rtp) {
38 rtp->codec = info.codecType;
Danil Chapovalov62a9a322020-11-11 16:15:07 +010039 rtp->is_last_frame_in_picture = info.end_of_picture;
Stefan Holmerf7044682018-07-17 10:16:41 +020040 switch (info.codecType) {
41 case kVideoCodecVP8: {
Philip Eliassond52a1a62018-09-07 13:03:55 +000042 auto& vp8_header = rtp->video_type_header.emplace<RTPVideoHeaderVP8>();
43 vp8_header.InitRTPVideoHeaderVP8();
44 vp8_header.nonReference = info.codecSpecific.VP8.nonReference;
45 vp8_header.temporalIdx = info.codecSpecific.VP8.temporalIdx;
46 vp8_header.layerSync = info.codecSpecific.VP8.layerSync;
47 vp8_header.keyIdx = info.codecSpecific.VP8.keyIdx;
Niels Möllerd3b8c632018-08-27 15:33:42 +020048 rtp->simulcastIdx = spatial_index.value_or(0);
Stefan Holmerf7044682018-07-17 10:16:41 +020049 return;
50 }
51 case kVideoCodecVP9: {
philipel29d88462018-08-08 14:26:00 +020052 auto& vp9_header = rtp->video_type_header.emplace<RTPVideoHeaderVP9>();
53 vp9_header.InitRTPVideoHeaderVP9();
54 vp9_header.inter_pic_predicted =
Stefan Holmerf7044682018-07-17 10:16:41 +020055 info.codecSpecific.VP9.inter_pic_predicted;
philipel29d88462018-08-08 14:26:00 +020056 vp9_header.flexible_mode = info.codecSpecific.VP9.flexible_mode;
57 vp9_header.ss_data_available = info.codecSpecific.VP9.ss_data_available;
58 vp9_header.non_ref_for_inter_layer_pred =
Stefan Holmerf7044682018-07-17 10:16:41 +020059 info.codecSpecific.VP9.non_ref_for_inter_layer_pred;
philipel29d88462018-08-08 14:26:00 +020060 vp9_header.temporal_idx = info.codecSpecific.VP9.temporal_idx;
philipel29d88462018-08-08 14:26:00 +020061 vp9_header.temporal_up_switch = info.codecSpecific.VP9.temporal_up_switch;
62 vp9_header.inter_layer_predicted =
Stefan Holmerf7044682018-07-17 10:16:41 +020063 info.codecSpecific.VP9.inter_layer_predicted;
philipel29d88462018-08-08 14:26:00 +020064 vp9_header.gof_idx = info.codecSpecific.VP9.gof_idx;
65 vp9_header.num_spatial_layers = info.codecSpecific.VP9.num_spatial_layers;
Ilya Nikolaevskiyf5d87782020-02-04 10:06:33 +000066 vp9_header.first_active_layer = info.codecSpecific.VP9.first_active_layer;
Niels Möllerd3b8c632018-08-27 15:33:42 +020067 if (vp9_header.num_spatial_layers > 1) {
68 vp9_header.spatial_idx = spatial_index.value_or(kNoSpatialIdx);
69 } else {
70 vp9_header.spatial_idx = kNoSpatialIdx;
71 }
Stefan Holmerf7044682018-07-17 10:16:41 +020072 if (info.codecSpecific.VP9.ss_data_available) {
philipel29d88462018-08-08 14:26:00 +020073 vp9_header.spatial_layer_resolution_present =
Stefan Holmerf7044682018-07-17 10:16:41 +020074 info.codecSpecific.VP9.spatial_layer_resolution_present;
75 if (info.codecSpecific.VP9.spatial_layer_resolution_present) {
76 for (size_t i = 0; i < info.codecSpecific.VP9.num_spatial_layers;
77 ++i) {
philipel29d88462018-08-08 14:26:00 +020078 vp9_header.width[i] = info.codecSpecific.VP9.width[i];
79 vp9_header.height[i] = info.codecSpecific.VP9.height[i];
Stefan Holmerf7044682018-07-17 10:16:41 +020080 }
81 }
philipel29d88462018-08-08 14:26:00 +020082 vp9_header.gof.CopyGofInfoVP9(info.codecSpecific.VP9.gof);
Stefan Holmerf7044682018-07-17 10:16:41 +020083 }
84
philipel29d88462018-08-08 14:26:00 +020085 vp9_header.num_ref_pics = info.codecSpecific.VP9.num_ref_pics;
Stefan Holmerf7044682018-07-17 10:16:41 +020086 for (int i = 0; i < info.codecSpecific.VP9.num_ref_pics; ++i) {
philipel29d88462018-08-08 14:26:00 +020087 vp9_header.pid_diff[i] = info.codecSpecific.VP9.p_diff[i];
Stefan Holmerf7044682018-07-17 10:16:41 +020088 }
Danil Chapovalov06bbeb32020-11-11 12:42:56 +010089 vp9_header.end_of_picture = info.end_of_picture;
Stefan Holmerf7044682018-07-17 10:16:41 +020090 return;
91 }
92 case kVideoCodecH264: {
philipel7d745e52018-08-02 14:03:53 +020093 auto& h264_header = rtp->video_type_header.emplace<RTPVideoHeaderH264>();
94 h264_header.packetization_mode =
Stefan Holmerf7044682018-07-17 10:16:41 +020095 info.codecSpecific.H264.packetization_mode;
Niels Möllerd3b8c632018-08-27 15:33:42 +020096 rtp->simulcastIdx = spatial_index.value_or(0);
Stefan Holmerf7044682018-07-17 10:16:41 +020097 return;
98 }
99 case kVideoCodecMultiplex:
100 case kVideoCodecGeneric:
101 rtp->codec = kVideoCodecGeneric;
Niels Möllerd3b8c632018-08-27 15:33:42 +0200102 rtp->simulcastIdx = spatial_index.value_or(0);
Stefan Holmerf7044682018-07-17 10:16:41 +0200103 return;
104 default:
105 return;
106 }
107}
108
109void SetVideoTiming(const EncodedImage& image, VideoSendTiming* timing) {
110 if (image.timing_.flags == VideoSendTiming::TimingFrameFlags::kInvalid ||
111 image.timing_.flags == VideoSendTiming::TimingFrameFlags::kNotTriggered) {
112 timing->flags = VideoSendTiming::TimingFrameFlags::kInvalid;
113 return;
114 }
115
116 timing->encode_start_delta_ms = VideoSendTiming::GetDeltaCappedMs(
117 image.capture_time_ms_, image.timing_.encode_start_ms);
118 timing->encode_finish_delta_ms = VideoSendTiming::GetDeltaCappedMs(
119 image.capture_time_ms_, image.timing_.encode_finish_ms);
120 timing->packetization_finish_delta_ms = 0;
121 timing->pacer_exit_delta_ms = 0;
122 timing->network_timestamp_delta_ms = 0;
123 timing->network2_timestamp_delta_ms = 0;
124 timing->flags = image.timing_.flags;
125}
126} // namespace
127
128RtpPayloadParams::RtpPayloadParams(const uint32_t ssrc,
Erik Språngcbc0cba2020-04-18 14:36:59 +0200129 const RtpPayloadState* state,
130 const WebRtcKeyValueConfig& trials)
philipelbf2b6202018-08-27 14:33:18 +0200131 : ssrc_(ssrc),
132 generic_picture_id_experiment_(
Erik Språngcbc0cba2020-04-18 14:36:59 +0200133 absl::StartsWith(trials.Lookup("WebRTC-GenericPictureId"),
Danil Chapovalovaf366442021-04-22 15:20:28 +0200134 "Enabled")),
135 simulate_generic_vp9_(
Emil Lundmark47270712021-07-28 13:04:36 +0200136 !absl::StartsWith(trials.Lookup("WebRTC-Vp9DependencyDescriptor"),
137 "Disabled")) {
philipelbf2b6202018-08-27 14:33:18 +0200138 for (auto& spatial_layer : last_shared_frame_id_)
139 spatial_layer.fill(-1);
140
Elad Alonf5b216a2019-01-28 14:25:17 +0100141 buffer_id_to_frame_id_.fill(-1);
142
Stefan Holmerf7044682018-07-17 10:16:41 +0200143 Random random(rtc::TimeMicros());
144 state_.picture_id =
145 state ? state->picture_id : (random.Rand<int16_t>() & 0x7FFF);
146 state_.tl0_pic_idx = state ? state->tl0_pic_idx : (random.Rand<uint8_t>());
147}
philipelbf2b6202018-08-27 14:33:18 +0200148
149RtpPayloadParams::RtpPayloadParams(const RtpPayloadParams& other) = default;
150
Stefan Holmerf7044682018-07-17 10:16:41 +0200151RtpPayloadParams::~RtpPayloadParams() {}
152
153RTPVideoHeader RtpPayloadParams::GetRtpVideoHeader(
154 const EncodedImage& image,
philipelbf2b6202018-08-27 14:33:18 +0200155 const CodecSpecificInfo* codec_specific_info,
156 int64_t shared_frame_id) {
Stefan Holmerf7044682018-07-17 10:16:41 +0200157 RTPVideoHeader rtp_video_header;
158 if (codec_specific_info) {
Niels Möllerd3b8c632018-08-27 15:33:42 +0200159 PopulateRtpWithCodecSpecifics(*codec_specific_info, image.SpatialIndex(),
160 &rtp_video_header);
Stefan Holmerf7044682018-07-17 10:16:41 +0200161 }
Zhaoliang Maf3dc47e2021-02-05 13:19:02 +0800162 rtp_video_header.frame_type = image._frameType;
Stefan Holmerf7044682018-07-17 10:16:41 +0200163 rtp_video_header.rotation = image.rotation_;
164 rtp_video_header.content_type = image.content_type_;
165 rtp_video_header.playout_delay = image.playout_delay_;
philipelfab91292018-10-17 14:36:08 +0200166 rtp_video_header.width = image._encodedWidth;
167 rtp_video_header.height = image._encodedHeight;
Johannes Krond0b69a82018-12-03 14:18:53 +0100168 rtp_video_header.color_space = image.ColorSpace()
169 ? absl::make_optional(*image.ColorSpace())
170 : absl::nullopt;
Jeremy Leconteb258c562021-03-18 13:50:42 +0100171 rtp_video_header.video_frame_tracking_id = image.VideoFrameTrackingId();
Stefan Holmerf7044682018-07-17 10:16:41 +0200172 SetVideoTiming(image, &rtp_video_header.video_timing);
173
Niels Möller8f7ce222019-03-21 15:43:58 +0100174 const bool is_keyframe = image._frameType == VideoFrameType::kVideoFrameKey;
Stefan Holmerf7044682018-07-17 10:16:41 +0200175 const bool first_frame_in_picture =
176 (codec_specific_info && codec_specific_info->codecType == kVideoCodecVP9)
177 ? codec_specific_info->codecSpecific.VP9.first_frame_in_picture
178 : true;
philipelbf2b6202018-08-27 14:33:18 +0200179
180 SetCodecSpecific(&rtp_video_header, first_frame_in_picture);
philipel569397f2018-09-26 12:25:31 +0200181
Danil Chapovalov636865e2020-06-03 14:11:26 +0200182 SetGeneric(codec_specific_info, shared_frame_id, is_keyframe,
183 &rtp_video_header);
philipelbf2b6202018-08-27 14:33:18 +0200184
Stefan Holmerf7044682018-07-17 10:16:41 +0200185 return rtp_video_header;
186}
187
188uint32_t RtpPayloadParams::ssrc() const {
189 return ssrc_;
190}
191
192RtpPayloadState RtpPayloadParams::state() const {
193 return state_;
194}
195
philipelbf2b6202018-08-27 14:33:18 +0200196void RtpPayloadParams::SetCodecSpecific(RTPVideoHeader* rtp_video_header,
197 bool first_frame_in_picture) {
Stefan Holmerf7044682018-07-17 10:16:41 +0200198 // Always set picture id. Set tl0_pic_idx iff temporal index is set.
199 if (first_frame_in_picture) {
200 state_.picture_id = (static_cast<uint16_t>(state_.picture_id) + 1) & 0x7FFF;
201 }
202 if (rtp_video_header->codec == kVideoCodecVP8) {
Philip Eliassond52a1a62018-09-07 13:03:55 +0000203 auto& vp8_header =
204 absl::get<RTPVideoHeaderVP8>(rtp_video_header->video_type_header);
205 vp8_header.pictureId = state_.picture_id;
Stefan Holmerf7044682018-07-17 10:16:41 +0200206
Philip Eliassond52a1a62018-09-07 13:03:55 +0000207 if (vp8_header.temporalIdx != kNoTemporalIdx) {
208 if (vp8_header.temporalIdx == 0) {
Stefan Holmerf7044682018-07-17 10:16:41 +0200209 ++state_.tl0_pic_idx;
210 }
Philip Eliassond52a1a62018-09-07 13:03:55 +0000211 vp8_header.tl0PicIdx = state_.tl0_pic_idx;
Stefan Holmerf7044682018-07-17 10:16:41 +0200212 }
213 }
214 if (rtp_video_header->codec == kVideoCodecVP9) {
philipel29d88462018-08-08 14:26:00 +0200215 auto& vp9_header =
216 absl::get<RTPVideoHeaderVP9>(rtp_video_header->video_type_header);
217 vp9_header.picture_id = state_.picture_id;
Stefan Holmerf7044682018-07-17 10:16:41 +0200218
219 // Note that in the case that we have no temporal layers but we do have
220 // spatial layers, packets will carry layering info with a temporal_idx of
221 // zero, and we then have to set and increment tl0_pic_idx.
philipel29d88462018-08-08 14:26:00 +0200222 if (vp9_header.temporal_idx != kNoTemporalIdx ||
223 vp9_header.spatial_idx != kNoSpatialIdx) {
Stefan Holmerf7044682018-07-17 10:16:41 +0200224 if (first_frame_in_picture &&
philipel29d88462018-08-08 14:26:00 +0200225 (vp9_header.temporal_idx == 0 ||
226 vp9_header.temporal_idx == kNoTemporalIdx)) {
Stefan Holmerf7044682018-07-17 10:16:41 +0200227 ++state_.tl0_pic_idx;
228 }
philipel29d88462018-08-08 14:26:00 +0200229 vp9_header.tl0_pic_idx = state_.tl0_pic_idx;
Stefan Holmerf7044682018-07-17 10:16:41 +0200230 }
231 }
philipelbf2b6202018-08-27 14:33:18 +0200232 if (generic_picture_id_experiment_ &&
233 rtp_video_header->codec == kVideoCodecGeneric) {
Danil Chapovalovb6bf0b22020-01-28 18:36:57 +0100234 rtp_video_header->video_type_header.emplace<RTPVideoHeaderLegacyGeneric>()
235 .picture_id = state_.picture_id;
philipelbf2b6202018-08-27 14:33:18 +0200236 }
Stefan Holmerf7044682018-07-17 10:16:41 +0200237}
philipelbf2b6202018-08-27 14:33:18 +0200238
Danil Chapovalov02d71fb2020-02-10 16:22:57 +0100239RTPVideoHeader::GenericDescriptorInfo
240RtpPayloadParams::GenericDescriptorFromFrameInfo(
241 const GenericFrameInfo& frame_info,
Danil Chapovalovcf1308f2020-11-18 18:27:37 +0100242 int64_t frame_id) {
Danil Chapovalov02d71fb2020-02-10 16:22:57 +0100243 RTPVideoHeader::GenericDescriptorInfo generic;
244 generic.frame_id = frame_id;
245 generic.dependencies = dependencies_calculator_.FromBuffersUsage(
Danil Chapovalovcf1308f2020-11-18 18:27:37 +0100246 frame_id, frame_info.encoder_buffers);
Danil Chapovalov4b860c12020-05-19 14:48:19 +0200247 generic.chain_diffs =
248 chains_calculator_.From(frame_id, frame_info.part_of_chain);
Danil Chapovalov02d71fb2020-02-10 16:22:57 +0100249 generic.spatial_index = frame_info.spatial_id;
250 generic.temporal_index = frame_info.temporal_id;
251 generic.decode_target_indications = frame_info.decode_target_indications;
Danil Chapovalove6ac8ff2020-06-26 13:51:08 +0200252 generic.active_decode_targets = frame_info.active_decode_targets;
Danil Chapovalov02d71fb2020-02-10 16:22:57 +0100253 return generic;
254}
255
Elad Alonf5b216a2019-01-28 14:25:17 +0100256void RtpPayloadParams::SetGeneric(const CodecSpecificInfo* codec_specific_info,
257 int64_t frame_id,
philipelbf2b6202018-08-27 14:33:18 +0200258 bool is_keyframe,
259 RTPVideoHeader* rtp_video_header) {
Danil Chapovalov02d71fb2020-02-10 16:22:57 +0100260 if (codec_specific_info && codec_specific_info->generic_frame_info &&
261 !codec_specific_info->generic_frame_info->encoder_buffers.empty()) {
Danil Chapovalov4b860c12020-05-19 14:48:19 +0200262 if (is_keyframe) {
263 // Key frame resets all chains it is in.
264 chains_calculator_.Reset(
265 codec_specific_info->generic_frame_info->part_of_chain);
266 }
Danil Chapovalovcf1308f2020-11-18 18:27:37 +0100267 rtp_video_header->generic = GenericDescriptorFromFrameInfo(
268 *codec_specific_info->generic_frame_info, frame_id);
Danil Chapovalov02d71fb2020-02-10 16:22:57 +0100269 return;
270 }
271
Elad Alonf5b216a2019-01-28 14:25:17 +0100272 switch (rtp_video_header->codec) {
273 case VideoCodecType::kVideoCodecGeneric:
philipel8aba8fe2019-06-13 15:13:16 +0200274 GenericToGeneric(frame_id, is_keyframe, rtp_video_header);
Elad Alonf5b216a2019-01-28 14:25:17 +0100275 return;
276 case VideoCodecType::kVideoCodecVP8:
277 if (codec_specific_info) {
278 Vp8ToGeneric(codec_specific_info->codecSpecific.VP8, frame_id,
279 is_keyframe, rtp_video_header);
280 }
281 return;
282 case VideoCodecType::kVideoCodecVP9:
Danil Chapovalovaf366442021-04-22 15:20:28 +0200283 if (simulate_generic_vp9_ && codec_specific_info != nullptr) {
284 Vp9ToGeneric(codec_specific_info->codecSpecific.VP9, frame_id,
285 *rtp_video_header);
286 }
287 return;
Danil Chapovalovdc368292019-11-26 14:48:20 +0100288 case VideoCodecType::kVideoCodecAV1:
Danil Chapovalovaf366442021-04-22 15:20:28 +0200289 // TODO(philipel): Implement AV1 to generic descriptor.
Elad Alonf5b216a2019-01-28 14:25:17 +0100290 return;
291 case VideoCodecType::kVideoCodecH264:
philipel8aba8fe2019-06-13 15:13:16 +0200292 if (codec_specific_info) {
293 H264ToGeneric(codec_specific_info->codecSpecific.H264, frame_id,
294 is_keyframe, rtp_video_header);
295 }
296 return;
Elad Alonf5b216a2019-01-28 14:25:17 +0100297 case VideoCodecType::kVideoCodecMultiplex:
298 return;
philipelbf2b6202018-08-27 14:33:18 +0200299 }
Elad Alonf5b216a2019-01-28 14:25:17 +0100300 RTC_NOTREACHED() << "Unsupported codec.";
philipelbf2b6202018-08-27 14:33:18 +0200301}
302
philipel8aba8fe2019-06-13 15:13:16 +0200303void RtpPayloadParams::GenericToGeneric(int64_t shared_frame_id,
304 bool is_keyframe,
305 RTPVideoHeader* rtp_video_header) {
306 RTPVideoHeader::GenericDescriptorInfo& generic =
307 rtp_video_header->generic.emplace();
308
309 generic.frame_id = shared_frame_id;
310
311 if (is_keyframe) {
312 last_shared_frame_id_[0].fill(-1);
313 } else {
314 int64_t frame_id = last_shared_frame_id_[0][0];
315 RTC_DCHECK_NE(frame_id, -1);
316 RTC_DCHECK_LT(frame_id, shared_frame_id);
317 generic.dependencies.push_back(frame_id);
318 }
319
320 last_shared_frame_id_[0][0] = shared_frame_id;
321}
322
323void RtpPayloadParams::H264ToGeneric(const CodecSpecificInfoH264& h264_info,
324 int64_t shared_frame_id,
325 bool is_keyframe,
326 RTPVideoHeader* rtp_video_header) {
327 const int temporal_index =
328 h264_info.temporal_idx != kNoTemporalIdx ? h264_info.temporal_idx : 0;
329
330 if (temporal_index >= RtpGenericFrameDescriptor::kMaxTemporalLayers) {
331 RTC_LOG(LS_WARNING) << "Temporal and/or spatial index is too high to be "
332 "used with generic frame descriptor.";
333 return;
334 }
335
336 RTPVideoHeader::GenericDescriptorInfo& generic =
337 rtp_video_header->generic.emplace();
338
339 generic.frame_id = shared_frame_id;
340 generic.temporal_index = temporal_index;
341
342 if (is_keyframe) {
343 RTC_DCHECK_EQ(temporal_index, 0);
344 last_shared_frame_id_[/*spatial index*/ 0].fill(-1);
345 last_shared_frame_id_[/*spatial index*/ 0][temporal_index] =
346 shared_frame_id;
347 return;
348 }
349
350 if (h264_info.base_layer_sync) {
351 int64_t tl0_frame_id = last_shared_frame_id_[/*spatial index*/ 0][0];
352
353 for (int i = 1; i < RtpGenericFrameDescriptor::kMaxTemporalLayers; ++i) {
354 if (last_shared_frame_id_[/*spatial index*/ 0][i] < tl0_frame_id) {
355 last_shared_frame_id_[/*spatial index*/ 0][i] = -1;
356 }
357 }
358
359 RTC_DCHECK_GE(tl0_frame_id, 0);
360 RTC_DCHECK_LT(tl0_frame_id, shared_frame_id);
361 generic.dependencies.push_back(tl0_frame_id);
362 } else {
363 for (int i = 0; i <= temporal_index; ++i) {
364 int64_t frame_id = last_shared_frame_id_[/*spatial index*/ 0][i];
365
366 if (frame_id != -1) {
367 RTC_DCHECK_LT(frame_id, shared_frame_id);
368 generic.dependencies.push_back(frame_id);
369 }
370 }
371 }
372
373 last_shared_frame_id_[/*spatial_index*/ 0][temporal_index] = shared_frame_id;
374}
375
Elad Alonf5b216a2019-01-28 14:25:17 +0100376void RtpPayloadParams::Vp8ToGeneric(const CodecSpecificInfoVP8& vp8_info,
377 int64_t shared_frame_id,
philipelbf2b6202018-08-27 14:33:18 +0200378 bool is_keyframe,
379 RTPVideoHeader* rtp_video_header) {
380 const auto& vp8_header =
381 absl::get<RTPVideoHeaderVP8>(rtp_video_header->video_type_header);
382 const int spatial_index = 0;
383 const int temporal_index =
384 vp8_header.temporalIdx != kNoTemporalIdx ? vp8_header.temporalIdx : 0;
385
386 if (temporal_index >= RtpGenericFrameDescriptor::kMaxTemporalLayers ||
387 spatial_index >= RtpGenericFrameDescriptor::kMaxSpatialLayers) {
388 RTC_LOG(LS_WARNING) << "Temporal and/or spatial index is too high to be "
389 "used with generic frame descriptor.";
390 return;
391 }
392
393 RTPVideoHeader::GenericDescriptorInfo& generic =
394 rtp_video_header->generic.emplace();
395
396 generic.frame_id = shared_frame_id;
397 generic.spatial_index = spatial_index;
398 generic.temporal_index = temporal_index;
399
Qingsi Wang1c1b99e2020-01-07 19:16:33 +0000400 if (vp8_info.useExplicitDependencies) {
401 SetDependenciesVp8New(vp8_info, shared_frame_id, is_keyframe,
402 vp8_header.layerSync, &generic);
403 } else {
404 SetDependenciesVp8Deprecated(vp8_info, shared_frame_id, is_keyframe,
405 spatial_index, temporal_index,
406 vp8_header.layerSync, &generic);
407 }
408}
409
Danil Chapovalov748550d2021-04-29 11:42:54 +0200410FrameDependencyStructure RtpPayloadParams::MinimalisticVp9Structure(
411 const CodecSpecificInfoVP9& vp9) {
412 const int num_spatial_layers = vp9.num_spatial_layers;
413 const int num_temporal_layers = kMaxTemporalStreams;
414 FrameDependencyStructure structure;
415 structure.num_decode_targets = num_spatial_layers * num_temporal_layers;
416 structure.num_chains = num_spatial_layers;
417 structure.templates.reserve(num_spatial_layers * num_temporal_layers);
418 for (int sid = 0; sid < num_spatial_layers; ++sid) {
419 for (int tid = 0; tid < num_temporal_layers; ++tid) {
420 FrameDependencyTemplate a_template;
421 a_template.spatial_id = sid;
422 a_template.temporal_id = tid;
423 for (int s = 0; s < num_spatial_layers; ++s) {
424 for (int t = 0; t < num_temporal_layers; ++t) {
425 // Prefer kSwitch for indication frame is part of the decode target
426 // because RtpPayloadParams::Vp9ToGeneric uses that indication more
427 // often that kRequired, increasing chance custom dti need not to
428 // use more bits in dependency descriptor on the wire.
429 a_template.decode_target_indications.push_back(
430 sid <= s && tid <= t ? DecodeTargetIndication::kSwitch
431 : DecodeTargetIndication::kNotPresent);
432 }
433 }
434 a_template.frame_diffs.push_back(tid == 0 ? num_spatial_layers *
435 num_temporal_layers
436 : num_spatial_layers);
437 a_template.chain_diffs.assign(structure.num_chains, 1);
438 structure.templates.push_back(a_template);
439
440 structure.decode_target_protected_by_chain.push_back(sid);
441 }
442 if (vp9.ss_data_available && vp9.spatial_layer_resolution_present) {
443 structure.resolutions.emplace_back(vp9.width[sid], vp9.height[sid]);
444 }
445 }
446 return structure;
447}
448
Danil Chapovalovaf366442021-04-22 15:20:28 +0200449void RtpPayloadParams::Vp9ToGeneric(const CodecSpecificInfoVP9& vp9_info,
450 int64_t shared_frame_id,
451 RTPVideoHeader& rtp_video_header) {
452 const auto& vp9_header =
453 absl::get<RTPVideoHeaderVP9>(rtp_video_header.video_type_header);
454 const int num_spatial_layers = vp9_header.num_spatial_layers;
455 const int num_temporal_layers = kMaxTemporalStreams;
456
457 int spatial_index =
458 vp9_header.spatial_idx != kNoSpatialIdx ? vp9_header.spatial_idx : 0;
459 int temporal_index =
460 vp9_header.temporal_idx != kNoTemporalIdx ? vp9_header.temporal_idx : 0;
461
462 if (spatial_index >= num_spatial_layers ||
463 temporal_index >= num_temporal_layers ||
464 num_spatial_layers > RtpGenericFrameDescriptor::kMaxSpatialLayers) {
465 // Prefer to generate no generic layering than an inconsistent one.
466 return;
467 }
468
469 RTPVideoHeader::GenericDescriptorInfo& result =
470 rtp_video_header.generic.emplace();
471
472 result.frame_id = shared_frame_id;
473 result.spatial_index = spatial_index;
474 result.temporal_index = temporal_index;
475
476 result.decode_target_indications.reserve(num_spatial_layers *
477 num_temporal_layers);
478 for (int sid = 0; sid < num_spatial_layers; ++sid) {
479 for (int tid = 0; tid < num_temporal_layers; ++tid) {
480 DecodeTargetIndication dti;
481 if (sid < spatial_index || tid < temporal_index) {
482 dti = DecodeTargetIndication::kNotPresent;
483 } else if (spatial_index != sid &&
484 vp9_header.non_ref_for_inter_layer_pred) {
485 dti = DecodeTargetIndication::kNotPresent;
486 } else if (sid == spatial_index && tid == temporal_index) {
487 // Assume that if frame is decodable, all of its own layer is decodable.
488 dti = DecodeTargetIndication::kSwitch;
489 } else if (sid == spatial_index && vp9_header.temporal_up_switch) {
490 dti = DecodeTargetIndication::kSwitch;
491 } else if (!vp9_header.inter_pic_predicted) {
492 // Key frame or spatial upswitch
493 dti = DecodeTargetIndication::kSwitch;
494 } else {
495 // Make no other assumptions. That should be safe, though suboptimal.
496 // To provide more accurate dti, encoder wrapper should fill in
497 // CodecSpecificInfo::generic_frame_info
498 dti = DecodeTargetIndication::kRequired;
499 }
500 result.decode_target_indications.push_back(dti);
501 }
502 }
503
504 // Calculate frame dependencies.
505 static constexpr int kPictureDiffLimit = 128;
506 if (last_vp9_frame_id_.empty()) {
507 // Create the array only if it is ever used.
508 last_vp9_frame_id_.resize(kPictureDiffLimit);
509 }
510 if (vp9_header.inter_layer_predicted && spatial_index > 0) {
511 result.dependencies.push_back(
512 last_vp9_frame_id_[vp9_header.picture_id % kPictureDiffLimit]
513 [spatial_index - 1]);
514 }
515 if (vp9_header.inter_pic_predicted) {
516 for (size_t i = 0; i < vp9_header.num_ref_pics; ++i) {
517 // picture_id is 15 bit number that wraps around. Though undeflow may
518 // produce picture that exceeds 2^15, it is ok because in this
519 // code block only last 7 bits of the picture_id are used.
520 uint16_t depend_on = vp9_header.picture_id - vp9_header.pid_diff[i];
521 result.dependencies.push_back(
522 last_vp9_frame_id_[depend_on % kPictureDiffLimit][spatial_index]);
523 }
524 }
525 last_vp9_frame_id_[vp9_header.picture_id % kPictureDiffLimit][spatial_index] =
526 shared_frame_id;
527
528 // Calculate chains, asuming chain includes all frames with temporal_id = 0
529 if (!vp9_header.inter_pic_predicted && !vp9_header.inter_layer_predicted) {
530 // Assume frames without dependencies also reset chains.
531 for (int sid = spatial_index; sid < num_spatial_layers; ++sid) {
532 chain_last_frame_id_[sid] = -1;
533 }
534 }
535 result.chain_diffs.resize(num_spatial_layers);
536 for (int sid = 0; sid < num_spatial_layers; ++sid) {
537 if (chain_last_frame_id_[sid] == -1) {
538 result.chain_diffs[sid] = 0;
539 continue;
540 }
541 result.chain_diffs[sid] = shared_frame_id - chain_last_frame_id_[sid];
542 }
543
544 if (temporal_index == 0) {
545 chain_last_frame_id_[spatial_index] = shared_frame_id;
546 if (!vp9_header.non_ref_for_inter_layer_pred) {
547 for (int sid = spatial_index + 1; sid < num_spatial_layers; ++sid) {
548 chain_last_frame_id_[sid] = shared_frame_id;
549 }
550 }
551 }
552}
553
Qingsi Wang1c1b99e2020-01-07 19:16:33 +0000554void RtpPayloadParams::SetDependenciesVp8Deprecated(
555 const CodecSpecificInfoVP8& vp8_info,
556 int64_t shared_frame_id,
557 bool is_keyframe,
558 int spatial_index,
559 int temporal_index,
560 bool layer_sync,
561 RTPVideoHeader::GenericDescriptorInfo* generic) {
562 RTC_DCHECK(!vp8_info.useExplicitDependencies);
563 RTC_DCHECK(!new_version_used_.has_value() || !new_version_used_.value());
564 new_version_used_ = false;
565
566 if (is_keyframe) {
567 RTC_DCHECK_EQ(temporal_index, 0);
568 last_shared_frame_id_[spatial_index].fill(-1);
569 last_shared_frame_id_[spatial_index][temporal_index] = shared_frame_id;
570 return;
571 }
572
573 if (layer_sync) {
574 int64_t tl0_frame_id = last_shared_frame_id_[spatial_index][0];
575
576 for (int i = 1; i < RtpGenericFrameDescriptor::kMaxTemporalLayers; ++i) {
577 if (last_shared_frame_id_[spatial_index][i] < tl0_frame_id) {
578 last_shared_frame_id_[spatial_index][i] = -1;
579 }
580 }
581
582 RTC_DCHECK_GE(tl0_frame_id, 0);
583 RTC_DCHECK_LT(tl0_frame_id, shared_frame_id);
584 generic->dependencies.push_back(tl0_frame_id);
585 } else {
586 for (int i = 0; i <= temporal_index; ++i) {
587 int64_t frame_id = last_shared_frame_id_[spatial_index][i];
588
589 if (frame_id != -1) {
590 RTC_DCHECK_LT(frame_id, shared_frame_id);
591 generic->dependencies.push_back(frame_id);
592 }
593 }
594 }
595
596 last_shared_frame_id_[spatial_index][temporal_index] = shared_frame_id;
597}
598
599void RtpPayloadParams::SetDependenciesVp8New(
600 const CodecSpecificInfoVP8& vp8_info,
601 int64_t shared_frame_id,
602 bool is_keyframe,
603 bool layer_sync,
604 RTPVideoHeader::GenericDescriptorInfo* generic) {
605 RTC_DCHECK(vp8_info.useExplicitDependencies);
606 RTC_DCHECK(!new_version_used_.has_value() || new_version_used_.value());
607 new_version_used_ = true;
608
Elad Alonf5b216a2019-01-28 14:25:17 +0100609 if (is_keyframe) {
610 RTC_DCHECK_EQ(vp8_info.referencedBuffersCount, 0u);
611 buffer_id_to_frame_id_.fill(shared_frame_id);
612 return;
613 }
614
615 constexpr size_t kBuffersCountVp8 = CodecSpecificInfoVP8::kBuffersCount;
616
617 RTC_DCHECK_GT(vp8_info.referencedBuffersCount, 0u);
618 RTC_DCHECK_LE(vp8_info.referencedBuffersCount,
619 arraysize(vp8_info.referencedBuffers));
620
621 for (size_t i = 0; i < vp8_info.referencedBuffersCount; ++i) {
622 const size_t referenced_buffer = vp8_info.referencedBuffers[i];
623 RTC_DCHECK_LT(referenced_buffer, kBuffersCountVp8);
624 RTC_DCHECK_LT(referenced_buffer, buffer_id_to_frame_id_.size());
625
626 const int64_t dependency_frame_id =
627 buffer_id_to_frame_id_[referenced_buffer];
628 RTC_DCHECK_GE(dependency_frame_id, 0);
629 RTC_DCHECK_LT(dependency_frame_id, shared_frame_id);
630
631 const bool is_new_dependency =
Qingsi Wang1c1b99e2020-01-07 19:16:33 +0000632 std::find(generic->dependencies.begin(), generic->dependencies.end(),
633 dependency_frame_id) == generic->dependencies.end();
Elad Alonf5b216a2019-01-28 14:25:17 +0100634 if (is_new_dependency) {
Qingsi Wang1c1b99e2020-01-07 19:16:33 +0000635 generic->dependencies.push_back(dependency_frame_id);
Elad Alonf5b216a2019-01-28 14:25:17 +0100636 }
637 }
638
639 RTC_DCHECK_LE(vp8_info.updatedBuffersCount, kBuffersCountVp8);
640 for (size_t i = 0; i < vp8_info.updatedBuffersCount; ++i) {
641 const size_t updated_id = vp8_info.updatedBuffers[i];
642 buffer_id_to_frame_id_[updated_id] = shared_frame_id;
643 }
644
645 RTC_DCHECK_LE(buffer_id_to_frame_id_.size(), kBuffersCountVp8);
646}
647
Stefan Holmerf7044682018-07-17 10:16:41 +0200648} // namespace webrtc