Add sender controlled playout delay limits
This CL adds support for an extension on RTP frames to allow the sender
to specify the minimum and maximum playout delay limits.
The receiver makes a best-effort attempt to keep the capture-to-render delay
within this range. This allows different types of application to specify
different end-to-end delay goals. For example gaming can support rendering
of frames as soon as received on receiver to minimize delay. A movie playback
application can specify a minimum playout delay to allow fixed buffering
in presence of network jitter.
There are no tests at this time and most of testing is done with chromium
webrtc prototype.
On chromoting performance tests, this extension helps bring down end-to-end
delay by about 150 ms on small frames.
BUG=webrtc:5895
Review-Url: https://codereview.webrtc.org/2007743003
Cr-Commit-Position: refs/heads/master@{#13059}
diff --git a/webrtc/modules/video_coding/decoding_state_unittest.cc b/webrtc/modules/video_coding/decoding_state_unittest.cc
index 5f5d0d3..3942dab 100644
--- a/webrtc/modules/video_coding/decoding_state_unittest.cc
+++ b/webrtc/modules/video_coding/decoding_state_unittest.cc
@@ -36,8 +36,8 @@
packet.timestamp = 1;
packet.seqNum = 0xffff;
packet.frameType = kVideoFrameDelta;
- packet.codecSpecificHeader.codec = kRtpVideoVp8;
- packet.codecSpecificHeader.codecHeader.VP8.pictureId = 0x007F;
+ packet.video_header.codec = kRtpVideoVp8;
+ packet.video_header.codecHeader.VP8.pictureId = 0x007F;
FrameData frame_data;
frame_data.rtt_ms = 0;
frame_data.rolling_average_packets_per_frame = -1;
@@ -53,17 +53,17 @@
packet.frameType = kVideoFrameDelta;
// Use pictureId
packet.isFirstPacket = false;
- packet.codecSpecificHeader.codecHeader.VP8.pictureId = 0x0002;
+ packet.video_header.codecHeader.VP8.pictureId = 0x0002;
EXPECT_LE(0, frame.InsertPacket(packet, 0, kNoErrors, frame_data));
EXPECT_FALSE(dec_state.ContinuousFrame(&frame));
frame.Reset();
- packet.codecSpecificHeader.codecHeader.VP8.pictureId = 0;
+ packet.video_header.codecHeader.VP8.pictureId = 0;
packet.seqNum = 10;
EXPECT_LE(0, frame.InsertPacket(packet, 0, kNoErrors, frame_data));
EXPECT_TRUE(dec_state.ContinuousFrame(&frame));
// Use sequence numbers.
- packet.codecSpecificHeader.codecHeader.VP8.pictureId = kNoPictureId;
+ packet.video_header.codecHeader.VP8.pictureId = kNoPictureId;
frame.Reset();
packet.seqNum = dec_state.sequence_num() - 1u;
EXPECT_LE(0, frame.InsertPacket(packet, 0, kNoErrors, frame_data));
@@ -82,9 +82,9 @@
// Insert packet with temporal info.
dec_state.Reset();
frame.Reset();
- packet.codecSpecificHeader.codecHeader.VP8.tl0PicIdx = 0;
- packet.codecSpecificHeader.codecHeader.VP8.temporalIdx = 0;
- packet.codecSpecificHeader.codecHeader.VP8.pictureId = 0;
+ packet.video_header.codecHeader.VP8.tl0PicIdx = 0;
+ packet.video_header.codecHeader.VP8.temporalIdx = 0;
+ packet.video_header.codecHeader.VP8.pictureId = 0;
packet.seqNum = 1;
packet.timestamp = 1;
EXPECT_TRUE(dec_state.full_sync());
@@ -93,9 +93,9 @@
EXPECT_TRUE(dec_state.full_sync());
frame.Reset();
// 1 layer up - still good.
- packet.codecSpecificHeader.codecHeader.VP8.tl0PicIdx = 0;
- packet.codecSpecificHeader.codecHeader.VP8.temporalIdx = 1;
- packet.codecSpecificHeader.codecHeader.VP8.pictureId = 1;
+ packet.video_header.codecHeader.VP8.tl0PicIdx = 0;
+ packet.video_header.codecHeader.VP8.temporalIdx = 1;
+ packet.video_header.codecHeader.VP8.pictureId = 1;
packet.seqNum = 2;
packet.timestamp = 2;
EXPECT_LE(0, frame.InsertPacket(packet, 0, kNoErrors, frame_data));
@@ -104,18 +104,18 @@
EXPECT_TRUE(dec_state.full_sync());
frame.Reset();
// Lost non-base layer packet => should update sync parameter.
- packet.codecSpecificHeader.codecHeader.VP8.tl0PicIdx = 0;
- packet.codecSpecificHeader.codecHeader.VP8.temporalIdx = 3;
- packet.codecSpecificHeader.codecHeader.VP8.pictureId = 3;
+ packet.video_header.codecHeader.VP8.tl0PicIdx = 0;
+ packet.video_header.codecHeader.VP8.temporalIdx = 3;
+ packet.video_header.codecHeader.VP8.pictureId = 3;
packet.seqNum = 4;
packet.timestamp = 4;
EXPECT_LE(0, frame.InsertPacket(packet, 0, kNoErrors, frame_data));
EXPECT_FALSE(dec_state.ContinuousFrame(&frame));
// Now insert the next non-base layer (belonging to a next tl0PicId).
frame.Reset();
- packet.codecSpecificHeader.codecHeader.VP8.tl0PicIdx = 1;
- packet.codecSpecificHeader.codecHeader.VP8.temporalIdx = 2;
- packet.codecSpecificHeader.codecHeader.VP8.pictureId = 4;
+ packet.video_header.codecHeader.VP8.tl0PicIdx = 1;
+ packet.video_header.codecHeader.VP8.temporalIdx = 2;
+ packet.video_header.codecHeader.VP8.pictureId = 4;
packet.seqNum = 5;
packet.timestamp = 5;
EXPECT_LE(0, frame.InsertPacket(packet, 0, kNoErrors, frame_data));
@@ -125,9 +125,9 @@
EXPECT_TRUE(dec_state.full_sync());
// Next base layer (dropped interim non-base layers) - should update sync.
frame.Reset();
- packet.codecSpecificHeader.codecHeader.VP8.tl0PicIdx = 1;
- packet.codecSpecificHeader.codecHeader.VP8.temporalIdx = 0;
- packet.codecSpecificHeader.codecHeader.VP8.pictureId = 5;
+ packet.video_header.codecHeader.VP8.tl0PicIdx = 1;
+ packet.video_header.codecHeader.VP8.temporalIdx = 0;
+ packet.video_header.codecHeader.VP8.pictureId = 5;
packet.seqNum = 6;
packet.timestamp = 6;
EXPECT_LE(0, frame.InsertPacket(packet, 0, kNoErrors, frame_data));
@@ -137,18 +137,18 @@
// Check wrap for temporal layers.
frame.Reset();
- packet.codecSpecificHeader.codecHeader.VP8.tl0PicIdx = 0x00FF;
- packet.codecSpecificHeader.codecHeader.VP8.temporalIdx = 0;
- packet.codecSpecificHeader.codecHeader.VP8.pictureId = 6;
+ packet.video_header.codecHeader.VP8.tl0PicIdx = 0x00FF;
+ packet.video_header.codecHeader.VP8.temporalIdx = 0;
+ packet.video_header.codecHeader.VP8.pictureId = 6;
packet.seqNum = 7;
packet.timestamp = 7;
EXPECT_LE(0, frame.InsertPacket(packet, 0, kNoErrors, frame_data));
dec_state.SetState(&frame);
EXPECT_FALSE(dec_state.full_sync());
frame.Reset();
- packet.codecSpecificHeader.codecHeader.VP8.tl0PicIdx = 0x0000;
- packet.codecSpecificHeader.codecHeader.VP8.temporalIdx = 0;
- packet.codecSpecificHeader.codecHeader.VP8.pictureId = 7;
+ packet.video_header.codecHeader.VP8.tl0PicIdx = 0x0000;
+ packet.video_header.codecHeader.VP8.temporalIdx = 0;
+ packet.video_header.codecHeader.VP8.pictureId = 7;
packet.seqNum = 8;
packet.timestamp = 8;
EXPECT_LE(0, frame.InsertPacket(packet, 0, kNoErrors, frame_data));
@@ -211,12 +211,12 @@
VCMFrameBuffer frame;
VCMPacket packet;
packet.frameType = kVideoFrameDelta;
- packet.codecSpecificHeader.codec = kRtpVideoVp8;
+ packet.video_header.codec = kRtpVideoVp8;
packet.timestamp = 0;
packet.seqNum = 0;
- packet.codecSpecificHeader.codecHeader.VP8.tl0PicIdx = 0;
- packet.codecSpecificHeader.codecHeader.VP8.temporalIdx = 0;
- packet.codecSpecificHeader.codecHeader.VP8.pictureId = 0;
+ packet.video_header.codecHeader.VP8.tl0PicIdx = 0;
+ packet.video_header.codecHeader.VP8.temporalIdx = 0;
+ packet.video_header.codecHeader.VP8.pictureId = 0;
FrameData frame_data;
frame_data.rtt_ms = 0;
frame_data.rolling_average_packets_per_frame = -1;
@@ -226,9 +226,9 @@
frame.Reset();
packet.timestamp = 1;
packet.seqNum = 1;
- packet.codecSpecificHeader.codecHeader.VP8.tl0PicIdx = 0;
- packet.codecSpecificHeader.codecHeader.VP8.temporalIdx = 1;
- packet.codecSpecificHeader.codecHeader.VP8.pictureId = 1;
+ packet.video_header.codecHeader.VP8.tl0PicIdx = 0;
+ packet.video_header.codecHeader.VP8.temporalIdx = 1;
+ packet.video_header.codecHeader.VP8.pictureId = 1;
EXPECT_LE(0, frame.InsertPacket(packet, 0, kNoErrors, frame_data));
EXPECT_TRUE(dec_state.ContinuousFrame(&frame));
dec_state.SetState(&frame);
@@ -238,9 +238,9 @@
frame.Reset();
packet.timestamp = 3;
packet.seqNum = 3;
- packet.codecSpecificHeader.codecHeader.VP8.tl0PicIdx = 0;
- packet.codecSpecificHeader.codecHeader.VP8.temporalIdx = 3;
- packet.codecSpecificHeader.codecHeader.VP8.pictureId = 3;
+ packet.video_header.codecHeader.VP8.tl0PicIdx = 0;
+ packet.video_header.codecHeader.VP8.temporalIdx = 3;
+ packet.video_header.codecHeader.VP8.pictureId = 3;
EXPECT_LE(0, frame.InsertPacket(packet, 0, kNoErrors, frame_data));
EXPECT_FALSE(dec_state.ContinuousFrame(&frame));
dec_state.SetState(&frame);
@@ -249,9 +249,9 @@
frame.Reset();
packet.timestamp = 4;
packet.seqNum = 4;
- packet.codecSpecificHeader.codecHeader.VP8.tl0PicIdx = 1;
- packet.codecSpecificHeader.codecHeader.VP8.temporalIdx = 0;
- packet.codecSpecificHeader.codecHeader.VP8.pictureId = 4;
+ packet.video_header.codecHeader.VP8.tl0PicIdx = 1;
+ packet.video_header.codecHeader.VP8.temporalIdx = 0;
+ packet.video_header.codecHeader.VP8.pictureId = 4;
EXPECT_LE(0, frame.InsertPacket(packet, 0, kNoErrors, frame_data));
EXPECT_TRUE(dec_state.ContinuousFrame(&frame));
dec_state.SetState(&frame);
@@ -263,9 +263,9 @@
packet.isFirstPacket = 1;
packet.timestamp = 5;
packet.seqNum = 5;
- packet.codecSpecificHeader.codecHeader.VP8.tl0PicIdx = 2;
- packet.codecSpecificHeader.codecHeader.VP8.temporalIdx = 0;
- packet.codecSpecificHeader.codecHeader.VP8.pictureId = 5;
+ packet.video_header.codecHeader.VP8.tl0PicIdx = 2;
+ packet.video_header.codecHeader.VP8.temporalIdx = 0;
+ packet.video_header.codecHeader.VP8.pictureId = 5;
EXPECT_LE(0, frame.InsertPacket(packet, 0, kNoErrors, frame_data));
EXPECT_TRUE(dec_state.ContinuousFrame(&frame));
dec_state.SetState(&frame);
@@ -276,9 +276,9 @@
packet.frameType = kVideoFrameDelta;
packet.timestamp = 6;
packet.seqNum = 6;
- packet.codecSpecificHeader.codecHeader.VP8.tl0PicIdx = 3;
- packet.codecSpecificHeader.codecHeader.VP8.temporalIdx = 0;
- packet.codecSpecificHeader.codecHeader.VP8.pictureId = 6;
+ packet.video_header.codecHeader.VP8.tl0PicIdx = 3;
+ packet.video_header.codecHeader.VP8.temporalIdx = 0;
+ packet.video_header.codecHeader.VP8.pictureId = 6;
EXPECT_LE(0, frame.InsertPacket(packet, 0, kNoErrors, frame_data));
EXPECT_TRUE(dec_state.ContinuousFrame(&frame));
EXPECT_TRUE(dec_state.full_sync());
@@ -287,9 +287,9 @@
packet.isFirstPacket = 1;
packet.timestamp = 8;
packet.seqNum = 8;
- packet.codecSpecificHeader.codecHeader.VP8.tl0PicIdx = 4;
- packet.codecSpecificHeader.codecHeader.VP8.temporalIdx = 0;
- packet.codecSpecificHeader.codecHeader.VP8.pictureId = 8;
+ packet.video_header.codecHeader.VP8.tl0PicIdx = 4;
+ packet.video_header.codecHeader.VP8.temporalIdx = 0;
+ packet.video_header.codecHeader.VP8.pictureId = 8;
EXPECT_LE(0, frame.InsertPacket(packet, 0, kNoErrors, frame_data));
EXPECT_FALSE(dec_state.ContinuousFrame(&frame));
EXPECT_TRUE(dec_state.full_sync());
@@ -302,10 +302,10 @@
packet.isFirstPacket = 1;
packet.timestamp = 9;
packet.seqNum = 9;
- packet.codecSpecificHeader.codecHeader.VP8.tl0PicIdx = 4;
- packet.codecSpecificHeader.codecHeader.VP8.temporalIdx = 2;
- packet.codecSpecificHeader.codecHeader.VP8.pictureId = 9;
- packet.codecSpecificHeader.codecHeader.VP8.layerSync = true;
+ packet.video_header.codecHeader.VP8.tl0PicIdx = 4;
+ packet.video_header.codecHeader.VP8.temporalIdx = 2;
+ packet.video_header.codecHeader.VP8.pictureId = 9;
+ packet.video_header.codecHeader.VP8.layerSync = true;
EXPECT_LE(0, frame.InsertPacket(packet, 0, kNoErrors, frame_data));
dec_state.SetState(&frame);
EXPECT_TRUE(dec_state.full_sync());
@@ -323,10 +323,10 @@
packet.markerBit = 1;
packet.timestamp = 0;
packet.seqNum = 0;
- packet.codecSpecificHeader.codecHeader.VP8.tl0PicIdx = 0;
- packet.codecSpecificHeader.codecHeader.VP8.temporalIdx = 0;
- packet.codecSpecificHeader.codecHeader.VP8.pictureId = 0;
- packet.codecSpecificHeader.codecHeader.VP8.layerSync = false;
+ packet.video_header.codecHeader.VP8.tl0PicIdx = 0;
+ packet.video_header.codecHeader.VP8.temporalIdx = 0;
+ packet.video_header.codecHeader.VP8.pictureId = 0;
+ packet.video_header.codecHeader.VP8.layerSync = false;
EXPECT_LE(0, frame.InsertPacket(packet, 0, kNoErrors, frame_data));
dec_state.SetState(&frame);
EXPECT_TRUE(dec_state.full_sync());
@@ -337,10 +337,10 @@
packet.markerBit = 0;
packet.timestamp = 1;
packet.seqNum = 1;
- packet.codecSpecificHeader.codecHeader.VP8.tl0PicIdx = 0;
- packet.codecSpecificHeader.codecHeader.VP8.temporalIdx = 2;
- packet.codecSpecificHeader.codecHeader.VP8.pictureId = 1;
- packet.codecSpecificHeader.codecHeader.VP8.layerSync = true;
+ packet.video_header.codecHeader.VP8.tl0PicIdx = 0;
+ packet.video_header.codecHeader.VP8.temporalIdx = 2;
+ packet.video_header.codecHeader.VP8.pictureId = 1;
+ packet.video_header.codecHeader.VP8.layerSync = true;
EXPECT_LE(0, frame.InsertPacket(packet, 0, kNoErrors, frame_data));
EXPECT_TRUE(dec_state.ContinuousFrame(&frame));
// Layer 1
@@ -350,10 +350,10 @@
packet.markerBit = 1;
packet.timestamp = 2;
packet.seqNum = 3;
- packet.codecSpecificHeader.codecHeader.VP8.tl0PicIdx = 0;
- packet.codecSpecificHeader.codecHeader.VP8.temporalIdx = 1;
- packet.codecSpecificHeader.codecHeader.VP8.pictureId = 2;
- packet.codecSpecificHeader.codecHeader.VP8.layerSync = true;
+ packet.video_header.codecHeader.VP8.tl0PicIdx = 0;
+ packet.video_header.codecHeader.VP8.temporalIdx = 1;
+ packet.video_header.codecHeader.VP8.pictureId = 2;
+ packet.video_header.codecHeader.VP8.layerSync = true;
EXPECT_LE(0, frame.InsertPacket(packet, 0, kNoErrors, frame_data));
EXPECT_FALSE(dec_state.ContinuousFrame(&frame));
EXPECT_TRUE(dec_state.full_sync());
@@ -365,12 +365,12 @@
VCMPacket packet;
frame.Reset();
packet.frameType = kVideoFrameKey;
- packet.codecSpecificHeader.codec = kRtpVideoVp8;
+ packet.video_header.codec = kRtpVideoVp8;
packet.timestamp = 0;
packet.seqNum = 0;
- packet.codecSpecificHeader.codecHeader.VP8.tl0PicIdx = 0;
- packet.codecSpecificHeader.codecHeader.VP8.temporalIdx = 0;
- packet.codecSpecificHeader.codecHeader.VP8.pictureId = 0;
+ packet.video_header.codecHeader.VP8.tl0PicIdx = 0;
+ packet.video_header.codecHeader.VP8.temporalIdx = 0;
+ packet.video_header.codecHeader.VP8.pictureId = 0;
FrameData frame_data;
frame_data.rtt_ms = 0;
frame_data.rolling_average_packets_per_frame = -1;
@@ -384,8 +384,8 @@
packet.frameType = kVideoFrameDelta;
packet.timestamp += 3000;
++packet.seqNum;
- packet.codecSpecificHeader.codecHeader.VP8.temporalIdx = 1;
- packet.codecSpecificHeader.codecHeader.VP8.pictureId = 2;
+ packet.video_header.codecHeader.VP8.temporalIdx = 1;
+ packet.video_header.codecHeader.VP8.pictureId = 2;
EXPECT_LE(0, frame.InsertPacket(packet, 0, kNoErrors, frame_data));
EXPECT_FALSE(dec_state.ContinuousFrame(&frame));
dec_state.SetState(&frame);
@@ -418,12 +418,12 @@
VCMFrameBuffer frame;
VCMPacket packet;
packet.frameType = kVideoFrameDelta;
- packet.codecSpecificHeader.codec = kRtpVideoVp8;
+ packet.video_header.codec = kRtpVideoVp8;
packet.timestamp = 0;
packet.seqNum = 0;
- packet.codecSpecificHeader.codecHeader.VP8.tl0PicIdx = 0;
- packet.codecSpecificHeader.codecHeader.VP8.temporalIdx = 0;
- packet.codecSpecificHeader.codecHeader.VP8.pictureId = 0;
+ packet.video_header.codecHeader.VP8.tl0PicIdx = 0;
+ packet.video_header.codecHeader.VP8.temporalIdx = 0;
+ packet.video_header.codecHeader.VP8.pictureId = 0;
FrameData frame_data;
frame_data.rtt_ms = 0;
frame_data.rolling_average_packets_per_frame = -1;
@@ -433,15 +433,15 @@
frame.Reset();
++packet.timestamp;
++packet.seqNum;
- packet.codecSpecificHeader.codecHeader.VP8.temporalIdx++;
- packet.codecSpecificHeader.codecHeader.VP8.pictureId++;
+ packet.video_header.codecHeader.VP8.temporalIdx++;
+ packet.video_header.codecHeader.VP8.pictureId++;
EXPECT_LE(0, frame.InsertPacket(packet, 0, kNoErrors, frame_data));
EXPECT_TRUE(dec_state.ContinuousFrame(&frame));
frame.Reset();
// Testing only gap in tl0PicIdx when tl0PicIdx in continuous.
- packet.codecSpecificHeader.codecHeader.VP8.tl0PicIdx += 3;
- packet.codecSpecificHeader.codecHeader.VP8.temporalIdx++;
- packet.codecSpecificHeader.codecHeader.VP8.tl0PicIdx = 1;
+ packet.video_header.codecHeader.VP8.tl0PicIdx += 3;
+ packet.video_header.codecHeader.VP8.temporalIdx++;
+ packet.video_header.codecHeader.VP8.tl0PicIdx = 1;
EXPECT_LE(0, frame.InsertPacket(packet, 0, kNoErrors, frame_data));
EXPECT_FALSE(dec_state.ContinuousFrame(&frame));
}
@@ -456,9 +456,9 @@
uint8_t data[] = "I need a data pointer for this test!";
packet.sizeBytes = sizeof(data);
packet.dataPtr = data;
- packet.codecSpecificHeader.codec = kRtpVideoVp9;
+ packet.video_header.codec = kRtpVideoVp9;
- RTPVideoHeaderVP9& vp9_hdr = packet.codecSpecificHeader.codecHeader.VP9;
+ RTPVideoHeaderVP9& vp9_hdr = packet.video_header.codecHeader.VP9;
vp9_hdr.picture_id = 10;
vp9_hdr.flexible_mode = true;
@@ -499,9 +499,9 @@
uint8_t data[] = "I need a data pointer for this test!";
packet.sizeBytes = sizeof(data);
packet.dataPtr = data;
- packet.codecSpecificHeader.codec = kRtpVideoVp9;
+ packet.video_header.codec = kRtpVideoVp9;
- RTPVideoHeaderVP9& vp9_hdr = packet.codecSpecificHeader.codecHeader.VP9;
+ RTPVideoHeaderVP9& vp9_hdr = packet.video_header.codecHeader.VP9;
vp9_hdr.picture_id = 10;
vp9_hdr.flexible_mode = true;
@@ -554,9 +554,9 @@
uint8_t data[] = "I need a data pointer for this test!";
packet.sizeBytes = sizeof(data);
packet.dataPtr = data;
- packet.codecSpecificHeader.codec = kRtpVideoVp9;
+ packet.video_header.codec = kRtpVideoVp9;
- RTPVideoHeaderVP9& vp9_hdr = packet.codecSpecificHeader.codecHeader.VP9;
+ RTPVideoHeaderVP9& vp9_hdr = packet.video_header.codecHeader.VP9;
vp9_hdr.picture_id = 10;
vp9_hdr.flexible_mode = true;