Add a flags field to video timing extension.
The rtp header extension for video timing shuold have an additional
field for signaling metadata, such as what triggered the extension for
this particular frame. This will allow separating frames select because
of outlier sizes from regular frames, for more accurate stats.
This implementation is backwards compatible in that it can read video
timing extensions without the new flag field, but it always sends with
it included.
BUG=webrtc:7594
Review-Url: https://codereview.webrtc.org/3000753002
Cr-Commit-Position: refs/heads/master@{#19353}
diff --git a/webrtc/modules/video_coding/codecs/h264/h264_encoder_impl.cc b/webrtc/modules/video_coding/codecs/h264/h264_encoder_impl.cc
index e84dffd..f5a1910 100644
--- a/webrtc/modules/video_coding/codecs/h264/h264_encoder_impl.cc
+++ b/webrtc/modules/video_coding/codecs/h264/h264_encoder_impl.cc
@@ -370,7 +370,7 @@
encoded_image_.content_type_ = (mode_ == kScreensharing)
? VideoContentType::SCREENSHARE
: VideoContentType::UNSPECIFIED;
- encoded_image_.timing_.is_timing_frame = false;
+ encoded_image_.timing_.flags = TimingFrameFlags::kInvalid;
encoded_image_._frameType = ConvertToVideoFrameType(info.eFrameType);
// Split encoded image up into fragments. This also updates |encoded_image_|.
diff --git a/webrtc/modules/video_coding/codecs/vp8/vp8_impl.cc b/webrtc/modules/video_coding/codecs/vp8/vp8_impl.cc
index 3f9f879..a225036 100644
--- a/webrtc/modules/video_coding/codecs/vp8/vp8_impl.cc
+++ b/webrtc/modules/video_coding/codecs/vp8/vp8_impl.cc
@@ -900,7 +900,7 @@
encoded_images_[encoder_idx].content_type_ =
(codec_.mode == kScreensharing) ? VideoContentType::SCREENSHARE
: VideoContentType::UNSPECIFIED;
- encoded_images_[encoder_idx].timing_.is_timing_frame = false;
+ encoded_images_[encoder_idx].timing_.flags = TimingFrameFlags::kInvalid;
int qp = -1;
vpx_codec_control(&encoders_[encoder_idx], VP8E_GET_LAST_QUANTIZER_64, &qp);
diff --git a/webrtc/modules/video_coding/codecs/vp9/vp9_impl.cc b/webrtc/modules/video_coding/codecs/vp9/vp9_impl.cc
index e1776fc..fa3e623 100644
--- a/webrtc/modules/video_coding/codecs/vp9/vp9_impl.cc
+++ b/webrtc/modules/video_coding/codecs/vp9/vp9_impl.cc
@@ -710,7 +710,7 @@
: VideoContentType::UNSPECIFIED;
encoded_image_._encodedHeight = raw_->d_h;
encoded_image_._encodedWidth = raw_->d_w;
- encoded_image_.timing_.is_timing_frame = false;
+ encoded_image_.timing_.flags = TimingFrameFlags::kInvalid;
int qp = -1;
vpx_codec_control(encoder_, VP8E_GET_LAST_QUANTIZER, &qp);
encoded_image_.qp_ = qp;
diff --git a/webrtc/modules/video_coding/encoded_frame.cc b/webrtc/modules/video_coding/encoded_frame.cc
index ec7f560..bd407c7 100644
--- a/webrtc/modules/video_coding/encoded_frame.cc
+++ b/webrtc/modules/video_coding/encoded_frame.cc
@@ -88,7 +88,7 @@
_codec = kVideoCodecUnknown;
rotation_ = kVideoRotation_0;
content_type_ = VideoContentType::UNSPECIFIED;
- timing_.is_timing_frame = false;
+ timing_.flags = TimingFrameFlags::kInvalid;
_rotation_set = false;
}
diff --git a/webrtc/modules/video_coding/frame_buffer.cc b/webrtc/modules/video_coding/frame_buffer.cc
index f67210f..41c7fc8 100644
--- a/webrtc/modules/video_coding/frame_buffer.cc
+++ b/webrtc/modules/video_coding/frame_buffer.cc
@@ -164,8 +164,7 @@
rotation_ = packet.video_header.rotation;
_rotation_set = true;
content_type_ = packet.video_header.content_type;
- if (packet.video_header.video_timing.is_timing_frame) {
- timing_.is_timing_frame = true;
+ if (packet.video_header.video_timing.flags != TimingFrameFlags::kInvalid) {
timing_.encode_start_ms =
ntp_time_ms_ + packet.video_header.video_timing.encode_start_delta_ms;
timing_.encode_finish_ms =
@@ -182,9 +181,8 @@
timing_.network2_timestamp_ms =
ntp_time_ms_ +
packet.video_header.video_timing.network2_timstamp_delta_ms;
- } else {
- timing_.is_timing_frame = false;
}
+ timing_.flags = packet.video_header.video_timing.flags;
}
if (packet.is_first_packet_in_frame) {
diff --git a/webrtc/modules/video_coding/frame_object.cc b/webrtc/modules/video_coding/frame_object.cc
index 1d858fc..86bcffb 100644
--- a/webrtc/modules/video_coding/frame_object.cc
+++ b/webrtc/modules/video_coding/frame_object.cc
@@ -32,6 +32,7 @@
: packet_buffer_(packet_buffer),
first_seq_num_(first_seq_num),
last_seq_num_(last_seq_num),
+ timestamp_(0),
received_time_(received_time),
times_nacked_(times_nacked) {
VCMPacket* first_packet = packet_buffer_->GetPacket(first_seq_num);
@@ -113,10 +114,10 @@
rotation_ = last_packet->video_header.rotation;
_rotation_set = true;
content_type_ = last_packet->video_header.content_type;
- if (last_packet->video_header.video_timing.is_timing_frame) {
+ if (last_packet->video_header.video_timing.flags !=
+ TimingFrameFlags::kInvalid) {
// ntp_time_ms_ may be -1 if not estimated yet. This is not a problem,
// as this will be dealt with at the time of reporting.
- timing_.is_timing_frame = true;
timing_.encode_start_ms =
ntp_time_ms_ +
last_packet->video_header.video_timing.encode_start_delta_ms;
@@ -138,9 +139,8 @@
timing_.receive_start_ms = first_packet->receive_time_ms;
timing_.receive_finish_ms = last_packet->receive_time_ms;
- } else {
- timing_.is_timing_frame = false;
}
+ timing_.flags = last_packet->video_header.video_timing.flags;
}
RtpFrameObject::~RtpFrameObject() {
diff --git a/webrtc/modules/video_coding/generic_decoder.cc b/webrtc/modules/video_coding/generic_decoder.cc
index cc14794..3b2a620 100644
--- a/webrtc/modules/video_coding/generic_decoder.cc
+++ b/webrtc/modules/video_coding/generic_decoder.cc
@@ -92,7 +92,7 @@
frameInfo->renderTimeMs);
// Report timing information.
- if (frameInfo->timing.is_timing_frame) {
+ if (frameInfo->timing.flags != TimingFrameFlags::kInvalid) {
int64_t capture_time_ms = decodedImage.ntp_time_ms() - ntp_offset_;
// Convert remote timestamps to local time from ntp timestamps.
frameInfo->timing.encode_start_ms -= ntp_offset_;
@@ -137,6 +137,7 @@
timing_frame_info.decode_finish_ms = now_ms;
timing_frame_info.render_time_ms = frameInfo->renderTimeMs;
timing_frame_info.rtp_timestamp = decodedImage.timestamp();
+ timing_frame_info.flags = frameInfo->timing.flags;
_timing->SetTimingFrameInfo(timing_frame_info);
}
diff --git a/webrtc/modules/video_coding/generic_encoder.cc b/webrtc/modules/video_coding/generic_encoder.cc
index 78eeef2..4c2a699 100644
--- a/webrtc/modules/video_coding/generic_encoder.cc
+++ b/webrtc/modules/video_coding/generic_encoder.cc
@@ -231,7 +231,7 @@
rtc::Optional<size_t> outlier_frame_size;
rtc::Optional<int64_t> encode_start_ms;
- bool is_timing_frame = false;
+ uint8_t timing_flags = TimingFrameFlags::kInvalid;
{
rtc::CritScope crit(&timing_params_lock_);
@@ -274,14 +274,16 @@
if (last_timing_frame_time_ms_ == -1 ||
timing_frame_delay_ms >= timing_frames_thresholds_.delay_ms ||
timing_frame_delay_ms == 0) {
- is_timing_frame = true;
+ timing_flags = TimingFrameFlags::kTriggeredByTimer;
last_timing_frame_time_ms_ = encoded_image.capture_time_ms_;
}
// Outliers trigger timing frames, but do not affect scheduled timing
// frames.
if (outlier_frame_size && encoded_image._length >= *outlier_frame_size) {
- is_timing_frame = true;
+ if (timing_flags == TimingFrameFlags::kInvalid)
+ timing_flags = 0;
+ timing_flags |= TimingFrameFlags::kTriggeredBySize;
}
}
@@ -290,9 +292,10 @@
// drift relative to rtc::TimeMillis(). We can't use it for Timing frames,
// because to being sent in the network capture time required to be less than
// all the other timestamps.
- if (is_timing_frame && encode_start_ms) {
+ if (timing_flags != TimingFrameFlags::kInvalid && encode_start_ms) {
encoded_image.SetEncodeTime(*encode_start_ms, rtc::TimeMillis());
}
+ encoded_image.timing_.flags = timing_flags;
Result result = post_encode_callback_->OnEncodedImage(
encoded_image, codec_specific, fragmentation_header);
diff --git a/webrtc/modules/video_coding/generic_encoder_unittest.cc b/webrtc/modules/video_coding/generic_encoder_unittest.cc
index eec5b98..f3ca088 100644
--- a/webrtc/modules/video_coding/generic_encoder_unittest.cc
+++ b/webrtc/modules/video_coding/generic_encoder_unittest.cc
@@ -31,7 +31,8 @@
Result OnEncodedImage(const EncodedImage& encoded_image,
const CodecSpecificInfo* codec_specific_info,
const RTPFragmentationHeader* fragmentation) override {
- last_frame_was_timing_ = encoded_image.timing_.is_timing_frame;
+ last_frame_was_timing_ =
+ encoded_image.timing_.flags != TimingFrameFlags::kInvalid;
return Result(Result::OK);
};