Add low-latency stream signaling to VideoFrame and VCMTiming
This is the first CL out of three to make the low-latency stream signaling
explicit. At the moment this is done by setting the render time to 0.
There's a dependency between Chromium and WebRTC which is why this is
split into three CLs to not break any existing functionality.
Bug: chromium:1327251
Change-Id: Ie6b268746d587a99334485db77181fb2c6e9b567
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/264502
Reviewed-by: Evan Shrubsole <eshr@webrtc.org>
Reviewed-by: Harald Alvestrand <hta@webrtc.org>
Commit-Queue: Johannes Kron <kron@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#37225}
diff --git a/modules/video_coding/generic_decoder.cc b/modules/video_coding/generic_decoder.cc
index 7070fa2..68eb1fc 100644
--- a/modules/video_coding/generic_decoder.cc
+++ b/modules/video_coding/generic_decoder.cc
@@ -104,17 +104,14 @@
decodedImage.set_ntp_time_ms(frameInfo->ntp_time_ms);
decodedImage.set_packet_infos(frameInfo->packet_infos);
decodedImage.set_rotation(frameInfo->rotation);
-
- absl::optional<int> max_composition_delay_in_frames =
- _timing->MaxCompositionDelayInFrames();
- if (max_composition_delay_in_frames) {
+ VideoFrame::RenderParameters render_parameters = _timing->RenderParameters();
+ if (render_parameters.max_composition_delay_in_frames) {
// Subtract frames that are in flight.
- *max_composition_delay_in_frames -= timestamp_map_size;
- *max_composition_delay_in_frames =
- std::max(0, *max_composition_delay_in_frames);
- decodedImage.set_max_composition_delay_in_frames(
- max_composition_delay_in_frames);
+ render_parameters.max_composition_delay_in_frames =
+ std::max(0, *render_parameters.max_composition_delay_in_frames -
+ timestamp_map_size);
}
+ decodedImage.set_render_parameters(render_parameters);
RTC_DCHECK(frameInfo->decodeStart);
const Timestamp now = _clock->CurrentTime();
diff --git a/modules/video_coding/generic_decoder_unittest.cc b/modules/video_coding/generic_decoder_unittest.cc
index 20ea9c3..65e8dba 100644
--- a/modules/video_coding/generic_decoder_unittest.cc
+++ b/modules/video_coding/generic_decoder_unittest.cc
@@ -124,23 +124,43 @@
generic_decoder_.Decode(encoded_frame, clock_.CurrentTime());
absl::optional<VideoFrame> decoded_frame = user_callback_.WaitForFrame(10);
ASSERT_TRUE(decoded_frame.has_value());
- EXPECT_FALSE(decoded_frame->max_composition_delay_in_frames());
+ EXPECT_THAT(
+ decoded_frame->render_parameters().max_composition_delay_in_frames,
+ testing::Eq(absl::nullopt));
}
TEST_F(GenericDecoderTest, MaxCompositionDelayActivatedByPlayoutDelay) {
VCMEncodedFrame encoded_frame;
// VideoReceiveStream2 would set MaxCompositionDelayInFrames if playout delay
// is specified as X,Y, where X=0, Y>0.
- const VideoPlayoutDelay kPlayoutDelay = {0, 50};
constexpr int kMaxCompositionDelayInFrames = 3; // ~50 ms at 60 fps.
- encoded_frame.SetPlayoutDelay(kPlayoutDelay);
timing_.SetMaxCompositionDelayInFrames(
absl::make_optional(kMaxCompositionDelayInFrames));
generic_decoder_.Decode(encoded_frame, clock_.CurrentTime());
absl::optional<VideoFrame> decoded_frame = user_callback_.WaitForFrame(10);
ASSERT_TRUE(decoded_frame.has_value());
- EXPECT_EQ(kMaxCompositionDelayInFrames,
- decoded_frame->max_composition_delay_in_frames());
+ EXPECT_THAT(
+ decoded_frame->render_parameters().max_composition_delay_in_frames,
+ testing::Optional(kMaxCompositionDelayInFrames));
+}
+
+TEST_F(GenericDecoderTest, IsLowLatencyStreamFalseByDefault) {
+ VCMEncodedFrame encoded_frame;
+ generic_decoder_.Decode(encoded_frame, clock_.CurrentTime());
+ absl::optional<VideoFrame> decoded_frame = user_callback_.WaitForFrame(10);
+ ASSERT_TRUE(decoded_frame.has_value());
+ EXPECT_FALSE(decoded_frame->render_parameters().use_low_latency_rendering);
+}
+
+TEST_F(GenericDecoderTest, IsLowLatencyStreamActivatedByPlayoutDelay) {
+ VCMEncodedFrame encoded_frame;
+ const VideoPlayoutDelay kPlayoutDelay = {0, 50};
+ timing_.set_min_playout_delay(TimeDelta::Millis(kPlayoutDelay.min_ms));
+ timing_.set_max_playout_delay(TimeDelta::Millis(kPlayoutDelay.max_ms));
+ generic_decoder_.Decode(encoded_frame, clock_.CurrentTime());
+ absl::optional<VideoFrame> decoded_frame = user_callback_.WaitForFrame(10);
+ ASSERT_TRUE(decoded_frame.has_value());
+ EXPECT_TRUE(decoded_frame->render_parameters().use_low_latency_rendering);
}
} // namespace video_coding
diff --git a/modules/video_coding/timing/BUILD.gn b/modules/video_coding/timing/BUILD.gn
index 369a680..b6415f5 100644
--- a/modules/video_coding/timing/BUILD.gn
+++ b/modules/video_coding/timing/BUILD.gn
@@ -71,6 +71,7 @@
":codec_timer",
"../../../api:field_trials_view",
"../../../api/units:time_delta",
+ "../../../api/video:video_frame",
"../../../api/video:video_rtp_headers",
"../../../rtc_base:logging",
"../../../rtc_base:macromagic",
diff --git a/modules/video_coding/timing/timing.cc b/modules/video_coding/timing/timing.cc
index 06754b2..0975195 100644
--- a/modules/video_coding/timing/timing.cc
+++ b/modules/video_coding/timing/timing.cc
@@ -23,7 +23,8 @@
// Default pacing that is used for the low-latency renderer path.
constexpr TimeDelta kZeroPlayoutDelayDefaultMinPacing = TimeDelta::Millis(8);
-constexpr TimeDelta kLowLatencyRendererMaxPlayoutDelay = TimeDelta::Millis(500);
+constexpr TimeDelta kLowLatencyStreamMaxPlayoutDelayThreshold =
+ TimeDelta::Millis(500);
void CheckDelaysValid(TimeDelta min_delay, TimeDelta max_delay) {
if (min_delay > max_delay) {
@@ -191,9 +192,7 @@
Timestamp VCMTiming::RenderTimeInternal(uint32_t frame_timestamp,
Timestamp now) const {
- if (min_playout_delay_.IsZero() &&
- (max_playout_delay_.IsZero() ||
- max_playout_delay_ <= kLowLatencyRendererMaxPlayoutDelay)) {
+ if (UseLowLatencyRendering()) {
// Render as soon as possible or with low-latency renderer algorithm.
return Timestamp::Zero();
}
@@ -250,6 +249,21 @@
jitter_delay_ + RequiredDecodeTime() + render_delay_);
}
+VideoFrame::RenderParameters VCMTiming::RenderParameters() const {
+ MutexLock lock(&mutex_);
+ return {.use_low_latency_rendering = UseLowLatencyRendering(),
+ .max_composition_delay_in_frames = max_composition_delay_in_frames_};
+}
+
+bool VCMTiming::UseLowLatencyRendering() const {
+ // min_playout_delay_==0,
+ // max_playout_delay_<=kLowLatencyStreamMaxPlayoutDelayThreshold indicates
+ // that the low-latency path should be used, which means that frames should be
+ // decoded and rendered as soon as possible.
+ return min_playout_delay_.IsZero() &&
+ max_playout_delay_ <= kLowLatencyStreamMaxPlayoutDelayThreshold;
+}
+
VCMTiming::VideoDelayTimings VCMTiming::GetTimings() const {
MutexLock lock(&mutex_);
return VideoDelayTimings{.max_decode_duration = RequiredDecodeTime(),
@@ -278,9 +292,4 @@
max_composition_delay_in_frames_ = max_composition_delay_in_frames;
}
-absl::optional<int> VCMTiming::MaxCompositionDelayInFrames() const {
- MutexLock lock(&mutex_);
- return max_composition_delay_in_frames_;
-}
-
} // namespace webrtc
diff --git a/modules/video_coding/timing/timing.h b/modules/video_coding/timing/timing.h
index 94950ba..6ee1cf4 100644
--- a/modules/video_coding/timing/timing.h
+++ b/modules/video_coding/timing/timing.h
@@ -16,6 +16,7 @@
#include "absl/types/optional.h"
#include "api/field_trials_view.h"
#include "api/units/time_delta.h"
+#include "api/video/video_frame.h"
#include "api/video/video_timing.h"
#include "modules/video_coding/timing/codec_timer.h"
#include "rtc_base/experiments/field_trial_parser.h"
@@ -111,7 +112,8 @@
void SetMaxCompositionDelayInFrames(
absl::optional<int> max_composition_delay_in_frames);
- absl::optional<int> MaxCompositionDelayInFrames() const;
+
+ VideoFrame::RenderParameters RenderParameters() const;
// Updates the last time a frame was scheduled for decoding.
void SetLastDecodeScheduledTimestamp(Timestamp last_decode_scheduled);
@@ -121,6 +123,7 @@
Timestamp RenderTimeInternal(uint32_t frame_timestamp, Timestamp now) const
RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_);
TimeDelta TargetDelayInternal() const RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_);
+ bool UseLowLatencyRendering() const RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_);
private:
mutable Mutex mutex_;
diff --git a/modules/video_coding/timing/timing_unittest.cc b/modules/video_coding/timing/timing_unittest.cc
index 31ee44a..e110071 100644
--- a/modules/video_coding/timing/timing_unittest.cc
+++ b/modules/video_coding/timing/timing_unittest.cc
@@ -138,6 +138,31 @@
}
}
+TEST(ReceiverTimingTest, UseLowLatencyRenderer) {
+ test::ScopedKeyValueConfig field_trials;
+ SimulatedClock clock(0);
+ VCMTiming timing(&clock, field_trials);
+ timing.Reset();
+ // Default is false.
+ EXPECT_FALSE(timing.RenderParameters().use_low_latency_rendering);
+ // False if min playout delay > 0.
+ timing.set_min_playout_delay(TimeDelta::Millis(10));
+ timing.set_max_playout_delay(TimeDelta::Millis(20));
+ EXPECT_FALSE(timing.RenderParameters().use_low_latency_rendering);
+ // True if min==0, max > 0.
+ timing.set_min_playout_delay(TimeDelta::Millis(0));
+ EXPECT_TRUE(timing.RenderParameters().use_low_latency_rendering);
+ // True if min==max==0.
+ timing.set_max_playout_delay(TimeDelta::Millis(0));
+ EXPECT_TRUE(timing.RenderParameters().use_low_latency_rendering);
+ // True also for max playout delay==500 ms.
+ timing.set_max_playout_delay(TimeDelta::Millis(500));
+ EXPECT_TRUE(timing.RenderParameters().use_low_latency_rendering);
+ // False if max playout delay > 500 ms.
+ timing.set_max_playout_delay(TimeDelta::Millis(501));
+ EXPECT_FALSE(timing.RenderParameters().use_low_latency_rendering);
+}
+
TEST(ReceiverTimingTest, MaxWaitingTimeIsZeroForZeroRenderTime) {
// This is the default path when the RTP playout delay header extension is set
// to min==0 and max==0.