Add fieldtrial to enable minimum pacing of video frames
If the RTP header extension playout-delay is used and set
to min=0, max>=0, frames are scheduled to be decoded as
soon as possible. There's a risk that too many frames are
sent to the decoder at once, which may cause problems
further down in the video pipeline.
This CL adds the fieldtrial WebRTC-ZeroPlayoutDelay with
the parameter min_pacing that determines the minimum
pacing interval between two frames scheduled for
decoding.
Bug: None
Change-Id: I471f7718761cfce9789b3aa8adea3e8a16ecb2fd
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/223742
Reviewed-by: Ilya Nikolaevskiy <ilnik@webrtc.org>
Commit-Queue: Johannes Kron <kron@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#34387}
diff --git a/modules/video_coding/frame_buffer2_unittest.cc b/modules/video_coding/frame_buffer2_unittest.cc
index 68acf81..d37efda 100644
--- a/modules/video_coding/frame_buffer2_unittest.cc
+++ b/modules/video_coding/frame_buffer2_unittest.cc
@@ -55,8 +55,7 @@
return last_ms_;
}
- int64_t MaxWaitingTime(int64_t render_time_ms,
- int64_t now_ms) const override {
+ int64_t MaxWaitingTime(int64_t render_time_ms, int64_t now_ms) override {
return render_time_ms - now_ms - kDecodeTime;
}
diff --git a/modules/video_coding/timing.cc b/modules/video_coding/timing.cc
index eddac4f..7ad5edf 100644
--- a/modules/video_coding/timing.cc
+++ b/modules/video_coding/timing.cc
@@ -34,9 +34,13 @@
prev_frame_timestamp_(0),
timing_frame_info_(),
num_decoded_frames_(0),
- low_latency_renderer_enabled_("enabled", true) {
+ low_latency_renderer_enabled_("enabled", true),
+ zero_playout_delay_min_pacing_("min_pacing", TimeDelta::Millis(0)),
+ earliest_next_decode_start_time_(0) {
ParseFieldTrial({&low_latency_renderer_enabled_},
field_trial::FindFullName("WebRTC-LowLatencyRenderer"));
+ ParseFieldTrial({&zero_playout_delay_min_pacing_},
+ field_trial::FindFullName("WebRTC-ZeroPlayoutDelay"));
}
void VCMTiming::Reset() {
@@ -199,14 +203,22 @@
return decode_time_ms;
}
-int64_t VCMTiming::MaxWaitingTime(int64_t render_time_ms,
- int64_t now_ms) const {
+int64_t VCMTiming::MaxWaitingTime(int64_t render_time_ms, int64_t now_ms) {
MutexLock lock(&mutex_);
- const int64_t max_wait_time_ms =
- render_time_ms - now_ms - RequiredDecodeTimeMs() - render_delay_ms_;
-
- return max_wait_time_ms;
+ if (render_time_ms == 0 && zero_playout_delay_min_pacing_->us() > 0) {
+ // |render_time_ms| == 0 indicates that the frame should be decoded and
+ // rendered as soon as possible. However, the decoder can be choked if too
+ // many frames are sent at ones. Therefore, limit the interframe delay to
+ // |zero_playout_delay_min_pacing_|.
+ int64_t max_wait_time_ms = now_ms >= earliest_next_decode_start_time_
+ ? 0
+ : earliest_next_decode_start_time_ - now_ms;
+ earliest_next_decode_start_time_ =
+ now_ms + max_wait_time_ms + zero_playout_delay_min_pacing_->ms();
+ return max_wait_time_ms;
+ }
+ return render_time_ms - now_ms - RequiredDecodeTimeMs() - render_delay_ms_;
}
int VCMTiming::TargetVideoDelay() const {
diff --git a/modules/video_coding/timing.h b/modules/video_coding/timing.h
index 736b5e9..1583082 100644
--- a/modules/video_coding/timing.h
+++ b/modules/video_coding/timing.h
@@ -14,6 +14,7 @@
#include <memory>
#include "absl/types/optional.h"
+#include "api/units/time_delta.h"
#include "api/video/video_timing.h"
#include "modules/video_coding/codec_timer.h"
#include "rtc_base/experiments/field_trial_parser.h"
@@ -82,7 +83,7 @@
// Returns the maximum time in ms that we can wait for a frame to become
// complete before we must pass it to the decoder.
- virtual int64_t MaxWaitingTime(int64_t render_time_ms, int64_t now_ms) const;
+ virtual int64_t MaxWaitingTime(int64_t render_time_ms, int64_t now_ms);
// Returns the current target delay which is required delay + decode time +
// render delay.
@@ -139,6 +140,15 @@
FieldTrialParameter<bool> low_latency_renderer_enabled_
RTC_GUARDED_BY(mutex_);
absl::optional<int> max_composition_delay_in_frames_ RTC_GUARDED_BY(mutex_);
+ // Set by the field trial WebRTC-ZeroPlayoutDelay. The parameter min_pacing
+ // determines the minimum delay between frames scheduled for decoding that is
+ // used when min playout delay=0 and max playout delay>=0.
+ FieldTrialParameter<TimeDelta> zero_playout_delay_min_pacing_
+ RTC_GUARDED_BY(mutex_);
+ // An estimate of when the last frame is scheduled to be sent to the decoder.
+ // Used only when the RTP header extension playout delay is set to min=0 ms
+ // which is indicated by a render time set to 0.
+ int64_t earliest_next_decode_start_time_ RTC_GUARDED_BY(mutex_);
};
} // namespace webrtc
diff --git a/modules/video_coding/timing_unittest.cc b/modules/video_coding/timing_unittest.cc
index ee86605..be6ac52 100644
--- a/modules/video_coding/timing_unittest.cc
+++ b/modules/video_coding/timing_unittest.cc
@@ -11,6 +11,7 @@
#include "modules/video_coding/timing.h"
#include "system_wrappers/include/clock.h"
+#include "test/field_trial.h"
#include "test/gtest.h"
namespace webrtc {
@@ -18,7 +19,7 @@
const int kFps = 25;
} // namespace
-TEST(ReceiverTiming, Tests) {
+TEST(ReceiverTimingTest, JitterDelay) {
SimulatedClock clock(0);
VCMTiming timing(&clock);
timing.Reset();
@@ -110,7 +111,7 @@
timing.UpdateCurrentDelay(timestamp);
}
-TEST(ReceiverTiming, WrapAround) {
+TEST(ReceiverTimingTest, TimestampWrapAround) {
SimulatedClock clock(0);
VCMTiming timing(&clock);
// Provoke a wrap-around. The fifth frame will have wrapped at 25 fps.
@@ -127,4 +128,89 @@
}
}
+TEST(ReceiverTimingTest, MaxWaitingTimeIsZeroForZeroRenderTime) {
+ // This is the default path when the RTP playout delay header extension is set
+ // to min==0.
+ constexpr int64_t kStartTimeUs = 3.15e13; // About one year in us.
+ constexpr int64_t kTimeDeltaMs = 1000.0 / 60.0;
+ constexpr int64_t kZeroRenderTimeMs = 0;
+ SimulatedClock clock(kStartTimeUs);
+ VCMTiming timing(&clock);
+ timing.Reset();
+ for (int i = 0; i < 10; ++i) {
+ clock.AdvanceTimeMilliseconds(kTimeDeltaMs);
+ int64_t now_ms = clock.TimeInMilliseconds();
+ EXPECT_LT(timing.MaxWaitingTime(kZeroRenderTimeMs, now_ms), 0);
+ }
+ // Another frame submitted at the same time also returns a negative max
+ // waiting time.
+ int64_t now_ms = clock.TimeInMilliseconds();
+ EXPECT_LT(timing.MaxWaitingTime(kZeroRenderTimeMs, now_ms), 0);
+ // MaxWaitingTime should be less than zero even if there's a burst of frames.
+ EXPECT_LT(timing.MaxWaitingTime(kZeroRenderTimeMs, now_ms), 0);
+ EXPECT_LT(timing.MaxWaitingTime(kZeroRenderTimeMs, now_ms), 0);
+ EXPECT_LT(timing.MaxWaitingTime(kZeroRenderTimeMs, now_ms), 0);
+}
+
+TEST(ReceiverTimingTest, MaxWaitingTimeZeroDelayPacingExperiment) {
+ // The minimum pacing is enabled by a field trial and active if the RTP
+ // playout delay header extension is set to min==0.
+ constexpr int64_t kMinPacingMs = 3;
+ test::ScopedFieldTrials override_field_trials(
+ "WebRTC-ZeroPlayoutDelay/min_pacing:3ms/");
+ constexpr int64_t kStartTimeUs = 3.15e13; // About one year in us.
+ constexpr int64_t kTimeDeltaMs = 1000.0 / 60.0;
+ constexpr int64_t kZeroRenderTimeMs = 0;
+ SimulatedClock clock(kStartTimeUs);
+ VCMTiming timing(&clock);
+ timing.Reset();
+ // MaxWaitingTime() returns zero for evenly spaced video frames.
+ for (int i = 0; i < 10; ++i) {
+ clock.AdvanceTimeMilliseconds(kTimeDeltaMs);
+ int64_t now_ms = clock.TimeInMilliseconds();
+ EXPECT_EQ(timing.MaxWaitingTime(kZeroRenderTimeMs, now_ms), 0);
+ }
+ // Another frame submitted at the same time is paced according to the field
+ // trial setting.
+ int64_t now_ms = clock.TimeInMilliseconds();
+ EXPECT_EQ(timing.MaxWaitingTime(kZeroRenderTimeMs, now_ms), kMinPacingMs);
+ // If there's a burst of frames, the min pacing interval is summed.
+ EXPECT_EQ(timing.MaxWaitingTime(kZeroRenderTimeMs, now_ms), 2 * kMinPacingMs);
+ EXPECT_EQ(timing.MaxWaitingTime(kZeroRenderTimeMs, now_ms), 3 * kMinPacingMs);
+ EXPECT_EQ(timing.MaxWaitingTime(kZeroRenderTimeMs, now_ms), 4 * kMinPacingMs);
+ // Allow a few ms to pass, this should be subtracted from the MaxWaitingTime.
+ constexpr int64_t kTwoMs = 2;
+ clock.AdvanceTimeMilliseconds(kTwoMs);
+ now_ms = clock.TimeInMilliseconds();
+ EXPECT_EQ(timing.MaxWaitingTime(kZeroRenderTimeMs, now_ms),
+ 5 * kMinPacingMs - kTwoMs);
+}
+
+TEST(ReceiverTimingTest, DefaultMaxWaitingTimeUnaffectedByPacingExperiment) {
+ // The minimum pacing is enabled by a field trial but should not have any
+ // effect if render_time_ms is greater than 0;
+ test::ScopedFieldTrials override_field_trials(
+ "WebRTC-ZeroPlayoutDelay/min_pacing:3ms/");
+ constexpr int64_t kStartTimeUs = 3.15e13; // About one year in us.
+ constexpr int64_t kTimeDeltaMs = 1000.0 / 60.0;
+ SimulatedClock clock(kStartTimeUs);
+ VCMTiming timing(&clock);
+ timing.Reset();
+ clock.AdvanceTimeMilliseconds(kTimeDeltaMs);
+ int64_t now_ms = clock.TimeInMilliseconds();
+ int64_t render_time_ms = now_ms + 30;
+ // Estimate the internal processing delay from the first frame.
+ int64_t estimated_processing_delay =
+ (render_time_ms - now_ms) - timing.MaxWaitingTime(render_time_ms, now_ms);
+ EXPECT_GT(estimated_processing_delay, 0);
+
+ // Any other frame submitted at the same time should be scheduled according to
+ // its render time.
+ for (int i = 0; i < 5; ++i) {
+ render_time_ms += kTimeDeltaMs;
+ EXPECT_EQ(timing.MaxWaitingTime(render_time_ms, now_ms),
+ render_time_ms - now_ms - estimated_processing_delay);
+ }
+}
+
} // namespace webrtc