Move VP9 frame rate controller to separate class.
Bug: webrtc:9669
Change-Id: I6f30587778e9783182af11d2410464024918e171
Reviewed-on: https://webrtc-review.googlesource.com/96201
Commit-Queue: Sergey Silkin <ssilkin@webrtc.org>
Reviewed-by: Erik Språng <sprang@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#24487}diff --git a/modules/video_coding/BUILD.gn b/modules/video_coding/BUILD.gn
index 0b76020..d4594f6 100644
--- a/modules/video_coding/BUILD.gn
+++ b/modules/video_coding/BUILD.gn
@@ -234,6 +234,8 @@
"utility/default_video_bitrate_allocator.h",
"utility/frame_dropper.cc",
"utility/frame_dropper.h",
+ "utility/framerate_controller.cc",
+ "utility/framerate_controller.h",
"utility/ivf_file_writer.cc",
"utility/ivf_file_writer.h",
"utility/moving_average.cc",
@@ -799,6 +801,7 @@
"timing_unittest.cc",
"utility/default_video_bitrate_allocator_unittest.cc",
"utility/frame_dropper_unittest.cc",
+ "utility/framerate_controller_unittest.cc",
"utility/ivf_file_writer_unittest.cc",
"utility/mock/mock_frame_dropper.h",
"utility/moving_average_unittest.cc",
diff --git a/modules/video_coding/codecs/vp9/vp9_impl.cc b/modules/video_coding/codecs/vp9/vp9_impl.cc
index b4c13f9..5805563 100644
--- a/modules/video_coding/codecs/vp9/vp9_impl.cc
+++ b/modules/video_coding/codecs/vp9/vp9_impl.cc
@@ -157,8 +157,7 @@
num_spatial_layers_(0),
is_svc_(false),
inter_layer_pred_(InterLayerPredMode::kOn),
- output_framerate_(1000.0, 1000.0),
- last_encoded_frame_rtp_timestamp_(0),
+ framerate_controller_(kMaxScreenSharingFramerateFps),
is_flexible_mode_(false) {
memset(&codec_, 0, sizeof(codec_));
memset(&svc_params_, 0, sizeof(vpx_svc_extra_cfg_t));
@@ -358,11 +357,9 @@
num_temporal_layers_ = 1;
// Init framerate controller.
- output_framerate_.Reset();
if (codec_.mode == VideoCodecMode::kScreensharing) {
- target_framerate_fps_ = kMaxScreenSharingFramerateFps;
- } else {
- target_framerate_fps_.reset();
+ framerate_controller_.Reset();
+ framerate_controller_.SetTargetRate(kMaxScreenSharingFramerateFps);
}
is_svc_ = (num_spatial_layers_ > 1 || num_temporal_layers_ > 1);
@@ -672,7 +669,8 @@
}
if (VideoCodecMode::kScreensharing == codec_.mode && !force_key_frame_) {
- if (DropFrame(input_image.timestamp())) {
+ if (framerate_controller_.DropFrame(1000 * input_image.timestamp() /
+ kVideoPayloadTypeFrequency)) {
return WEBRTC_VIDEO_CODEC_OK;
}
}
@@ -734,8 +732,10 @@
}
RTC_CHECK_GT(codec_.maxFramerate, 0);
- uint32_t duration =
- 90000 / target_framerate_fps_.value_or(codec_.maxFramerate);
+ uint32_t target_framerate_fps = codec_.mode == VideoCodecMode::kScreensharing
+ ? kMaxScreenSharingFramerateFps
+ : codec_.maxFramerate;
+ uint32_t duration = 90000 / target_framerate_fps;
const vpx_codec_err_t rv = vpx_codec_encode(encoder_, raw_, timestamp_,
duration, flags, VPX_DL_REALTIME);
if (rv != VPX_CODEC_OK) {
@@ -1064,45 +1064,14 @@
&frag_info);
encoded_image_._length = 0;
- if (end_of_picture) {
+ if (end_of_picture && codec_.mode == VideoCodecMode::kScreensharing) {
const uint32_t timestamp_ms =
1000 * encoded_image_.Timestamp() / kVideoPayloadTypeFrequency;
- output_framerate_.Update(1, timestamp_ms);
- last_encoded_frame_rtp_timestamp_ = encoded_image_.Timestamp();
+ framerate_controller_.AddFrame(timestamp_ms);
}
}
}
-bool VP9EncoderImpl::DropFrame(uint32_t rtp_timestamp) {
- if (target_framerate_fps_) {
- if (rtp_timestamp < last_encoded_frame_rtp_timestamp_) {
- // Timestamp has wrapped around. Reset framerate statistic.
- output_framerate_.Reset();
- return false;
- }
-
- const uint32_t timestamp_ms =
- 1000 * rtp_timestamp / kVideoPayloadTypeFrequency;
- const uint32_t framerate_fps =
- output_framerate_.Rate(timestamp_ms).value_or(0);
- if (framerate_fps > *target_framerate_fps_) {
- return true;
- }
-
- // Primarily check if frame interval is too short using frame timestamps,
- // as if they are correct they won't be affected by queuing in webrtc.
- const uint32_t expected_frame_interval =
- kVideoPayloadTypeFrequency / *target_framerate_fps_;
-
- const uint32_t ts_diff = rtp_timestamp - last_encoded_frame_rtp_timestamp_;
- if (ts_diff < 85 * expected_frame_interval / 100) {
- return true;
- }
- }
-
- return false;
-}
-
int VP9EncoderImpl::SetChannelParameters(uint32_t packet_loss, int64_t rtt) {
return WEBRTC_VIDEO_CODEC_OK;
}
diff --git a/modules/video_coding/codecs/vp9/vp9_impl.h b/modules/video_coding/codecs/vp9/vp9_impl.h
index d448306..e8fd606 100644
--- a/modules/video_coding/codecs/vp9/vp9_impl.h
+++ b/modules/video_coding/codecs/vp9/vp9_impl.h
@@ -20,7 +20,7 @@
#include "media/base/vp9_profile.h"
#include "modules/video_coding/codecs/vp9/vp9_frame_buffer_pool.h"
-#include "rtc_base/rate_statistics.h"
+#include "modules/video_coding/utility/framerate_controller.h"
#include "vpx/vp8cx.h"
#include "vpx/vpx_decoder.h"
@@ -82,7 +82,7 @@
void DeliverBufferedFrame(bool end_of_picture);
- bool DropFrame(uint32_t rtp_timestamp);
+ bool DropFrame(uint8_t spatial_idx, uint32_t rtp_timestamp);
// Determine maximum target for Intra frames
//
@@ -117,9 +117,7 @@
InterLayerPredMode inter_layer_pred_;
// Framerate controller.
- absl::optional<float> target_framerate_fps_;
- RateStatistics output_framerate_;
- uint32_t last_encoded_frame_rtp_timestamp_;
+ FramerateController framerate_controller_;
// Used for flexible mode.
bool is_flexible_mode_;
diff --git a/modules/video_coding/utility/framerate_controller.cc b/modules/video_coding/utility/framerate_controller.cc
new file mode 100644
index 0000000..ce8ab12
--- /dev/null
+++ b/modules/video_coding/utility/framerate_controller.cc
@@ -0,0 +1,81 @@
+/*
+ * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "modules/video_coding/utility/framerate_controller.h"
+
+#include "rtc_base/checks.h"
+
+namespace webrtc {
+
+FramerateController::FramerateController(float target_framerate_fps)
+ : min_frame_interval_ms_(0), framerate_estimator_(1000.0, 1000.0) {
+ SetTargetRate(target_framerate_fps);
+}
+
+void FramerateController::SetTargetRate(float target_framerate_fps) {
+ if (target_framerate_fps_ != target_framerate_fps) {
+ framerate_estimator_.Reset();
+ if (last_timestamp_ms_) {
+ framerate_estimator_.Update(1, *last_timestamp_ms_);
+ }
+
+ const size_t target_frame_interval_ms = 1000 / target_framerate_fps;
+ target_framerate_fps_ = target_framerate_fps;
+ min_frame_interval_ms_ = 85 * target_frame_interval_ms / 100;
+ }
+}
+
+float FramerateController::GetTargetRate() {
+ return *target_framerate_fps_;
+}
+
+void FramerateController::Reset() {
+ framerate_estimator_.Reset();
+ last_timestamp_ms_.reset();
+}
+
+bool FramerateController::DropFrame(uint32_t timestamp_ms) const {
+ if (timestamp_ms < last_timestamp_ms_) {
+ // Timestamp jumps backward. We can't make adequate drop decision. Don't
+ // drop this frame. Stats will be reset in AddFrame().
+ return false;
+ }
+
+ if (Rate(timestamp_ms).value_or(*target_framerate_fps_) >
+ target_framerate_fps_) {
+ return true;
+ }
+
+ if (last_timestamp_ms_) {
+ const int64_t diff_ms =
+ static_cast<int64_t>(timestamp_ms) - *last_timestamp_ms_;
+ if (diff_ms < min_frame_interval_ms_) {
+ return true;
+ }
+ }
+
+ return false;
+}
+
+void FramerateController::AddFrame(uint32_t timestamp_ms) {
+ if (timestamp_ms < last_timestamp_ms_) {
+ // Timestamp jumps backward.
+ Reset();
+ }
+
+ framerate_estimator_.Update(1, timestamp_ms);
+ last_timestamp_ms_ = timestamp_ms;
+}
+
+absl::optional<float> FramerateController::Rate(uint32_t timestamp_ms) const {
+ return framerate_estimator_.Rate(timestamp_ms);
+}
+
+} // namespace webrtc
diff --git a/modules/video_coding/utility/framerate_controller.h b/modules/video_coding/utility/framerate_controller.h
new file mode 100644
index 0000000..63493ea
--- /dev/null
+++ b/modules/video_coding/utility/framerate_controller.h
@@ -0,0 +1,44 @@
+/*
+ * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef MODULES_VIDEO_CODING_UTILITY_FRAMERATE_CONTROLLER_H_
+#define MODULES_VIDEO_CODING_UTILITY_FRAMERATE_CONTROLLER_H_
+
+#include "absl/types/optional.h"
+#include "rtc_base/rate_statistics.h"
+
+namespace webrtc {
+
+class FramerateController {
+ public:
+ explicit FramerateController(float target_framerate_fps);
+
+ void SetTargetRate(float target_framerate_fps);
+ float GetTargetRate();
+
+ // Advices user to drop next frame in order to reach target framerate.
+ bool DropFrame(uint32_t timestamp_ms) const;
+
+ void AddFrame(uint32_t timestamp_ms);
+
+ void Reset();
+
+ private:
+ absl::optional<float> Rate(uint32_t timestamp_ms) const;
+
+ absl::optional<float> target_framerate_fps_;
+ absl::optional<uint32_t> last_timestamp_ms_;
+ uint32_t min_frame_interval_ms_;
+ RateStatistics framerate_estimator_;
+};
+
+} // namespace webrtc
+
+#endif // MODULES_VIDEO_CODING_UTILITY_FRAMERATE_CONTROLLER_H_
diff --git a/modules/video_coding/utility/framerate_controller_unittest.cc b/modules/video_coding/utility/framerate_controller_unittest.cc
new file mode 100644
index 0000000..6c06ea3
--- /dev/null
+++ b/modules/video_coding/utility/framerate_controller_unittest.cc
@@ -0,0 +1,88 @@
+/*
+ * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "modules/video_coding/utility/framerate_controller.h"
+
+#include "test/gtest.h"
+
+namespace webrtc {
+
+TEST(FramerateController, KeepTargetFramerate) {
+ const float input_framerate_fps = 20;
+ const float target_framerate_fps = 5;
+ const float max_abs_framerate_error_fps = target_framerate_fps * 0.1f;
+ const size_t input_duration_secs = 3;
+ const size_t num_input_frames = input_duration_secs * input_framerate_fps;
+
+ FramerateController framerate_controller(target_framerate_fps);
+ size_t num_dropped_frames = 0;
+ for (size_t frame_num = 0; frame_num < num_input_frames; ++frame_num) {
+ const uint32_t timestamp_ms =
+ static_cast<uint32_t>(1000 * frame_num / input_framerate_fps);
+ if (framerate_controller.DropFrame(timestamp_ms)) {
+ ++num_dropped_frames;
+ } else {
+ framerate_controller.AddFrame(timestamp_ms);
+ }
+ }
+
+ const float output_framerate_fps =
+ static_cast<float>(num_input_frames - num_dropped_frames) /
+ input_duration_secs;
+ EXPECT_NEAR(output_framerate_fps, target_framerate_fps,
+ max_abs_framerate_error_fps);
+}
+
+TEST(FramerateController, DoNotDropAnyFramesIfTargerEqualsInput) {
+ const float input_framerate_fps = 30;
+ const size_t input_duration_secs = 3;
+ const size_t num_input_frames = input_duration_secs * input_framerate_fps;
+
+ FramerateController framerate_controller(input_framerate_fps);
+ size_t num_dropped_frames = 0;
+ for (size_t frame_num = 0; frame_num < num_input_frames; ++frame_num) {
+ const uint32_t timestamp_ms =
+ static_cast<uint32_t>(1000 * frame_num / input_framerate_fps);
+ if (framerate_controller.DropFrame(timestamp_ms)) {
+ ++num_dropped_frames;
+ } else {
+ framerate_controller.AddFrame(timestamp_ms);
+ }
+ }
+
+ EXPECT_EQ(num_dropped_frames, 0U);
+}
+
+TEST(FramerateController, DoNotDropFrameWhenTimestampJumpsBackward) {
+ FramerateController framerate_controller(30);
+ ASSERT_FALSE(framerate_controller.DropFrame(66));
+ framerate_controller.AddFrame(66);
+ EXPECT_FALSE(framerate_controller.DropFrame(33));
+}
+
+TEST(FramerateController, DropFrameIfItIsTooCloseToPreviousFrame) {
+ FramerateController framerate_controller(30);
+ ASSERT_FALSE(framerate_controller.DropFrame(33));
+ framerate_controller.AddFrame(33);
+ EXPECT_TRUE(framerate_controller.DropFrame(34));
+}
+
+TEST(FramerateController, FrameDroppingStartsFromSecondInputFrame) {
+ const float input_framerate_fps = 23;
+ const float target_framerate_fps = 19;
+ const uint32_t input_frame_duration_ms =
+ static_cast<uint32_t>(1000 / input_framerate_fps);
+ FramerateController framerate_controller(target_framerate_fps);
+ ASSERT_FALSE(framerate_controller.DropFrame(1 * input_frame_duration_ms));
+ framerate_controller.AddFrame(1 * input_frame_duration_ms);
+ EXPECT_TRUE(framerate_controller.DropFrame(2 * input_frame_duration_ms));
+}
+
+} // namespace webrtc