Add an HD resolution perf test.
Also update existing perf tests to use send side bwe.
BUG=webrtc:4604, chromium:522001
Review-Url: https://codereview.webrtc.org/2227733004
Cr-Commit-Position: refs/heads/master@{#13726}
diff --git a/resources/ConferenceMotion_1280_720_50.yuv.sha1 b/resources/ConferenceMotion_1280_720_50.yuv.sha1
new file mode 100644
index 0000000..c929e3f
--- /dev/null
+++ b/resources/ConferenceMotion_1280_720_50.yuv.sha1
@@ -0,0 +1 @@
+60a92ea32e238bc2801ac2ca26827b8b10155978
\ No newline at end of file
diff --git a/webrtc/video/full_stack.cc b/webrtc/video/full_stack.cc
index 3683413..26aae3c 100644
--- a/webrtc/video/full_stack.cc
+++ b/webrtc/video/full_stack.cc
@@ -68,7 +68,7 @@
TEST_F(FullStackTest, ParisQcifWithoutPacketLoss) {
VideoQualityTest::Params paris_qcif = {
- {176, 144, 30, 300000, 300000, 300000, false, "VP8", 1},
+ {176, 144, 30, 300000, 300000, 300000, false, "VP8", 1, 0, 0, true},
{"paris_qcif"},
{},
{"net_delay_0_0_plr_0", 36.0, 0.96, kFullStackTestDurationSecs}};
@@ -78,17 +78,17 @@
TEST_F(FullStackTest, ForemanCifWithoutPacketLoss) {
// TODO(pbos): Decide on psnr/ssim thresholds for foreman_cif.
VideoQualityTest::Params foreman_cif = {
- {352, 288, 30, 700000, 700000, 700000, false, "VP8", 1},
+ {352, 288, 30, 700000, 700000, 700000, false, "VP8", 1, 0, 0, true},
{"foreman_cif"},
{},
- {"foreman_cif_net_delay_0_0_plr_0", 0.0, 0.0, kFullStackTestDurationSecs}
- };
+ {"foreman_cif_net_delay_0_0_plr_0", 0.0, 0.0,
+ kFullStackTestDurationSecs}};
RunTest(foreman_cif);
}
TEST_F(FullStackTest, ForemanCifPlr5) {
VideoQualityTest::Params foreman_cif = {
- {352, 288, 30, 30000, 500000, 2000000, false, "VP8", 1},
+ {352, 288, 30, 30000, 500000, 2000000, false, "VP8", 1, 0, 0, true},
{"foreman_cif"},
{},
{"foreman_cif_delay_50_0_plr_5", 0.0, 0.0, kFullStackTestDurationSecs}};
@@ -99,7 +99,7 @@
TEST_F(FullStackTest, ForemanCif500kbps) {
VideoQualityTest::Params foreman_cif = {
- {352, 288, 30, 30000, 500000, 2000000, false, "VP8", 1},
+ {352, 288, 30, 30000, 500000, 2000000, false, "VP8", 1, 0, 0, true},
{"foreman_cif"},
{},
{"foreman_cif_500kbps", 0.0, 0.0, kFullStackTestDurationSecs}};
@@ -111,11 +111,11 @@
TEST_F(FullStackTest, ForemanCif500kbpsLimitedQueue) {
VideoQualityTest::Params foreman_cif = {
- {352, 288, 30, 30000, 500000, 2000000, false, "VP8", 1},
+ {352, 288, 30, 30000, 500000, 2000000, false, "VP8", 1, 0, 0, true},
{"foreman_cif"},
{},
- {"foreman_cif_500kbps_32pkts_queue", 0.0, 0.0, kFullStackTestDurationSecs}
- };
+ {"foreman_cif_500kbps_32pkts_queue", 0.0, 0.0,
+ kFullStackTestDurationSecs}};
foreman_cif.pipe.queue_length_packets = 32;
foreman_cif.pipe.queue_delay_ms = 0;
foreman_cif.pipe.link_capacity_kbps = 500;
@@ -124,7 +124,7 @@
TEST_F(FullStackTest, ForemanCif500kbps100ms) {
VideoQualityTest::Params foreman_cif = {
- {352, 288, 30, 30000, 500000, 2000000, false, "VP8", 1},
+ {352, 288, 30, 30000, 500000, 2000000, false, "VP8", 1, 0, 0, true},
{"foreman_cif"},
{},
{"foreman_cif_500kbps_100ms", 0.0, 0.0, kFullStackTestDurationSecs}};
@@ -136,11 +136,24 @@
TEST_F(FullStackTest, ForemanCif500kbps100msLimitedQueue) {
VideoQualityTest::Params foreman_cif = {
- {352, 288, 30, 30000, 500000, 2000000, false, "VP8", 1},
+ {352, 288, 30, 30000, 500000, 2000000, false, "VP8", 1, 0, 0, true},
{"foreman_cif"},
{},
{"foreman_cif_500kbps_100ms_32pkts_queue", 0.0, 0.0,
- kFullStackTestDurationSecs}};
+ kFullStackTestDurationSecs}};
+ foreman_cif.pipe.queue_length_packets = 32;
+ foreman_cif.pipe.queue_delay_ms = 100;
+ foreman_cif.pipe.link_capacity_kbps = 500;
+ RunTest(foreman_cif);
+}
+
+TEST_F(FullStackTest, ForemanCif500kbps100msLimitedQueueRecvBwe) {
+ VideoQualityTest::Params foreman_cif = {
+ {352, 288, 30, 30000, 500000, 2000000, false, "VP8", 1, 0, 0, false},
+ {"foreman_cif"},
+ {},
+ {"foreman_cif_500kbps_100ms_32pkts_queue", 0.0, 0.0,
+ kFullStackTestDurationSecs}};
foreman_cif.pipe.queue_length_packets = 32;
foreman_cif.pipe.queue_delay_ms = 100;
foreman_cif.pipe.link_capacity_kbps = 500;
@@ -149,20 +162,33 @@
TEST_F(FullStackTest, ForemanCif1000kbps100msLimitedQueue) {
VideoQualityTest::Params foreman_cif = {
- {352, 288, 30, 30000, 2000000, 2000000, false, "VP8", 1},
+ {352, 288, 30, 30000, 2000000, 2000000, false, "VP8", 1, 0, 0, true},
{"foreman_cif"},
{},
{"foreman_cif_1000kbps_100ms_32pkts_queue", 0.0, 0.0,
- kFullStackTestDurationSecs}};
+ kFullStackTestDurationSecs}};
foreman_cif.pipe.queue_length_packets = 32;
foreman_cif.pipe.queue_delay_ms = 100;
foreman_cif.pipe.link_capacity_kbps = 1000;
RunTest(foreman_cif);
}
+TEST_F(FullStackTest, ConferenceMotionHd2000kbps100msLimitedQueue) {
+ VideoQualityTest::Params conf_motion_hd = {
+ {1280, 720, 50, 30000, 3000000, 3000000, false, "VP8", 1, 0, 0, true},
+ {"ConferenceMotion_1280_720_50"},
+ {},
+ {"conference_motion_hd_2000kbps_100ms_32pkts_queue", 0.0, 0.0,
+ kFullStackTestDurationSecs}};
+ conf_motion_hd.pipe.queue_length_packets = 32;
+ conf_motion_hd.pipe.queue_delay_ms = 100;
+ conf_motion_hd.pipe.link_capacity_kbps = 2000;
+ RunTest(conf_motion_hd);
+}
+
TEST_F(FullStackTest, ScreenshareSlidesVP8_2TL) {
VideoQualityTest::Params screenshare = {
- {1850, 1110, 5, 50000, 200000, 2000000, false, "VP8", 2, 1, 400000},
+ {1850, 1110, 5, 50000, 200000, 2000000, false, "VP8", 2, 1, 400000, true},
{},
{true, 10},
{"screenshare_slides", 0.0, 0.0, kFullStackTestDurationSecs}};
@@ -171,7 +197,8 @@
TEST_F(FullStackTest, ScreenshareSlidesVP8_2TL_Scroll) {
VideoQualityTest::Params config = {
- {1850, 1110 / 2, 5, 50000, 200000, 2000000, false, "VP8", 2, 1, 400000},
+ {1850, 1110 / 2, 5, 50000, 200000, 2000000, false, "VP8", 2, 1, 400000,
+ true},
{},
{true, 10, 2},
{"screenshare_slides_scrolling", 0.0, 0.0, kFullStackTestDurationSecs}};
@@ -180,7 +207,7 @@
TEST_F(FullStackTest, ScreenshareSlidesVP8_2TL_LossyNet) {
VideoQualityTest::Params screenshare = {
- {1850, 1110, 5, 50000, 200000, 2000000, false, "VP8", 2, 1, 400000},
+ {1850, 1110, 5, 50000, 200000, 2000000, false, "VP8", 2, 1, 400000, true},
{}, // Video-specific.
{true, 10}, // Screenshare-specific.
{"screenshare_slides_lossy_net", 0.0, 0.0, kFullStackTestDurationSecs}};
@@ -192,7 +219,7 @@
TEST_F(FullStackTest, ScreenshareSlidesVP8_2TL_VeryLossyNet) {
VideoQualityTest::Params screenshare = {
- {1850, 1110, 5, 50000, 200000, 2000000, false, "VP8", 2, 1, 400000},
+ {1850, 1110, 5, 50000, 200000, 2000000, false, "VP8", 2, 1, 400000, true},
{}, // Video-specific.
{true, 10}, // Screenshare-specific.
{"screenshare_slides_very_lossy", 0.0, 0.0, kFullStackTestDurationSecs}};
@@ -205,7 +232,7 @@
#if !defined(RTC_DISABLE_VP9)
TEST_F(FullStackTest, ScreenshareSlidesVP9_2SL) {
VideoQualityTest::Params screenshare = {
- {1850, 1110, 5, 50000, 200000, 2000000, false, "VP9", 1, 0, 400000},
+ {1850, 1110, 5, 50000, 200000, 2000000, false, "VP9", 1, 0, 400000, true},
{},
{true, 10},
{"screenshare_slides_vp9_2sl", 0.0, 0.0, kFullStackTestDurationSecs},
diff --git a/webrtc/video/screenshare_loopback.cc b/webrtc/video/screenshare_loopback.cc
index 8508cd3..05ebc2e 100644
--- a/webrtc/video/screenshare_loopback.cc
+++ b/webrtc/video/screenshare_loopback.cc
@@ -228,8 +228,8 @@
flags::MinBitrateKbps() * 1000, flags::TargetBitrateKbps() * 1000,
flags::MaxBitrateKbps() * 1000, false, flags::Codec(),
flags::NumTemporalLayers(), flags::SelectedTL(),
- flags::MinTransmitBitrateKbps() * 1000, call_bitrate_config,
- flags::FLAGS_send_side_bwe},
+ flags::MinTransmitBitrateKbps() * 1000, flags::FLAGS_send_side_bwe,
+ false, call_bitrate_config},
{}, // Video specific.
{true, flags::SlideChangeInterval(), flags::ScrollDuration()},
{"screenshare", 0.0, 0.0, flags::DurationSecs(), flags::OutputFilename(),
diff --git a/webrtc/video/video_loopback.cc b/webrtc/video/video_loopback.cc
index f3e13de..5181294 100644
--- a/webrtc/video/video_loopback.cc
+++ b/webrtc/video/video_loopback.cc
@@ -227,14 +227,10 @@
VideoQualityTest::Params params{
{flags::Width(), flags::Height(), flags::Fps(),
flags::MinBitrateKbps() * 1000, flags::TargetBitrateKbps() * 1000,
- flags::MaxBitrateKbps() * 1000,
- flags::FLAGS_suspend_below_min_bitrate,
- flags::Codec(),
- flags::NumTemporalLayers(), flags::SelectedTL(),
+ flags::MaxBitrateKbps() * 1000, flags::FLAGS_suspend_below_min_bitrate,
+ flags::Codec(), flags::NumTemporalLayers(), flags::SelectedTL(),
0, // No min transmit bitrate.
- call_bitrate_config,
- flags::FLAGS_send_side_bwe,
- flags::FLAGS_use_fec},
+ flags::FLAGS_send_side_bwe, flags::FLAGS_use_fec, call_bitrate_config},
{flags::Clip()},
{}, // Screenshare specific.
{"video", 0.0, 0.0, flags::DurationSecs(), flags::OutputFilename(),
diff --git a/webrtc/video/video_quality_test.cc b/webrtc/video/video_quality_test.cc
index 66210d0..a401d6d 100644
--- a/webrtc/video/video_quality_test.cc
+++ b/webrtc/video/video_quality_test.cc
@@ -41,6 +41,7 @@
static const int kPayloadTypeH264 = 122;
static const int kPayloadTypeVP8 = 123;
static const int kPayloadTypeVP9 = 124;
+static const size_t kMaxComparisons = 10;
class VideoAnalyzer : public PacketReceiver,
public Transport,
@@ -408,10 +409,16 @@
VideoFrame reference_copy;
VideoFrame render_copy;
- reference_copy.CopyFrame(reference);
- render_copy.CopyFrame(render);
rtc::CritScope crit(&comparison_lock_);
+ if (comparisons_.size() < kMaxComparisons) {
+ reference_copy.CopyFrame(reference);
+ render_copy.CopyFrame(render);
+ } else {
+ // Copy the time to ensure that delay calculations can still be made.
+ reference_copy.set_ntp_time_ms(reference.ntp_time_ms());
+ render_copy.set_ntp_time_ms(render.ntp_time_ms());
+ }
comparisons_.push_back(FrameComparison(reference_copy, render_copy, dropped,
send_time_ms, recv_time_ms,
render_time_ms, encoded_size));
@@ -546,8 +553,12 @@
void PerformFrameComparison(const FrameComparison& comparison) {
// Perform expensive psnr and ssim calculations while not holding lock.
- double psnr = I420PSNR(&comparison.reference, &comparison.render);
- double ssim = I420SSIM(&comparison.reference, &comparison.render);
+ double psnr = -1.0;
+ double ssim = -1.0;
+ if (!comparison.reference.IsZeroSize()) {
+ psnr = I420PSNR(&comparison.reference, &comparison.render);
+ ssim = I420SSIM(&comparison.reference, &comparison.render);
+ }
int64_t input_time_ms = comparison.reference.ntp_time_ms();
@@ -558,8 +569,10 @@
comparison.recv_time_ms, comparison.render_time_ms,
comparison.encoded_frame_size, psnr, ssim));
}
- psnr_.AddSample(psnr);
- ssim_.AddSample(ssim);
+ if (psnr >= 0.0)
+ psnr_.AddSample(psnr);
+ if (ssim >= 0.0)
+ ssim_.AddSample(ssim);
if (comparison.dropped) {
++dropped_frames_;
diff --git a/webrtc/video/video_quality_test.h b/webrtc/video/video_quality_test.h
index b1f59e5..9c7f0e7 100644
--- a/webrtc/video/video_quality_test.h
+++ b/webrtc/video/video_quality_test.h
@@ -39,10 +39,10 @@
int num_temporal_layers;
int selected_tl;
int min_transmit_bps;
-
- Call::Config::BitrateConfig call_bitrate_config;
bool send_side_bwe;
bool fec;
+
+ Call::Config::BitrateConfig call_bitrate_config;
} common;
struct { // Video-specific settings.
std::string clip_name;
diff --git a/webrtc/webrtc_perf_tests.isolate b/webrtc/webrtc_perf_tests.isolate
index b66054c..7d58fd6 100644
--- a/webrtc/webrtc_perf_tests.isolate
+++ b/webrtc/webrtc_perf_tests.isolate
@@ -12,6 +12,7 @@
'files': [
'<(DEPTH)/resources/audio_coding/speech_mono_16kHz.pcm',
'<(DEPTH)/resources/audio_coding/testfile32kHz.pcm',
+ '<(DEPTH)/resources/ConferenceMotion_1280_720_50.yuv',
'<(DEPTH)/resources/foreman_cif.yuv',
'<(DEPTH)/resources/paris_qcif.yuv',
'<(DEPTH)/resources/voice_engine/audio_long16.pcm',