Convert video quality test from a TEST_F to a TEST fixture.
The purpose is to make the fixture reusable in downstream
projects. The CL adds the following things to API:
- api/test/video_quality_test_fixture.h
- api/test/create_video_quality_test_fixture.h
The following things are moved to API:
- call/bitrate_constraints.h (api/bitrate_constraints.h)
- call/simulated_network.h (api/test/simulated_network.h)
- call/media_type.h (api/mediatypes.h)
These are required by the params struct passed to the
fixture. I didn't attempt to split the params struct into
an internal-only and public version in this CL, and as
a result we need to pull in the above things. They are
quite harmless though, so I think it's worth it in order
to avoid splitting up the test config struct.
This CL doesn't solve all the problems we need to
implement downstream tests; we probably need to upstream
tracing variants of FakeNetworkPipe for instance, but
that will come later. This puts in place the basic
structure for now.
Bug: None
Change-Id: I35e26ed126fad27bc7b2a465400291084f6ac911
Reviewed-on: https://webrtc-review.googlesource.com/69601
Commit-Queue: Patrik Höglund <phoglund@webrtc.org>
Reviewed-by: Fredrik Solenberg <solenberg@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#23714}diff --git a/api/BUILD.gn b/api/BUILD.gn
index 41e1ad8..6ec5f23 100644
--- a/api/BUILD.gn
+++ b/api/BUILD.gn
@@ -48,6 +48,7 @@
visibility = [ "*" ]
cflags = []
sources = [
+ "bitrate_constraints.h",
"candidate.cc",
"candidate.h",
"cryptoparams.h",
@@ -130,6 +131,48 @@
}
}
+rtc_source_set("video_quality_test_fixture_api") {
+ visibility = [ "*" ]
+ testonly = true
+ sources = [
+ "test/video_quality_test_fixture.h",
+ ]
+ deps = [
+ ":libjingle_peerconnection_api",
+ ":simulated_network_api",
+ "../call:fake_network",
+ "../call:rtp_interfaces",
+ "../test:test_common",
+ "../test:video_test_common",
+ "video_codecs:video_codecs_api",
+ ]
+ if (!build_with_chromium && is_clang) {
+ # Suppress warnings from the Chromium Clang plugin (bugs.webrtc.org/163).
+ suppressed_configs += [ "//build/config/clang:find_bad_constructs" ]
+ }
+}
+
+if (rtc_include_tests) {
+ rtc_source_set("create_video_quality_test_fixture_api") {
+ visibility = [ "*" ]
+ testonly = true
+ sources = [
+ "test/create_video_quality_test_fixture.cc",
+ "test/create_video_quality_test_fixture.h",
+ ]
+ deps = [
+ ":fec_controller_api",
+ ":video_quality_test_fixture_api",
+ "../rtc_base:ptr_util",
+ "../video:video_quality_test",
+ ]
+ if (!build_with_chromium && is_clang) {
+ # Suppress warnings from the Chromium Clang plugin (bugs.webrtc.org/163).
+ suppressed_configs += [ "//build/config/clang:find_bad_constructs" ]
+ }
+ }
+}
+
rtc_source_set("libjingle_logging_api") {
visibility = [ "*" ]
sources = [
@@ -207,6 +250,18 @@
]
}
+rtc_source_set("simulated_network_api") {
+ visibility = [ "*" ]
+ sources = [
+ "test/simulated_network.h",
+ ]
+ deps = [
+ ":optional",
+ "../rtc_base:criticalsection",
+ "../rtc_base:rtc_base",
+ ]
+}
+
rtc_source_set("fec_controller_api") {
visibility = [ "*" ]
sources = [
diff --git a/api/bitrate_constraints.h b/api/bitrate_constraints.h
new file mode 100644
index 0000000..98e89c0
--- /dev/null
+++ b/api/bitrate_constraints.h
@@ -0,0 +1,41 @@
+/*
+ * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef API_BITRATE_CONSTRAINTS_H_
+#define API_BITRATE_CONSTRAINTS_H_
+
+#include <algorithm>
+
+namespace webrtc {
+// TODO(srte): BitrateConstraints and BitrateSettings should be merged.
+// Both represent the same kind data, but are using different default
+// initializer and representation of unset values.
+struct BitrateConstraints {
+ int min_bitrate_bps = 0;
+ int start_bitrate_bps = kDefaultStartBitrateBps;
+ int max_bitrate_bps = -1;
+
+ private:
+ static constexpr int kDefaultStartBitrateBps = 300000;
+};
+
+// Like std::min, but considers non-positive values to be unset.
+template <typename T>
+static T MinPositive(T a, T b) {
+ if (a <= 0) {
+ return b;
+ }
+ if (b <= 0) {
+ return a;
+ }
+ return std::min(a, b);
+}
+} // namespace webrtc
+#endif // API_BITRATE_CONSTRAINTS_H_
diff --git a/api/mediatypes.h b/api/mediatypes.h
index 076fc04..f281276 100644
--- a/api/mediatypes.h
+++ b/api/mediatypes.h
@@ -13,6 +13,9 @@
#include <string>
+// The cricket and webrtc have separate definitions for what a media type is.
+// They're not compatible. Watch out for this.
+
namespace cricket {
enum MediaType { MEDIA_TYPE_AUDIO, MEDIA_TYPE_VIDEO, MEDIA_TYPE_DATA };
@@ -24,4 +27,10 @@
} // namespace cricket
+namespace webrtc {
+
+enum class MediaType { ANY, AUDIO, VIDEO, DATA };
+
+} // namespace webrtc
+
#endif // API_MEDIATYPES_H_
diff --git a/api/test/DEPS b/api/test/DEPS
index 0a269fe..98b1ad3 100644
--- a/api/test/DEPS
+++ b/api/test/DEPS
@@ -2,4 +2,7 @@
".*": [
"+modules/video_coding",
],
+ ".*": [
+ "+video"
+ ],
}
diff --git a/api/test/create_video_quality_test_fixture.cc b/api/test/create_video_quality_test_fixture.cc
new file mode 100644
index 0000000..bc71861
--- /dev/null
+++ b/api/test/create_video_quality_test_fixture.cc
@@ -0,0 +1,34 @@
+/*
+ * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include <memory>
+#include <utility>
+
+#include "api/test/create_video_quality_test_fixture.h"
+#include "video/video_quality_test.h"
+#include "rtc_base/ptr_util.h"
+
+namespace webrtc {
+
+std::unique_ptr<VideoQualityTestFixtureInterface>
+CreateVideoQualityTestFixture() {
+ // By default, we don't override the FEC module, so pass an empty factory.
+ return rtc::MakeUnique<VideoQualityTest>(nullptr);
+}
+
+std::unique_ptr<VideoQualityTestFixtureInterface>
+CreateVideoQualityTestFixture(
+ std::unique_ptr<FecControllerFactoryInterface> fec_controller_factory) {
+ return rtc::MakeUnique<VideoQualityTest>(std::move(fec_controller_factory));
+}
+
+} // namespace webrtc
+
+
diff --git a/api/test/create_video_quality_test_fixture.h b/api/test/create_video_quality_test_fixture.h
new file mode 100644
index 0000000..07c222f
--- /dev/null
+++ b/api/test/create_video_quality_test_fixture.h
@@ -0,0 +1,29 @@
+/*
+ * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+#ifndef API_TEST_CREATE_VIDEO_QUALITY_TEST_FIXTURE_H_
+#define API_TEST_CREATE_VIDEO_QUALITY_TEST_FIXTURE_H_
+
+#include <memory>
+
+#include "api/fec_controller.h"
+#include "api/test/video_quality_test_fixture.h"
+
+namespace webrtc {
+
+std::unique_ptr<VideoQualityTestFixtureInterface>
+CreateVideoQualityTestFixture();
+
+std::unique_ptr<VideoQualityTestFixtureInterface>
+CreateVideoQualityTestFixture(
+ std::unique_ptr<FecControllerFactoryInterface> fec_controller_factory);
+
+}
+
+#endif // API_TEST_CREATE_VIDEO_QUALITY_TEST_FIXTURE_H_
diff --git a/api/test/simulated_network.h b/api/test/simulated_network.h
new file mode 100644
index 0000000..0006be2
--- /dev/null
+++ b/api/test/simulated_network.h
@@ -0,0 +1,77 @@
+/*
+ * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef API_TEST_SIMULATED_NETWORK_H_
+#define API_TEST_SIMULATED_NETWORK_H_
+
+#include <stddef.h>
+#include <stdint.h>
+#include <deque>
+#include <queue>
+#include <vector>
+
+#include "api/optional.h"
+#include "rtc_base/criticalsection.h"
+#include "rtc_base/random.h"
+#include "rtc_base/thread_annotations.h"
+
+namespace webrtc {
+
+struct PacketInFlightInfo {
+ PacketInFlightInfo(size_t size, int64_t send_time_us, uint64_t packet_id)
+ : size(size), send_time_us(send_time_us), packet_id(packet_id) {}
+
+ size_t size;
+ int64_t send_time_us;
+ // Unique identifier for the packet in relation to other packets in flight.
+ uint64_t packet_id;
+};
+
+struct PacketDeliveryInfo {
+ static constexpr int kNotReceived = -1;
+ PacketDeliveryInfo(PacketInFlightInfo source, int64_t receive_time_us)
+ : receive_time_us(receive_time_us), packet_id(source.packet_id) {}
+ int64_t receive_time_us;
+ uint64_t packet_id;
+};
+
+class NetworkSimulationInterface {
+ public:
+ // TODO(phoglund): this one shouldn't really be here; make fake network pipes
+ // injectable instead in the video quality test fixture.
+ struct SimulatedNetworkConfig {
+ SimulatedNetworkConfig() {}
+ // Queue length in number of packets.
+ size_t queue_length_packets = 0;
+ // Delay in addition to capacity induced delay.
+ int queue_delay_ms = 0;
+ // Standard deviation of the extra delay.
+ int delay_standard_deviation_ms = 0;
+ // Link capacity in kbps.
+ int link_capacity_kbps = 0;
+ // Random packet loss.
+ int loss_percent = 0;
+ // If packets are allowed to be reordered.
+ bool allow_reordering = false;
+ // The average length of a burst of lost packets.
+ int avg_burst_loss_length = -1;
+ };
+
+ virtual bool EnqueuePacket(PacketInFlightInfo packet_info) = 0;
+ // Retrieves all packets that should be delivered by the given receive time.
+ virtual std::vector<PacketDeliveryInfo> DequeueDeliverablePackets(
+ int64_t receive_time_us) = 0;
+ virtual absl::optional<int64_t> NextDeliveryTimeUs() const = 0;
+ virtual ~NetworkSimulationInterface() = default;
+};
+
+} // namespace webrtc
+
+#endif // API_TEST_SIMULATED_NETWORK_H_
diff --git a/api/test/video_quality_test_fixture.h b/api/test/video_quality_test_fixture.h
new file mode 100644
index 0000000..b8a268c
--- /dev/null
+++ b/api/test/video_quality_test_fixture.h
@@ -0,0 +1,111 @@
+/*
+ * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef API_TEST_VIDEO_QUALITY_TEST_FIXTURE_H_
+#define API_TEST_VIDEO_QUALITY_TEST_FIXTURE_H_
+
+#include <map>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "api/bitrate_constraints.h"
+#include "api/mediatypes.h"
+#include "api/test/simulated_network.h"
+#include "api/video_codecs/video_encoder_config.h"
+
+namespace webrtc {
+
+class VideoQualityTestFixtureInterface {
+ public:
+ // Parameters are grouped into smaller structs to make it easier to set
+ // the desired elements and skip unused, using aggregate initialization.
+ // Unfortunately, C++11 (as opposed to C11) doesn't support unnamed structs,
+ // which makes the implementation of VideoQualityTest a bit uglier.
+ struct Params {
+ Params();
+ ~Params();
+ struct CallConfig {
+ bool send_side_bwe;
+ BitrateConstraints call_bitrate_config;
+ int num_thumbnails;
+ // Indicates if secondary_(video|ss|screenshare) structures are used.
+ bool dual_video;
+ } call;
+ struct Video {
+ bool enabled;
+ size_t width;
+ size_t height;
+ int32_t fps;
+ int min_bitrate_bps;
+ int target_bitrate_bps;
+ int max_bitrate_bps;
+ bool suspend_below_min_bitrate;
+ std::string codec;
+ int num_temporal_layers;
+ int selected_tl;
+ int min_transmit_bps;
+ bool ulpfec;
+ bool flexfec;
+ bool automatic_scaling;
+ std::string clip_name; // "Generator" to generate frames instead.
+ size_t capture_device_index;
+ } video[2];
+ struct Audio {
+ bool enabled;
+ bool sync_video;
+ bool dtx;
+ } audio;
+ struct Screenshare {
+ bool enabled;
+ bool generate_slides;
+ int32_t slide_change_interval;
+ int32_t scroll_duration;
+ std::vector<std::string> slides;
+ } screenshare[2];
+ struct Analyzer {
+ std::string test_label;
+ double avg_psnr_threshold; // (*)
+ double avg_ssim_threshold; // (*)
+ int test_durations_secs;
+ std::string graph_data_output_filename;
+ std::string graph_title;
+ } analyzer;
+ NetworkSimulationInterface::SimulatedNetworkConfig pipe;
+ struct SS { // Spatial scalability.
+ std::vector<VideoStream> streams; // If empty, one stream is assumed.
+ size_t selected_stream;
+ int num_spatial_layers;
+ int selected_sl;
+ InterLayerPredMode inter_layer_pred;
+ // If empty, bitrates are generated in VP9Impl automatically.
+ std::vector<SpatialLayer> spatial_layers;
+ // If set, default parameters will be used instead of |streams|.
+ bool infer_streams;
+ } ss[2];
+ struct Logging {
+ bool logs;
+ std::string rtc_event_log_name;
+ std::string rtp_dump_name;
+ std::string encoded_frame_base_path;
+ } logging;
+ };
+
+ virtual ~VideoQualityTestFixtureInterface() = default;
+
+ virtual void RunWithAnalyzer(const Params& params) = 0;
+ virtual void RunWithRenderers(const Params& params) = 0;
+
+ virtual const std::map<uint8_t, webrtc::MediaType>& payload_type_map() = 0;
+};
+
+} // namespace webrtc
+
+#endif // API_TEST_VIDEO_QUALITY_TEST_FIXTURE_H_