Add per flow throughput and delay metrics.
BUG=4548
R=pbos@webrtc.org
Review URL: https://webrtc-codereview.appspot.com/48299004
Cr-Commit-Position: refs/heads/master@{#9112}
diff --git a/webrtc/modules/remote_bitrate_estimator/remote_bitrate_estimators_test.cc b/webrtc/modules/remote_bitrate_estimator/remote_bitrate_estimators_test.cc
index 42f4e1c..227ac4c 100644
--- a/webrtc/modules/remote_bitrate_estimator/remote_bitrate_estimators_test.cc
+++ b/webrtc/modules/remote_bitrate_estimator/remote_bitrate_estimators_test.cc
@@ -280,8 +280,8 @@
filter.SetCapacity(kHighCapacityKbps);
RunFor(60 * 1000);
PrintResults((2 * kHighCapacityKbps + kLowCapacityKbps) / 3.0,
- counter.GetBitrateStats(), filter.GetDelayStats(),
- std::vector<Stats<double>>());
+ counter.GetBitrateStats(), 0, receiver.GetDelayStats(),
+ counter.GetBitrateStats());
}
TEST_P(BweFeedbackTest, Choke200kbps30kbps200kbps) {
@@ -301,8 +301,8 @@
RunFor(60 * 1000);
PrintResults((2 * kHighCapacityKbps + kLowCapacityKbps) / 3.0,
- counter.GetBitrateStats(), filter.GetDelayStats(),
- std::vector<Stats<double>>());
+ counter.GetBitrateStats(), 0, receiver.GetDelayStats(),
+ counter.GetBitrateStats());
}
TEST_P(BweFeedbackTest, Verizon4gDownlinkTest) {
@@ -315,7 +315,7 @@
ASSERT_TRUE(filter.Init(test::ResourcePath("verizon4g-downlink", "rx")));
RunFor(22 * 60 * 1000);
PrintResults(filter.GetBitrateStats().GetMean(), counter2.GetBitrateStats(),
- filter.GetDelayStats(), std::vector<Stats<double>>());
+ 0, receiver.GetDelayStats(), counter2.GetBitrateStats());
}
// webrtc:3277
@@ -330,7 +330,7 @@
ASSERT_TRUE(filter.Init(test::ResourcePath("google-wifi-3mbps", "rx")));
RunFor(300 * 1000);
PrintResults(filter.GetBitrateStats().GetMean(), counter2.GetBitrateStats(),
- filter.GetDelayStats(), std::vector<Stats<double>>());
+ 0, receiver.GetDelayStats(), counter2.GetBitrateStats());
}
TEST_P(BweFeedbackTest, PacedSelfFairness50msTest) {
diff --git a/webrtc/modules/remote_bitrate_estimator/test/bwe_test.cc b/webrtc/modules/remote_bitrate_estimator/test/bwe_test.cc
index 83c76a5..86e8cb0 100644
--- a/webrtc/modules/remote_bitrate_estimator/test/bwe_test.cc
+++ b/webrtc/modules/remote_bitrate_estimator/test/bwe_test.cc
@@ -177,11 +177,23 @@
return string(test_info->name());
}
-void BweTest::PrintResults(
- double max_throughput_kbps,
- Stats<double> throughput_kbps,
- Stats<double> delay_ms,
- std::vector<Stats<double>> flow_throughput_kbps) {
+void BweTest::PrintResults(double max_throughput_kbps,
+ Stats<double> throughput_kbps,
+ int flow_id,
+ Stats<double> flow_delay_ms,
+ Stats<double> flow_throughput_kbps) {
+ std::map<int, Stats<double>> flow_delays_ms;
+ flow_delays_ms[flow_id] = flow_delay_ms;
+ std::map<int, Stats<double>> flow_throughputs_kbps;
+ flow_throughputs_kbps[flow_id] = flow_throughput_kbps;
+ PrintResults(max_throughput_kbps, throughput_kbps, flow_delays_ms,
+ flow_throughputs_kbps);
+}
+
+void BweTest::PrintResults(double max_throughput_kbps,
+ Stats<double> throughput_kbps,
+ std::map<int, Stats<double>> flow_delay_ms,
+ std::map<int, Stats<double>> flow_throughput_kbps) {
double utilization = throughput_kbps.GetMean() / max_throughput_kbps;
webrtc::test::PrintResult("BwePerformance", GetTestName(), "Utilization",
utilization * 100.0, "%", false);
@@ -189,15 +201,27 @@
ss << throughput_kbps.GetStdDev() / throughput_kbps.GetMean();
webrtc::test::PrintResult("BwePerformance", GetTestName(),
"Utilization var coeff", ss.str(), "", false);
- webrtc::test::PrintResultMeanAndError("BwePerformance", GetTestName(),
- "Average delay", delay_ms.AsString(),
- "ms", false);
+ for (auto& kv : flow_throughput_kbps) {
+ ss.str("");
+ ss << "Throughput flow " << kv.first;
+ webrtc::test::PrintResultMeanAndError("BwePerformance", GetTestName(),
+ ss.str(), kv.second.AsString(),
+ "kbps", false);
+ }
+ for (auto& kv : flow_delay_ms) {
+ ss.str("");
+ ss << "Delay flow " << kv.first;
+ webrtc::test::PrintResultMeanAndError("BwePerformance", GetTestName(),
+ ss.str(), kv.second.AsString(), "ms",
+ false);
+ }
double fairness_index = 1.0;
if (!flow_throughput_kbps.empty()) {
double squared_bitrate_sum = 0.0;
- for (Stats<double> flow : flow_throughput_kbps) {
- squared_bitrate_sum += flow.GetMean() * flow.GetMean();
- fairness_index += flow.GetMean();
+ fairness_index = 0.0;
+ for (auto kv : flow_throughput_kbps) {
+ squared_bitrate_sum += kv.second.GetMean() * kv.second.GetMean();
+ fairness_index += kv.second.GetMean();
}
fairness_index *= fairness_index;
fairness_index /= flow_throughput_kbps.size() * squared_bitrate_sum;
@@ -234,13 +258,13 @@
for (int media_flow : media_flow_ids) {
// Streams started 20 seconds apart to give them different advantage when
// competing for the bandwidth.
- const int64_t kFlowStartOffsetMs = i++ * (rand() % 40000);
+ const int64_t kFlowStartOffsetMs = i++ * (rand() % 10000);
sources.push_back(new AdaptiveVideoSource(media_flow, 30, 300, 0,
kFlowStartOffsetMs));
senders.push_back(new PacedVideoSender(&uplink_, sources.back(), bwe_type));
}
- const int64_t kTcpStartOffsetMs = 20000;
+ const int64_t kTcpStartOffsetMs = 5000;
for (int tcp_flow : tcp_flow_ids)
senders.push_back(new TcpSender(&uplink_, tcp_flow, kTcpStartOffsetMs));
@@ -276,11 +300,20 @@
RunFor(run_time_seconds * 1000);
- std::vector<Stats<double>> flow_throughput_kbps;
- for (i = 0; i < all_flow_ids.size(); ++i)
- flow_throughput_kbps.push_back(rate_counters[i]->GetBitrateStats());
+ std::map<int, Stats<double>> flow_throughput_kbps;
+ for (RateCounterFilter* rate_counter : rate_counters) {
+ int flow_id = *rate_counter->flow_ids().begin();
+ flow_throughput_kbps[flow_id] = rate_counter->GetBitrateStats();
+ }
+
+ std::map<int, Stats<double>> flow_delay_ms;
+ for (PacketReceiver* receiver : receivers) {
+ int flow_id = *receiver->flow_ids().begin();
+ flow_delay_ms[flow_id] = receiver->GetDelayStats();
+ }
+
PrintResults(capacity_kbps, total_utilization.GetBitrateStats(),
- choke.GetDelayStats(), flow_throughput_kbps);
+ flow_delay_ms, flow_throughput_kbps);
for (VideoSource* source : sources)
delete source;
diff --git a/webrtc/modules/remote_bitrate_estimator/test/bwe_test.h b/webrtc/modules/remote_bitrate_estimator/test/bwe_test.h
index 2de5ac6..303aca5 100644
--- a/webrtc/modules/remote_bitrate_estimator/test/bwe_test.h
+++ b/webrtc/modules/remote_bitrate_estimator/test/bwe_test.h
@@ -78,8 +78,14 @@
void PrintResults(double max_throughput_kbps,
Stats<double> throughput_kbps,
- Stats<double> delay_ms,
- std::vector<Stats<double>> flow_throughput_kbps);
+ int flow_id,
+ Stats<double> flow_delay_ms,
+ Stats<double> flow_throughput_kbps);
+
+ void PrintResults(double max_throughput_kbps,
+ Stats<double> throughput_kbps,
+ std::map<int, Stats<double>> flow_delay_ms,
+ std::map<int, Stats<double>> flow_throughput_kbps);
void RunFairnessTest(BandwidthEstimatorType bwe_type,
size_t num_media_flows,
diff --git a/webrtc/modules/remote_bitrate_estimator/test/packet_receiver.cc b/webrtc/modules/remote_bitrate_estimator/test/packet_receiver.cc
index 1640137..aa429d0 100644
--- a/webrtc/modules/remote_bitrate_estimator/test/packet_receiver.cc
+++ b/webrtc/modules/remote_bitrate_estimator/test/packet_receiver.cc
@@ -58,8 +58,9 @@
// We're treating the send time (from previous filter) as the arrival
// time once packet reaches the estimator.
int64_t arrival_time_ms = (media_packet->send_time_us() + 500) / 1000;
- PlotDelay(arrival_time_ms,
- (media_packet->creation_time_us() + 500) / 1000);
+ int64_t send_time_ms = (media_packet->creation_time_us() + 500) / 1000;
+ delay_stats_.Push(arrival_time_ms - send_time_ms);
+ PlotDelay(arrival_time_ms, send_time_ms);
bwe_receiver_->ReceivePacket(arrival_time_ms, *media_packet);
FeedbackPacket* fb = bwe_receiver_->GetFeedback(arrival_time_ms);
@@ -85,6 +86,10 @@
last_delay_plot_ms_ = arrival_time_ms;
}
}
+
+Stats<double> PacketReceiver::GetDelayStats() const {
+ return delay_stats_;
+}
} // namespace bwe
} // namespace testing
} // namespace webrtc
diff --git a/webrtc/modules/remote_bitrate_estimator/test/packet_receiver.h b/webrtc/modules/remote_bitrate_estimator/test/packet_receiver.h
index 6d94ea7..0589d66 100644
--- a/webrtc/modules/remote_bitrate_estimator/test/packet_receiver.h
+++ b/webrtc/modules/remote_bitrate_estimator/test/packet_receiver.h
@@ -36,6 +36,8 @@
void LogStats();
+ Stats<double> GetDelayStats() const;
+
protected:
void PlotDelay(int64_t arrival_time_ms, int64_t send_time_ms);
@@ -43,6 +45,7 @@
std::string delay_log_prefix_;
int64_t last_delay_plot_ms_;
bool plot_delay_;
+ Stats<double> delay_stats_;
rtc::scoped_ptr<BweReceiver> bwe_receiver_;
private: