blob: 0a2a7384d8dd7d0ad1eec0a1c68a88ee3b8b939c [file] [log] [blame]
pbos@webrtc.org744fbc72013-09-10 09:26:25 +00001/*
2 * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
pbos@webrtc.org744fbc72013-09-10 09:26:25 +000010
stefanff483612015-12-21 03:14:00 -080011#include "webrtc/call/rampup_tests.h"
12
pbos@webrtc.org2b4ce3a2015-03-23 13:12:24 +000013#include "webrtc/base/checks.h"
philipel5ef2bc12017-02-21 07:28:31 -080014#include "webrtc/base/logging.h"
pbos12411ef2015-11-23 14:47:56 -080015#include "webrtc/base/platform_thread.h"
perkjfa10b552016-10-02 23:45:26 -070016#include "webrtc/test/encoder_settings.h"
kwibergac9f8762016-09-30 22:29:43 -070017#include "webrtc/test/gtest.h"
stefan@webrtc.org7e9315b2013-12-04 10:24:26 +000018#include "webrtc/test/testsupport/perf_test.h"
pbos@webrtc.org744fbc72013-09-10 09:26:25 +000019
20namespace webrtc {
pbos@webrtc.org29023282013-09-11 10:14:56 +000021namespace {
stefan@webrtc.org3d7da882014-07-08 13:59:46 +000022
Stefan Holmer723dff12015-10-05 14:59:41 +020023static const int64_t kPollIntervalMs = 20;
philipel5ef2bc12017-02-21 07:28:31 -080024static const int kExpectedHighVideoBitrateBps = 80000;
stefandb752f92016-12-05 08:23:40 -080025static const int kExpectedHighAudioBitrateBps = 30000;
26static const int kLowBandwidthLimitBps = 20000;
27static const int kExpectedLowBitrateBps = 20000;
pbos@webrtc.org29023282013-09-11 10:14:56 +000028
stefanff483612015-12-21 03:14:00 -080029std::vector<uint32_t> GenerateSsrcs(size_t num_streams, uint32_t ssrc_offset) {
stefan@webrtc.org3d7da882014-07-08 13:59:46 +000030 std::vector<uint32_t> ssrcs;
31 for (size_t i = 0; i != num_streams; ++i)
32 ssrcs.push_back(static_cast<uint32_t>(ssrc_offset + i));
33 return ssrcs;
34}
mflodman@webrtc.orgeb16b812014-06-16 08:57:39 +000035} // namespace
henrik.lundin@webrtc.org845862f2014-03-06 07:19:28 +000036
stefanff483612015-12-21 03:14:00 -080037RampUpTester::RampUpTester(size_t num_video_streams,
38 size_t num_audio_streams,
philipel5ef2bc12017-02-21 07:28:31 -080039 size_t num_flexfec_streams,
stefan4fbd1452015-09-28 03:57:14 -070040 unsigned int start_bitrate_bps,
stefan5a2c5062017-01-27 06:43:18 -080041 int64_t min_run_time_ms,
stefan4fbd1452015-09-28 03:57:14 -070042 const std::string& extension_type,
43 bool rtx,
stefan5a2c5062017-01-27 06:43:18 -080044 bool red,
45 bool report_perf_stats)
stefan4fbd1452015-09-28 03:57:14 -070046 : EndToEndTest(test::CallTest::kLongTimeoutMs),
tommi0f8b4032017-02-22 11:22:05 -080047 stop_event_(false, false),
stefan4fbd1452015-09-28 03:57:14 -070048 clock_(Clock::GetRealTimeClock()),
stefanff483612015-12-21 03:14:00 -080049 num_video_streams_(num_video_streams),
50 num_audio_streams_(num_audio_streams),
philipel5ef2bc12017-02-21 07:28:31 -080051 num_flexfec_streams_(num_flexfec_streams),
stefan4fbd1452015-09-28 03:57:14 -070052 rtx_(rtx),
53 red_(red),
stefan45b5fe52017-03-09 06:27:02 -080054 report_perf_stats_(report_perf_stats),
mflodman86cc6ff2016-07-26 04:44:06 -070055 sender_call_(nullptr),
stefan4fbd1452015-09-28 03:57:14 -070056 send_stream_(nullptr),
57 start_bitrate_bps_(start_bitrate_bps),
stefan5a2c5062017-01-27 06:43:18 -080058 min_run_time_ms_(min_run_time_ms),
stefan@webrtc.org3d7da882014-07-08 13:59:46 +000059 expected_bitrate_bps_(0),
Erik Språngf3a7c9d2015-10-05 14:03:22 +020060 test_start_ms_(-1),
stefan4fbd1452015-09-28 03:57:14 -070061 ramp_up_finished_ms_(-1),
62 extension_type_(extension_type),
stefanff483612015-12-21 03:14:00 -080063 video_ssrcs_(GenerateSsrcs(num_video_streams_, 100)),
64 video_rtx_ssrcs_(GenerateSsrcs(num_video_streams_, 200)),
65 audio_ssrcs_(GenerateSsrcs(num_audio_streams_, 300)),
Peter Boström8c38e8b2015-11-26 17:45:47 +010066 poller_thread_(&BitrateStatsPollingThread,
67 this,
mflodman86cc6ff2016-07-26 04:44:06 -070068 "BitrateStatsPollingThread") {
philipel5ef2bc12017-02-21 07:28:31 -080069 if (red_)
70 EXPECT_EQ(0u, num_flexfec_streams_);
Stefan Holmerff2a6352016-01-14 10:00:21 +010071 EXPECT_LE(num_audio_streams_, 1u);
stefan4fbd1452015-09-28 03:57:14 -070072}
73
74RampUpTester::~RampUpTester() {
stefan4fbd1452015-09-28 03:57:14 -070075}
76
77Call::Config RampUpTester::GetSenderCallConfig() {
skvlad11a9cbf2016-10-07 11:53:05 -070078 Call::Config call_config(&event_log_);
stefan4fbd1452015-09-28 03:57:14 -070079 if (start_bitrate_bps_ != 0) {
80 call_config.bitrate_config.start_bitrate_bps = start_bitrate_bps_;
81 }
82 call_config.bitrate_config.min_bitrate_bps = 10000;
83 return call_config;
84}
85
stefanff483612015-12-21 03:14:00 -080086void RampUpTester::OnVideoStreamsCreated(
stefan4fbd1452015-09-28 03:57:14 -070087 VideoSendStream* send_stream,
88 const std::vector<VideoReceiveStream*>& receive_streams) {
89 send_stream_ = send_stream;
90}
91
nissee5ad5ca2017-03-29 23:57:43 -070092MediaType RampUpTester::SelectMediaType() {
93 if (num_video_streams_ > 0) {
94 if (num_audio_streams_ > 0) {
95 // Rely on call to set media type from payload type.
96 return MediaType::ANY;
97 } else {
98 return MediaType::VIDEO;
99 }
100 } else {
101 return MediaType::AUDIO;
102 }
103}
104
stefane74eef12016-01-08 06:47:13 -0800105test::PacketTransport* RampUpTester::CreateSendTransport(Call* sender_call) {
106 send_transport_ = new test::PacketTransport(sender_call, this,
107 test::PacketTransport::kSender,
nissee5ad5ca2017-03-29 23:57:43 -0700108 SelectMediaType(),
stefane74eef12016-01-08 06:47:13 -0800109 forward_transport_config_);
110 return send_transport_;
stefanf116bd02015-10-27 08:29:42 -0700111}
112
nissee5ad5ca2017-03-29 23:57:43 -0700113test::PacketTransport* RampUpTester::CreateReceiveTransport() {
114 return new test::PacketTransport(nullptr, this,
115 test::PacketTransport::kReceiver,
116 SelectMediaType(),
117 FakeNetworkPipe::Config());
118}
119
Stefan Holmerd20e6512016-01-12 15:51:22 +0100120size_t RampUpTester::GetNumVideoStreams() const {
121 return num_video_streams_;
122}
123
Stefan Holmerff2a6352016-01-14 10:00:21 +0100124size_t RampUpTester::GetNumAudioStreams() const {
125 return num_audio_streams_;
126}
127
philipel5ef2bc12017-02-21 07:28:31 -0800128size_t RampUpTester::GetNumFlexfecStreams() const {
129 return num_flexfec_streams_;
130}
131
perkjfa10b552016-10-02 23:45:26 -0700132class RampUpTester::VideoStreamFactory
133 : public VideoEncoderConfig::VideoStreamFactoryInterface {
134 public:
135 VideoStreamFactory() {}
136
137 private:
138 std::vector<VideoStream> CreateEncoderStreams(
139 int width,
140 int height,
141 const VideoEncoderConfig& encoder_config) override {
142 std::vector<VideoStream> streams =
143 test::CreateVideoStreams(width, height, encoder_config);
144 if (encoder_config.number_of_streams == 1) {
145 streams[0].target_bitrate_bps = streams[0].max_bitrate_bps = 2000000;
146 }
147 return streams;
148 }
149};
150
stefanff483612015-12-21 03:14:00 -0800151void RampUpTester::ModifyVideoConfigs(
stefan4fbd1452015-09-28 03:57:14 -0700152 VideoSendStream::Config* send_config,
153 std::vector<VideoReceiveStream::Config>* receive_configs,
154 VideoEncoderConfig* encoder_config) {
155 send_config->suspend_below_min_bitrate = true;
perkjfa10b552016-10-02 23:45:26 -0700156 encoder_config->number_of_streams = num_video_streams_;
157 encoder_config->max_bitrate_bps = 2000000;
158 encoder_config->video_stream_factory =
159 new rtc::RefCountedObject<RampUpTester::VideoStreamFactory>();
stefanff483612015-12-21 03:14:00 -0800160 if (num_video_streams_ == 1) {
stefan4fbd1452015-09-28 03:57:14 -0700161 // For single stream rampup until 1mbps
162 expected_bitrate_bps_ = kSingleStreamTargetBps;
163 } else {
164 // For multi stream rampup until all streams are being sent. That means
perkjfa10b552016-10-02 23:45:26 -0700165 // enough bitrate to send all the target streams plus the min bitrate of
stefan4fbd1452015-09-28 03:57:14 -0700166 // the last one.
perkjfa10b552016-10-02 23:45:26 -0700167 std::vector<VideoStream> streams = test::CreateVideoStreams(
168 test::CallTest::kDefaultWidth, test::CallTest::kDefaultHeight,
169 *encoder_config);
170 expected_bitrate_bps_ = streams.back().min_bitrate_bps;
171 for (size_t i = 0; i < streams.size() - 1; ++i) {
172 expected_bitrate_bps_ += streams[i].target_bitrate_bps;
stefan@webrtc.org7e9315b2013-12-04 10:24:26 +0000173 }
stefan@webrtc.org3d7da882014-07-08 13:59:46 +0000174 }
stefan@webrtc.org7e9315b2013-12-04 10:24:26 +0000175
stefan4fbd1452015-09-28 03:57:14 -0700176 send_config->rtp.extensions.clear();
177
178 bool remb;
stefan43edf0f2015-11-20 18:05:48 -0800179 bool transport_cc;
isheriff6f8d6862016-05-26 11:24:55 -0700180 if (extension_type_ == RtpExtension::kAbsSendTimeUri) {
stefan4fbd1452015-09-28 03:57:14 -0700181 remb = true;
stefan43edf0f2015-11-20 18:05:48 -0800182 transport_cc = false;
stefan4fbd1452015-09-28 03:57:14 -0700183 send_config->rtp.extensions.push_back(
184 RtpExtension(extension_type_.c_str(), kAbsSendTimeExtensionId));
isheriff6f8d6862016-05-26 11:24:55 -0700185 } else if (extension_type_ == RtpExtension::kTransportSequenceNumberUri) {
stefan4fbd1452015-09-28 03:57:14 -0700186 remb = false;
stefan43edf0f2015-11-20 18:05:48 -0800187 transport_cc = true;
stefan4fbd1452015-09-28 03:57:14 -0700188 send_config->rtp.extensions.push_back(RtpExtension(
189 extension_type_.c_str(), kTransportSequenceNumberExtensionId));
stefan@webrtc.org3d7da882014-07-08 13:59:46 +0000190 } else {
stefan4fbd1452015-09-28 03:57:14 -0700191 remb = true;
stefan43edf0f2015-11-20 18:05:48 -0800192 transport_cc = false;
stefan4fbd1452015-09-28 03:57:14 -0700193 send_config->rtp.extensions.push_back(RtpExtension(
194 extension_type_.c_str(), kTransmissionTimeOffsetExtensionId));
stefan@webrtc.org3d7da882014-07-08 13:59:46 +0000195 }
stefan4fbd1452015-09-28 03:57:14 -0700196
197 send_config->rtp.nack.rtp_history_ms = test::CallTest::kNackRtpHistoryMs;
stefanff483612015-12-21 03:14:00 -0800198 send_config->rtp.ssrcs = video_ssrcs_;
stefan4fbd1452015-09-28 03:57:14 -0700199 if (rtx_) {
200 send_config->rtp.rtx.payload_type = test::CallTest::kSendRtxPayloadType;
stefanff483612015-12-21 03:14:00 -0800201 send_config->rtp.rtx.ssrcs = video_rtx_ssrcs_;
stefan4fbd1452015-09-28 03:57:14 -0700202 }
203 if (red_) {
brandtrb5f2c3f2016-10-04 23:28:39 -0700204 send_config->rtp.ulpfec.ulpfec_payload_type =
stefan4fbd1452015-09-28 03:57:14 -0700205 test::CallTest::kUlpfecPayloadType;
brandtrb5f2c3f2016-10-04 23:28:39 -0700206 send_config->rtp.ulpfec.red_payload_type = test::CallTest::kRedPayloadType;
brandtrfbfb5362016-11-17 04:18:37 -0800207 if (rtx_) {
208 send_config->rtp.ulpfec.red_rtx_payload_type =
209 test::CallTest::kRtxRedPayloadType;
210 }
stefan4fbd1452015-09-28 03:57:14 -0700211 }
212
213 size_t i = 0;
214 for (VideoReceiveStream::Config& recv_config : *receive_configs) {
215 recv_config.rtp.remb = remb;
stefan43edf0f2015-11-20 18:05:48 -0800216 recv_config.rtp.transport_cc = transport_cc;
stefan4fbd1452015-09-28 03:57:14 -0700217 recv_config.rtp.extensions = send_config->rtp.extensions;
218
stefanff483612015-12-21 03:14:00 -0800219 recv_config.rtp.remote_ssrc = video_ssrcs_[i];
stefan4fbd1452015-09-28 03:57:14 -0700220 recv_config.rtp.nack.rtp_history_ms = send_config->rtp.nack.rtp_history_ms;
221
222 if (red_) {
brandtrb5f2c3f2016-10-04 23:28:39 -0700223 recv_config.rtp.ulpfec.red_payload_type =
224 send_config->rtp.ulpfec.red_payload_type;
225 recv_config.rtp.ulpfec.ulpfec_payload_type =
226 send_config->rtp.ulpfec.ulpfec_payload_type;
brandtrfbfb5362016-11-17 04:18:37 -0800227 if (rtx_) {
228 recv_config.rtp.ulpfec.red_rtx_payload_type =
229 send_config->rtp.ulpfec.red_rtx_payload_type;
230 }
stefan4fbd1452015-09-28 03:57:14 -0700231 }
232
233 if (rtx_) {
brandtr14742122017-01-27 04:53:07 -0800234 recv_config.rtp.rtx_ssrc = video_rtx_ssrcs_[i];
235 recv_config.rtp
236 .rtx_payload_types[send_config->encoder_settings.payload_type] =
237 send_config->rtp.rtx.payload_type;
stefan4fbd1452015-09-28 03:57:14 -0700238 }
239 ++i;
240 }
philipel5ef2bc12017-02-21 07:28:31 -0800241
242 RTC_DCHECK_LE(num_flexfec_streams_, 1);
243 if (num_flexfec_streams_ == 1) {
244 send_config->rtp.flexfec.payload_type = test::CallTest::kFlexfecPayloadType;
245 send_config->rtp.flexfec.ssrc = test::CallTest::kFlexfecSendSsrc;
246 send_config->rtp.flexfec.protected_media_ssrcs = {video_ssrcs_[0]};
247 }
stefan@webrtc.org3d7da882014-07-08 13:59:46 +0000248}
249
Stefan Holmerff2a6352016-01-14 10:00:21 +0100250void RampUpTester::ModifyAudioConfigs(
251 AudioSendStream::Config* send_config,
252 std::vector<AudioReceiveStream::Config>* receive_configs) {
253 if (num_audio_streams_ == 0)
254 return;
255
isheriff6f8d6862016-05-26 11:24:55 -0700256 EXPECT_NE(RtpExtension::kTimestampOffsetUri, extension_type_)
Stefan Holmerff2a6352016-01-14 10:00:21 +0100257 << "Audio BWE not supported with toffset.";
philipel5ef2bc12017-02-21 07:28:31 -0800258 EXPECT_NE(RtpExtension::kAbsSendTimeUri, extension_type_)
259 << "Audio BWE not supported with abs-send-time.";
Stefan Holmerff2a6352016-01-14 10:00:21 +0100260
261 send_config->rtp.ssrc = audio_ssrcs_[0];
262 send_config->rtp.extensions.clear();
263
minyue10cbb462016-11-07 09:29:22 -0800264 send_config->min_bitrate_bps = 6000;
265 send_config->max_bitrate_bps = 60000;
mflodman86cc6ff2016-07-26 04:44:06 -0700266
Stefan Holmerff2a6352016-01-14 10:00:21 +0100267 bool transport_cc = false;
philipel5ef2bc12017-02-21 07:28:31 -0800268 if (extension_type_ == RtpExtension::kTransportSequenceNumberUri) {
Stefan Holmerff2a6352016-01-14 10:00:21 +0100269 transport_cc = true;
270 send_config->rtp.extensions.push_back(RtpExtension(
271 extension_type_.c_str(), kTransportSequenceNumberExtensionId));
272 }
273
274 for (AudioReceiveStream::Config& recv_config : *receive_configs) {
Stefan Holmerff2a6352016-01-14 10:00:21 +0100275 recv_config.rtp.transport_cc = transport_cc;
276 recv_config.rtp.extensions = send_config->rtp.extensions;
277 recv_config.rtp.remote_ssrc = send_config->rtp.ssrc;
278 }
279}
280
philipel5ef2bc12017-02-21 07:28:31 -0800281void RampUpTester::ModifyFlexfecConfigs(
282 std::vector<FlexfecReceiveStream::Config>* receive_configs) {
283 if (num_flexfec_streams_ == 0)
284 return;
285 RTC_DCHECK_EQ(1, num_flexfec_streams_);
286 (*receive_configs)[0].payload_type = test::CallTest::kFlexfecPayloadType;
287 (*receive_configs)[0].remote_ssrc = test::CallTest::kFlexfecSendSsrc;
288 (*receive_configs)[0].protected_media_ssrcs = {video_ssrcs_[0]};
289 (*receive_configs)[0].local_ssrc = video_ssrcs_[0];
290 if (extension_type_ == RtpExtension::kAbsSendTimeUri) {
291 (*receive_configs)[0].transport_cc = false;
292 (*receive_configs)[0].rtp_header_extensions.push_back(
293 RtpExtension(extension_type_.c_str(), kAbsSendTimeExtensionId));
294 } else if (extension_type_ == RtpExtension::kTransportSequenceNumberUri) {
295 (*receive_configs)[0].transport_cc = true;
296 (*receive_configs)[0].rtp_header_extensions.push_back(RtpExtension(
297 extension_type_.c_str(), kTransportSequenceNumberExtensionId));
298 }
299}
300
stefan4fbd1452015-09-28 03:57:14 -0700301void RampUpTester::OnCallsCreated(Call* sender_call, Call* receiver_call) {
302 sender_call_ = sender_call;
stefan@webrtc.org3d7da882014-07-08 13:59:46 +0000303}
304
tommi0f8b4032017-02-22 11:22:05 -0800305void RampUpTester::BitrateStatsPollingThread(void* obj) {
306 static_cast<RampUpTester*>(obj)->PollStats();
pbos@webrtc.org32452b22014-10-22 12:15:24 +0000307}
stefan@webrtc.org3d7da882014-07-08 13:59:46 +0000308
tommi0f8b4032017-02-22 11:22:05 -0800309void RampUpTester::PollStats() {
310 do {
311 if (sender_call_) {
312 Call::Stats stats = sender_call_->GetStats();
stefan4fbd1452015-09-28 03:57:14 -0700313
tommi0f8b4032017-02-22 11:22:05 -0800314 EXPECT_GE(stats.send_bandwidth_bps, start_bitrate_bps_);
315 EXPECT_GE(expected_bitrate_bps_, 0);
316 if (stats.send_bandwidth_bps >= expected_bitrate_bps_ &&
317 (min_run_time_ms_ == -1 ||
318 clock_->TimeInMilliseconds() - test_start_ms_ >= min_run_time_ms_)) {
319 ramp_up_finished_ms_ = clock_->TimeInMilliseconds();
320 observation_complete_.Set();
321 }
stefan4fbd1452015-09-28 03:57:14 -0700322 }
tommi0f8b4032017-02-22 11:22:05 -0800323 } while (!stop_event_.Wait(kPollIntervalMs));
Erik Språng468e62a2015-07-06 10:50:47 +0200324}
325
stefan4fbd1452015-09-28 03:57:14 -0700326void RampUpTester::ReportResult(const std::string& measurement,
327 size_t value,
328 const std::string& units) const {
stefan@webrtc.org3d7da882014-07-08 13:59:46 +0000329 webrtc::test::PrintResult(
330 measurement, "",
stefanff483612015-12-21 03:14:00 -0800331 ::testing::UnitTest::GetInstance()->current_test_info()->name(), value,
332 units, false);
stefan@webrtc.org3d7da882014-07-08 13:59:46 +0000333}
334
stefan092508a2015-09-29 02:26:42 -0700335void RampUpTester::AccumulateStats(const VideoSendStream::StreamStats& stream,
336 size_t* total_packets_sent,
337 size_t* total_sent,
338 size_t* padding_sent,
339 size_t* media_sent) const {
stefan4fbd1452015-09-28 03:57:14 -0700340 *total_packets_sent += stream.rtp_stats.transmitted.packets +
341 stream.rtp_stats.retransmitted.packets +
342 stream.rtp_stats.fec.packets;
343 *total_sent += stream.rtp_stats.transmitted.TotalBytes() +
344 stream.rtp_stats.retransmitted.TotalBytes() +
345 stream.rtp_stats.fec.TotalBytes();
346 *padding_sent += stream.rtp_stats.transmitted.padding_bytes +
347 stream.rtp_stats.retransmitted.padding_bytes +
348 stream.rtp_stats.fec.padding_bytes;
349 *media_sent += stream.rtp_stats.MediaPayloadBytes();
stefan@webrtc.org3d7da882014-07-08 13:59:46 +0000350}
351
stefan4fbd1452015-09-28 03:57:14 -0700352void RampUpTester::TriggerTestDone() {
Erik Språngf3a7c9d2015-10-05 14:03:22 +0200353 RTC_DCHECK_GE(test_start_ms_, 0);
354
Stefan Holmerff2a6352016-01-14 10:00:21 +0100355 // TODO(holmer): Add audio send stats here too when those APIs are available.
mflodman86cc6ff2016-07-26 04:44:06 -0700356 if (!send_stream_)
357 return;
358
stefan4fbd1452015-09-28 03:57:14 -0700359 VideoSendStream::Stats send_stats = send_stream_->GetStats();
360
361 size_t total_packets_sent = 0;
362 size_t total_sent = 0;
363 size_t padding_sent = 0;
364 size_t media_sent = 0;
stefanff483612015-12-21 03:14:00 -0800365 for (uint32_t ssrc : video_ssrcs_) {
stefan092508a2015-09-29 02:26:42 -0700366 AccumulateStats(send_stats.substreams[ssrc], &total_packets_sent,
367 &total_sent, &padding_sent, &media_sent);
stefan4fbd1452015-09-28 03:57:14 -0700368 }
369
370 size_t rtx_total_packets_sent = 0;
371 size_t rtx_total_sent = 0;
372 size_t rtx_padding_sent = 0;
373 size_t rtx_media_sent = 0;
stefanff483612015-12-21 03:14:00 -0800374 for (uint32_t rtx_ssrc : video_rtx_ssrcs_) {
stefan092508a2015-09-29 02:26:42 -0700375 AccumulateStats(send_stats.substreams[rtx_ssrc], &rtx_total_packets_sent,
376 &rtx_total_sent, &rtx_padding_sent, &rtx_media_sent);
stefan4fbd1452015-09-28 03:57:14 -0700377 }
378
stefan5a2c5062017-01-27 06:43:18 -0800379 if (report_perf_stats_) {
stefan5a2c5062017-01-27 06:43:18 -0800380 ReportResult("ramp-up-media-sent", media_sent, "bytes");
381 ReportResult("ramp-up-padding-sent", padding_sent, "bytes");
stefan5a2c5062017-01-27 06:43:18 -0800382 ReportResult("ramp-up-rtx-media-sent", rtx_media_sent, "bytes");
383 ReportResult("ramp-up-rtx-padding-sent", rtx_padding_sent, "bytes");
384 if (ramp_up_finished_ms_ >= 0) {
385 ReportResult("ramp-up-time", ramp_up_finished_ms_ - test_start_ms_,
386 "milliseconds");
387 }
388 ReportResult("ramp-up-average-network-latency",
389 send_transport_->GetAverageDelayMs(), "milliseconds");
stefan4fbd1452015-09-28 03:57:14 -0700390 }
391}
392
393void RampUpTester::PerformTest() {
Erik Språngf3a7c9d2015-10-05 14:03:22 +0200394 test_start_ms_ = clock_->TimeInMilliseconds();
Peter Boström8c38e8b2015-11-26 17:45:47 +0100395 poller_thread_.Start();
Peter Boström5811a392015-12-10 13:02:50 +0100396 EXPECT_TRUE(Wait()) << "Timed out while waiting for ramp-up to complete.";
stefan4fbd1452015-09-28 03:57:14 -0700397 TriggerTestDone();
tommi0f8b4032017-02-22 11:22:05 -0800398 stop_event_.Set();
Peter Boström8c38e8b2015-11-26 17:45:47 +0100399 poller_thread_.Stop();
stefan4fbd1452015-09-28 03:57:14 -0700400}
401
Stefan Holmerff2a6352016-01-14 10:00:21 +0100402RampUpDownUpTester::RampUpDownUpTester(size_t num_video_streams,
403 size_t num_audio_streams,
philipel5ef2bc12017-02-21 07:28:31 -0800404 size_t num_flexfec_streams,
stefan4fbd1452015-09-28 03:57:14 -0700405 unsigned int start_bitrate_bps,
406 const std::string& extension_type,
407 bool rtx,
philipel5ef2bc12017-02-21 07:28:31 -0800408 bool red,
stefan45b5fe52017-03-09 06:27:02 -0800409 const std::vector<int>& loss_rates,
410 bool report_perf_stats)
Stefan Holmerff2a6352016-01-14 10:00:21 +0100411 : RampUpTester(num_video_streams,
412 num_audio_streams,
philipel5ef2bc12017-02-21 07:28:31 -0800413 num_flexfec_streams,
Stefan Holmerff2a6352016-01-14 10:00:21 +0100414 start_bitrate_bps,
stefan5a2c5062017-01-27 06:43:18 -0800415 0,
Stefan Holmerff2a6352016-01-14 10:00:21 +0100416 extension_type,
417 rtx,
stefan5a2c5062017-01-27 06:43:18 -0800418 red,
stefan45b5fe52017-03-09 06:27:02 -0800419 report_perf_stats),
lliuu029f7cc2017-03-31 14:17:42 -0700420 link_rates_({GetHighLinkCapacity(), kLowBandwidthLimitBps / 1000,
421 GetHighLinkCapacity(), 0}),
stefan@webrtc.org3d7da882014-07-08 13:59:46 +0000422 test_state_(kFirstRampup),
philipel5ef2bc12017-02-21 07:28:31 -0800423 next_state_(kTransitionToNextState),
stefan@webrtc.org3d7da882014-07-08 13:59:46 +0000424 state_start_ms_(clock_->TimeInMilliseconds()),
stefan4fbd1452015-09-28 03:57:14 -0700425 interval_start_ms_(clock_->TimeInMilliseconds()),
philipel5ef2bc12017-02-21 07:28:31 -0800426 sent_bytes_(0),
427 loss_rates_(loss_rates) {
428 forward_transport_config_.link_capacity_kbps = link_rates_[test_state_];
429 forward_transport_config_.queue_delay_ms = 100;
430 forward_transport_config_.loss_percent = loss_rates_[test_state_];
stefan@webrtc.org3d7da882014-07-08 13:59:46 +0000431}
432
stefan4fbd1452015-09-28 03:57:14 -0700433RampUpDownUpTester::~RampUpDownUpTester() {}
stefan@webrtc.org3d7da882014-07-08 13:59:46 +0000434
tommi0f8b4032017-02-22 11:22:05 -0800435void RampUpDownUpTester::PollStats() {
436 do {
437 if (send_stream_) {
438 webrtc::VideoSendStream::Stats stats = send_stream_->GetStats();
439 int transmit_bitrate_bps = 0;
440 for (auto it : stats.substreams) {
441 transmit_bitrate_bps += it.second.total_bitrate_bps;
442 }
443 EvolveTestState(transmit_bitrate_bps, stats.suspended);
444 } else if (num_audio_streams_ > 0 && sender_call_ != nullptr) {
445 // An audio send stream doesn't have bitrate stats, so the call send BW is
446 // currently used instead.
447 int transmit_bitrate_bps = sender_call_->GetStats().send_bandwidth_bps;
448 EvolveTestState(transmit_bitrate_bps, false);
mflodman@webrtc.orgeb16b812014-06-16 08:57:39 +0000449 }
tommi0f8b4032017-02-22 11:22:05 -0800450 } while (!stop_event_.Wait(kPollIntervalMs));
stefan@webrtc.org3d7da882014-07-08 13:59:46 +0000451}
stefan@webrtc.org7e9315b2013-12-04 10:24:26 +0000452
stefan4fbd1452015-09-28 03:57:14 -0700453Call::Config RampUpDownUpTester::GetReceiverCallConfig() {
skvlad11a9cbf2016-10-07 11:53:05 -0700454 Call::Config config(&event_log_);
stefan4fbd1452015-09-28 03:57:14 -0700455 config.bitrate_config.min_bitrate_bps = 10000;
456 return config;
stefan@webrtc.org3d7da882014-07-08 13:59:46 +0000457}
stefan@webrtc.org7e9315b2013-12-04 10:24:26 +0000458
stefan4fbd1452015-09-28 03:57:14 -0700459std::string RampUpDownUpTester::GetModifierString() const {
stefan@webrtc.org3d7da882014-07-08 13:59:46 +0000460 std::string str("_");
stefanff483612015-12-21 03:14:00 -0800461 if (num_video_streams_ > 0) {
462 std::ostringstream s;
463 s << num_video_streams_;
464 str += s.str();
465 str += "stream";
466 str += (num_video_streams_ > 1 ? "s" : "");
467 str += "_";
468 }
469 if (num_audio_streams_ > 0) {
470 std::ostringstream s;
471 s << num_audio_streams_;
472 str += s.str();
473 str += "stream";
474 str += (num_audio_streams_ > 1 ? "s" : "");
475 str += "_";
476 }
stefan4fbd1452015-09-28 03:57:14 -0700477 str += (rtx_ ? "" : "no");
stefan@webrtc.org3d7da882014-07-08 13:59:46 +0000478 str += "rtx";
479 return str;
480}
andresp@webrtc.orgc1480792014-03-20 03:23:55 +0000481
stefandb752f92016-12-05 08:23:40 -0800482int RampUpDownUpTester::GetExpectedHighBitrate() const {
stefan38d8b3c2017-01-09 04:19:24 -0800483 int expected_bitrate_bps = 0;
484 if (num_audio_streams_ > 0)
485 expected_bitrate_bps += kExpectedHighAudioBitrateBps;
486 if (num_video_streams_ > 0)
487 expected_bitrate_bps += kExpectedHighVideoBitrateBps;
488 return expected_bitrate_bps;
489}
490
lliuu029f7cc2017-03-31 14:17:42 -0700491int RampUpDownUpTester::GetHighLinkCapacity() const {
492 return 4 * GetExpectedHighBitrate() / (3 * 1000);
493}
494
philipel5ef2bc12017-02-21 07:28:31 -0800495size_t RampUpDownUpTester::GetFecBytes() const {
496 size_t flex_fec_bytes = 0;
497 if (num_flexfec_streams_ > 0) {
498 webrtc::VideoSendStream::Stats stats = send_stream_->GetStats();
499 for (const auto& kv : stats.substreams)
500 flex_fec_bytes += kv.second.rtp_stats.fec.TotalBytes();
501 }
502 return flex_fec_bytes;
503}
504
505bool RampUpDownUpTester::ExpectingFec() const {
506 return num_flexfec_streams_ > 0 && forward_transport_config_.loss_percent > 0;
507}
508
stefan4fbd1452015-09-28 03:57:14 -0700509void RampUpDownUpTester::EvolveTestState(int bitrate_bps, bool suspended) {
stefan@webrtc.org3d7da882014-07-08 13:59:46 +0000510 int64_t now = clock_->TimeInMilliseconds();
stefan@webrtc.org3d7da882014-07-08 13:59:46 +0000511 switch (test_state_) {
philipel5ef2bc12017-02-21 07:28:31 -0800512 case kFirstRampup:
stefan4fbd1452015-09-28 03:57:14 -0700513 EXPECT_FALSE(suspended);
stefandb752f92016-12-05 08:23:40 -0800514 if (bitrate_bps >= GetExpectedHighBitrate()) {
stefan45b5fe52017-03-09 06:27:02 -0800515 if (report_perf_stats_) {
516 webrtc::test::PrintResult("ramp_up_down_up", GetModifierString(),
517 "first_rampup", now - state_start_ms_, "ms",
518 false);
519 }
philipel5ef2bc12017-02-21 07:28:31 -0800520 // Apply loss during the transition between states if FEC is enabled.
521 forward_transport_config_.loss_percent = loss_rates_[test_state_];
522 test_state_ = kTransitionToNextState;
523 next_state_ = kLowRate;
andresp@webrtc.orgc1480792014-03-20 03:23:55 +0000524 }
stefan@webrtc.org3d7da882014-07-08 13:59:46 +0000525 break;
stefan@webrtc.org3d7da882014-07-08 13:59:46 +0000526 case kLowRate: {
mflodman86cc6ff2016-07-26 04:44:06 -0700527 // Audio streams are never suspended.
528 bool check_suspend_state = num_video_streams_ > 0;
529 if (bitrate_bps < kExpectedLowBitrateBps &&
530 suspended == check_suspend_state) {
stefan45b5fe52017-03-09 06:27:02 -0800531 if (report_perf_stats_) {
532 webrtc::test::PrintResult("ramp_up_down_up", GetModifierString(),
533 "rampdown", now - state_start_ms_, "ms",
534 false);
535 }
philipel5ef2bc12017-02-21 07:28:31 -0800536 // Apply loss during the transition between states if FEC is enabled.
537 forward_transport_config_.loss_percent = loss_rates_[test_state_];
538 test_state_ = kTransitionToNextState;
539 next_state_ = kSecondRampup;
stefan@webrtc.org3d7da882014-07-08 13:59:46 +0000540 }
541 break;
542 }
philipel5ef2bc12017-02-21 07:28:31 -0800543 case kSecondRampup:
stefandb752f92016-12-05 08:23:40 -0800544 if (bitrate_bps >= GetExpectedHighBitrate() && !suspended) {
stefan45b5fe52017-03-09 06:27:02 -0800545 if (report_perf_stats_) {
546 webrtc::test::PrintResult("ramp_up_down_up", GetModifierString(),
547 "second_rampup", now - state_start_ms_,
548 "ms", false);
549 ReportResult("ramp-up-down-up-average-network-latency",
550 send_transport_->GetAverageDelayMs(), "milliseconds");
551 }
philipel5ef2bc12017-02-21 07:28:31 -0800552 // Apply loss during the transition between states if FEC is enabled.
553 forward_transport_config_.loss_percent = loss_rates_[test_state_];
554 test_state_ = kTransitionToNextState;
555 next_state_ = kTestEnd;
stefan@webrtc.org3d7da882014-07-08 13:59:46 +0000556 }
557 break;
philipel5ef2bc12017-02-21 07:28:31 -0800558 case kTestEnd:
559 observation_complete_.Set();
560 break;
561 case kTransitionToNextState:
562 if (!ExpectingFec() || GetFecBytes() > 0) {
563 test_state_ = next_state_;
564 forward_transport_config_.link_capacity_kbps = link_rates_[test_state_];
565 // No loss while ramping up and down as it may affect the BWE
566 // negatively, making the test flaky.
567 forward_transport_config_.loss_percent = 0;
568 state_start_ms_ = now;
569 interval_start_ms_ = now;
570 sent_bytes_ = 0;
571 send_transport_->SetConfig(forward_transport_config_);
572 }
573 break;
stefan@webrtc.org3d7da882014-07-08 13:59:46 +0000574 }
575}
pbos@webrtc.orgf577ae92014-03-19 08:43:57 +0000576
stefan4fbd1452015-09-28 03:57:14 -0700577class RampUpTest : public test::CallTest {
Erik Språng6b8d3552015-09-24 15:06:57 +0200578 public:
stefan4fbd1452015-09-28 03:57:14 -0700579 RampUpTest() {}
Erik Språng6b8d3552015-09-24 15:06:57 +0200580
stefan4fbd1452015-09-28 03:57:14 -0700581 virtual ~RampUpTest() {
stefanff483612015-12-21 03:14:00 -0800582 EXPECT_EQ(nullptr, video_send_stream_);
583 EXPECT_TRUE(video_receive_streams_.empty());
Erik Språng6b8d3552015-09-24 15:06:57 +0200584 }
Erik Språng6b8d3552015-09-24 15:06:57 +0200585};
586
Stefan Holmerff2a6352016-01-14 10:00:21 +0100587static const uint32_t kStartBitrateBps = 60000;
588
stefan38d8b3c2017-01-09 04:19:24 -0800589TEST_F(RampUpTest, UpDownUpAbsSendTimeSimulcastRedRtx) {
philipel5ef2bc12017-02-21 07:28:31 -0800590 std::vector<int> loss_rates = {0, 0, 0, 0};
591 RampUpDownUpTester test(3, 0, 0, kStartBitrateBps,
stefan45b5fe52017-03-09 06:27:02 -0800592 RtpExtension::kAbsSendTimeUri, true, true, loss_rates,
593 true);
stefane74eef12016-01-08 06:47:13 -0800594 RunBaseTest(&test);
Shao Changbine62202f2015-04-21 20:24:50 +0800595}
Stefan Holmerff2a6352016-01-14 10:00:21 +0100596
stefan38d8b3c2017-01-09 04:19:24 -0800597TEST_F(RampUpTest, UpDownUpTransportSequenceNumberRtx) {
philipel5ef2bc12017-02-21 07:28:31 -0800598 std::vector<int> loss_rates = {0, 0, 0, 0};
599 RampUpDownUpTester test(3, 0, 0, kStartBitrateBps,
isheriff6f8d6862016-05-26 11:24:55 -0700600 RtpExtension::kTransportSequenceNumberUri, true,
stefan45b5fe52017-03-09 06:27:02 -0800601 false, loss_rates, true);
philipel5ef2bc12017-02-21 07:28:31 -0800602 RunBaseTest(&test);
603}
604
stefan45b5fe52017-03-09 06:27:02 -0800605// TODO(holmer): Tests which don't report perf stats should be moved to a
606// different executable since they per definition are not perf tests.
philipel5ef2bc12017-02-21 07:28:31 -0800607TEST_F(RampUpTest, UpDownUpTransportSequenceNumberPacketLoss) {
608 std::vector<int> loss_rates = {20, 0, 0, 0};
609 RampUpDownUpTester test(1, 0, 1, kStartBitrateBps,
610 RtpExtension::kTransportSequenceNumberUri, true,
stefan45b5fe52017-03-09 06:27:02 -0800611 false, loss_rates, false);
Stefan Holmerff2a6352016-01-14 10:00:21 +0100612 RunBaseTest(&test);
613}
614
stefan38d8b3c2017-01-09 04:19:24 -0800615TEST_F(RampUpTest, UpDownUpAudioVideoTransportSequenceNumberRtx) {
philipel5ef2bc12017-02-21 07:28:31 -0800616 std::vector<int> loss_rates = {0, 0, 0, 0};
617 RampUpDownUpTester test(3, 1, 0, kStartBitrateBps,
isheriff6f8d6862016-05-26 11:24:55 -0700618 RtpExtension::kTransportSequenceNumberUri, true,
stefanff2ebf52017-03-13 01:27:03 -0700619 false, loss_rates, false);
Stefan Holmerff2a6352016-01-14 10:00:21 +0100620 RunBaseTest(&test);
621}
622
stefan38d8b3c2017-01-09 04:19:24 -0800623TEST_F(RampUpTest, UpDownUpAudioTransportSequenceNumberRtx) {
philipel5ef2bc12017-02-21 07:28:31 -0800624 std::vector<int> loss_rates = {0, 0, 0, 0};
625 RampUpDownUpTester test(0, 1, 0, kStartBitrateBps,
mflodman86cc6ff2016-07-26 04:44:06 -0700626 RtpExtension::kTransportSequenceNumberUri, true,
stefanff2ebf52017-03-13 01:27:03 -0700627 false, loss_rates, false);
mflodman86cc6ff2016-07-26 04:44:06 -0700628 RunBaseTest(&test);
629}
630
stefan38d8b3c2017-01-09 04:19:24 -0800631TEST_F(RampUpTest, TOffsetSimulcastRedRtx) {
philipel5ef2bc12017-02-21 07:28:31 -0800632 RampUpTester test(3, 0, 0, 0, 0, RtpExtension::kTimestampOffsetUri, true,
633 true, true);
stefan38d8b3c2017-01-09 04:19:24 -0800634 RunBaseTest(&test);
635}
636
637TEST_F(RampUpTest, AbsSendTime) {
philipel5ef2bc12017-02-21 07:28:31 -0800638 RampUpTester test(1, 0, 0, 0, 0, RtpExtension::kAbsSendTimeUri, false, false,
stefanff2ebf52017-03-13 01:27:03 -0700639 false);
stefane74eef12016-01-08 06:47:13 -0800640 RunBaseTest(&test);
pbos@webrtc.org85bd53e2014-12-10 10:36:20 +0000641}
642
stefan38d8b3c2017-01-09 04:19:24 -0800643TEST_F(RampUpTest, AbsSendTimeSimulcastRedRtx) {
philipel5ef2bc12017-02-21 07:28:31 -0800644 RampUpTester test(3, 0, 0, 0, 0, RtpExtension::kAbsSendTimeUri, true, true,
stefan5a2c5062017-01-27 06:43:18 -0800645 true);
stefane74eef12016-01-08 06:47:13 -0800646 RunBaseTest(&test);
pbos@webrtc.org85bd53e2014-12-10 10:36:20 +0000647}
648
stefan38d8b3c2017-01-09 04:19:24 -0800649TEST_F(RampUpTest, TransportSequenceNumber) {
philipel5ef2bc12017-02-21 07:28:31 -0800650 RampUpTester test(1, 0, 0, 0, 0, RtpExtension::kTransportSequenceNumberUri,
stefanff2ebf52017-03-13 01:27:03 -0700651 false, false, false);
stefane74eef12016-01-08 06:47:13 -0800652 RunBaseTest(&test);
Erik Språng6b8d3552015-09-24 15:06:57 +0200653}
654
655TEST_F(RampUpTest, TransportSequenceNumberSimulcast) {
philipel5ef2bc12017-02-21 07:28:31 -0800656 RampUpTester test(3, 0, 0, 0, 0, RtpExtension::kTransportSequenceNumberUri,
stefanff2ebf52017-03-13 01:27:03 -0700657 false, false, false);
stefane74eef12016-01-08 06:47:13 -0800658 RunBaseTest(&test);
Erik Språng6b8d3552015-09-24 15:06:57 +0200659}
660
stefan38d8b3c2017-01-09 04:19:24 -0800661TEST_F(RampUpTest, TransportSequenceNumberSimulcastRedRtx) {
philipel5ef2bc12017-02-21 07:28:31 -0800662 RampUpTester test(3, 0, 0, 0, 0, RtpExtension::kTransportSequenceNumberUri,
stefan5a2c5062017-01-27 06:43:18 -0800663 true, true, true);
664 RunBaseTest(&test);
665}
666
667TEST_F(RampUpTest, AudioTransportSequenceNumber) {
philipel5ef2bc12017-02-21 07:28:31 -0800668 RampUpTester test(0, 1, 0, 300000, 10000,
stefan5a2c5062017-01-27 06:43:18 -0800669 RtpExtension::kTransportSequenceNumberUri, false, false,
670 false);
stefane74eef12016-01-08 06:47:13 -0800671 RunBaseTest(&test);
Erik Språng6b8d3552015-09-24 15:06:57 +0200672}
pbos@webrtc.org744fbc72013-09-10 09:26:25 +0000673} // namespace webrtc