blob: 6f4227dcf96265e8718fed143fb18548b3026ee7 [file] [log] [blame]
pbos@webrtc.org744fbc72013-09-10 09:26:25 +00001/*
2 * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
pbos@webrtc.org744fbc72013-09-10 09:26:25 +000010
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020011#include "call/rampup_tests.h"
stefanff483612015-12-21 03:14:00 -080012
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020013#include "rtc_base/checks.h"
14#include "rtc_base/logging.h"
15#include "rtc_base/platform_thread.h"
Jonas Olsson0a713b62018-04-04 15:49:32 +020016#include "rtc_base/stringencode.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020017#include "test/encoder_settings.h"
18#include "test/gtest.h"
19#include "test/testsupport/perf_test.h"
pbos@webrtc.org744fbc72013-09-10 09:26:25 +000020
21namespace webrtc {
pbos@webrtc.org29023282013-09-11 10:14:56 +000022namespace {
stefan@webrtc.org3d7da882014-07-08 13:59:46 +000023
Stefan Holmer723dff12015-10-05 14:59:41 +020024static const int64_t kPollIntervalMs = 20;
philipel5ef2bc12017-02-21 07:28:31 -080025static const int kExpectedHighVideoBitrateBps = 80000;
stefandb752f92016-12-05 08:23:40 -080026static const int kExpectedHighAudioBitrateBps = 30000;
27static const int kLowBandwidthLimitBps = 20000;
Sebastian Jansson12fb1702018-02-23 11:34:18 +010028// Set target detected bitrate to slightly larger than the target bitrate to
29// avoid flakiness.
30static const int kLowBitrateMarginBps = 2000;
pbos@webrtc.org29023282013-09-11 10:14:56 +000031
stefanff483612015-12-21 03:14:00 -080032std::vector<uint32_t> GenerateSsrcs(size_t num_streams, uint32_t ssrc_offset) {
stefan@webrtc.org3d7da882014-07-08 13:59:46 +000033 std::vector<uint32_t> ssrcs;
34 for (size_t i = 0; i != num_streams; ++i)
35 ssrcs.push_back(static_cast<uint32_t>(ssrc_offset + i));
36 return ssrcs;
37}
mflodman@webrtc.orgeb16b812014-06-16 08:57:39 +000038} // namespace
henrik.lundin@webrtc.org845862f2014-03-06 07:19:28 +000039
stefanff483612015-12-21 03:14:00 -080040RampUpTester::RampUpTester(size_t num_video_streams,
41 size_t num_audio_streams,
philipel5ef2bc12017-02-21 07:28:31 -080042 size_t num_flexfec_streams,
stefan4fbd1452015-09-28 03:57:14 -070043 unsigned int start_bitrate_bps,
stefan5a2c5062017-01-27 06:43:18 -080044 int64_t min_run_time_ms,
stefan4fbd1452015-09-28 03:57:14 -070045 const std::string& extension_type,
46 bool rtx,
stefan5a2c5062017-01-27 06:43:18 -080047 bool red,
48 bool report_perf_stats)
stefan4fbd1452015-09-28 03:57:14 -070049 : EndToEndTest(test::CallTest::kLongTimeoutMs),
tommi0f8b4032017-02-22 11:22:05 -080050 stop_event_(false, false),
stefan4fbd1452015-09-28 03:57:14 -070051 clock_(Clock::GetRealTimeClock()),
stefanff483612015-12-21 03:14:00 -080052 num_video_streams_(num_video_streams),
53 num_audio_streams_(num_audio_streams),
philipel5ef2bc12017-02-21 07:28:31 -080054 num_flexfec_streams_(num_flexfec_streams),
stefan4fbd1452015-09-28 03:57:14 -070055 rtx_(rtx),
56 red_(red),
stefan45b5fe52017-03-09 06:27:02 -080057 report_perf_stats_(report_perf_stats),
mflodman86cc6ff2016-07-26 04:44:06 -070058 sender_call_(nullptr),
stefan4fbd1452015-09-28 03:57:14 -070059 send_stream_(nullptr),
sprang4847ae62017-06-27 07:06:52 -070060 send_transport_(nullptr),
stefan4fbd1452015-09-28 03:57:14 -070061 start_bitrate_bps_(start_bitrate_bps),
stefan5a2c5062017-01-27 06:43:18 -080062 min_run_time_ms_(min_run_time_ms),
stefan@webrtc.org3d7da882014-07-08 13:59:46 +000063 expected_bitrate_bps_(0),
Erik Språngf3a7c9d2015-10-05 14:03:22 +020064 test_start_ms_(-1),
stefan4fbd1452015-09-28 03:57:14 -070065 ramp_up_finished_ms_(-1),
66 extension_type_(extension_type),
stefanff483612015-12-21 03:14:00 -080067 video_ssrcs_(GenerateSsrcs(num_video_streams_, 100)),
68 video_rtx_ssrcs_(GenerateSsrcs(num_video_streams_, 200)),
69 audio_ssrcs_(GenerateSsrcs(num_audio_streams_, 300)),
Peter Boström8c38e8b2015-11-26 17:45:47 +010070 poller_thread_(&BitrateStatsPollingThread,
71 this,
mflodman86cc6ff2016-07-26 04:44:06 -070072 "BitrateStatsPollingThread") {
philipel5ef2bc12017-02-21 07:28:31 -080073 if (red_)
74 EXPECT_EQ(0u, num_flexfec_streams_);
Stefan Holmerff2a6352016-01-14 10:00:21 +010075 EXPECT_LE(num_audio_streams_, 1u);
stefan4fbd1452015-09-28 03:57:14 -070076}
77
78RampUpTester::~RampUpTester() {
stefan4fbd1452015-09-28 03:57:14 -070079}
80
81Call::Config RampUpTester::GetSenderCallConfig() {
skvlad11a9cbf2016-10-07 11:53:05 -070082 Call::Config call_config(&event_log_);
stefan4fbd1452015-09-28 03:57:14 -070083 if (start_bitrate_bps_ != 0) {
84 call_config.bitrate_config.start_bitrate_bps = start_bitrate_bps_;
85 }
86 call_config.bitrate_config.min_bitrate_bps = 10000;
87 return call_config;
88}
89
stefanff483612015-12-21 03:14:00 -080090void RampUpTester::OnVideoStreamsCreated(
stefan4fbd1452015-09-28 03:57:14 -070091 VideoSendStream* send_stream,
92 const std::vector<VideoReceiveStream*>& receive_streams) {
93 send_stream_ = send_stream;
94}
95
eladalon413ee9a2017-08-22 04:02:52 -070096test::PacketTransport* RampUpTester::CreateSendTransport(
97 test::SingleThreadedTaskQueueForTesting* task_queue,
98 Call* sender_call) {
minyue20c84cc2017-04-10 16:57:57 -070099 send_transport_ = new test::PacketTransport(
eladalon413ee9a2017-08-22 04:02:52 -0700100 task_queue, sender_call, this, test::PacketTransport::kSender,
minyue20c84cc2017-04-10 16:57:57 -0700101 test::CallTest::payload_type_map_, forward_transport_config_);
stefane74eef12016-01-08 06:47:13 -0800102 return send_transport_;
stefanf116bd02015-10-27 08:29:42 -0700103}
104
Stefan Holmerd20e6512016-01-12 15:51:22 +0100105size_t RampUpTester::GetNumVideoStreams() const {
106 return num_video_streams_;
107}
108
Stefan Holmerff2a6352016-01-14 10:00:21 +0100109size_t RampUpTester::GetNumAudioStreams() const {
110 return num_audio_streams_;
111}
112
philipel5ef2bc12017-02-21 07:28:31 -0800113size_t RampUpTester::GetNumFlexfecStreams() const {
114 return num_flexfec_streams_;
115}
116
perkjfa10b552016-10-02 23:45:26 -0700117class RampUpTester::VideoStreamFactory
118 : public VideoEncoderConfig::VideoStreamFactoryInterface {
119 public:
120 VideoStreamFactory() {}
121
122 private:
123 std::vector<VideoStream> CreateEncoderStreams(
124 int width,
125 int height,
126 const VideoEncoderConfig& encoder_config) override {
127 std::vector<VideoStream> streams =
128 test::CreateVideoStreams(width, height, encoder_config);
129 if (encoder_config.number_of_streams == 1) {
130 streams[0].target_bitrate_bps = streams[0].max_bitrate_bps = 2000000;
131 }
132 return streams;
133 }
134};
135
stefanff483612015-12-21 03:14:00 -0800136void RampUpTester::ModifyVideoConfigs(
stefan4fbd1452015-09-28 03:57:14 -0700137 VideoSendStream::Config* send_config,
138 std::vector<VideoReceiveStream::Config>* receive_configs,
139 VideoEncoderConfig* encoder_config) {
140 send_config->suspend_below_min_bitrate = true;
perkjfa10b552016-10-02 23:45:26 -0700141 encoder_config->number_of_streams = num_video_streams_;
142 encoder_config->max_bitrate_bps = 2000000;
143 encoder_config->video_stream_factory =
144 new rtc::RefCountedObject<RampUpTester::VideoStreamFactory>();
stefanff483612015-12-21 03:14:00 -0800145 if (num_video_streams_ == 1) {
stefan4fbd1452015-09-28 03:57:14 -0700146 // For single stream rampup until 1mbps
147 expected_bitrate_bps_ = kSingleStreamTargetBps;
148 } else {
149 // For multi stream rampup until all streams are being sent. That means
perkjfa10b552016-10-02 23:45:26 -0700150 // enough bitrate to send all the target streams plus the min bitrate of
stefan4fbd1452015-09-28 03:57:14 -0700151 // the last one.
perkjfa10b552016-10-02 23:45:26 -0700152 std::vector<VideoStream> streams = test::CreateVideoStreams(
153 test::CallTest::kDefaultWidth, test::CallTest::kDefaultHeight,
154 *encoder_config);
155 expected_bitrate_bps_ = streams.back().min_bitrate_bps;
156 for (size_t i = 0; i < streams.size() - 1; ++i) {
157 expected_bitrate_bps_ += streams[i].target_bitrate_bps;
stefan@webrtc.org7e9315b2013-12-04 10:24:26 +0000158 }
stefan@webrtc.org3d7da882014-07-08 13:59:46 +0000159 }
stefan@webrtc.org7e9315b2013-12-04 10:24:26 +0000160
stefan4fbd1452015-09-28 03:57:14 -0700161 send_config->rtp.extensions.clear();
162
163 bool remb;
stefan43edf0f2015-11-20 18:05:48 -0800164 bool transport_cc;
isheriff6f8d6862016-05-26 11:24:55 -0700165 if (extension_type_ == RtpExtension::kAbsSendTimeUri) {
stefan4fbd1452015-09-28 03:57:14 -0700166 remb = true;
stefan43edf0f2015-11-20 18:05:48 -0800167 transport_cc = false;
stefan4fbd1452015-09-28 03:57:14 -0700168 send_config->rtp.extensions.push_back(
169 RtpExtension(extension_type_.c_str(), kAbsSendTimeExtensionId));
isheriff6f8d6862016-05-26 11:24:55 -0700170 } else if (extension_type_ == RtpExtension::kTransportSequenceNumberUri) {
stefan4fbd1452015-09-28 03:57:14 -0700171 remb = false;
stefan43edf0f2015-11-20 18:05:48 -0800172 transport_cc = true;
stefan4fbd1452015-09-28 03:57:14 -0700173 send_config->rtp.extensions.push_back(RtpExtension(
174 extension_type_.c_str(), kTransportSequenceNumberExtensionId));
stefan@webrtc.org3d7da882014-07-08 13:59:46 +0000175 } else {
stefan4fbd1452015-09-28 03:57:14 -0700176 remb = true;
stefan43edf0f2015-11-20 18:05:48 -0800177 transport_cc = false;
stefan4fbd1452015-09-28 03:57:14 -0700178 send_config->rtp.extensions.push_back(RtpExtension(
179 extension_type_.c_str(), kTransmissionTimeOffsetExtensionId));
stefan@webrtc.org3d7da882014-07-08 13:59:46 +0000180 }
stefan4fbd1452015-09-28 03:57:14 -0700181
182 send_config->rtp.nack.rtp_history_ms = test::CallTest::kNackRtpHistoryMs;
stefanff483612015-12-21 03:14:00 -0800183 send_config->rtp.ssrcs = video_ssrcs_;
stefan4fbd1452015-09-28 03:57:14 -0700184 if (rtx_) {
185 send_config->rtp.rtx.payload_type = test::CallTest::kSendRtxPayloadType;
stefanff483612015-12-21 03:14:00 -0800186 send_config->rtp.rtx.ssrcs = video_rtx_ssrcs_;
stefan4fbd1452015-09-28 03:57:14 -0700187 }
188 if (red_) {
brandtrb5f2c3f2016-10-04 23:28:39 -0700189 send_config->rtp.ulpfec.ulpfec_payload_type =
stefan4fbd1452015-09-28 03:57:14 -0700190 test::CallTest::kUlpfecPayloadType;
brandtrb5f2c3f2016-10-04 23:28:39 -0700191 send_config->rtp.ulpfec.red_payload_type = test::CallTest::kRedPayloadType;
brandtrfbfb5362016-11-17 04:18:37 -0800192 if (rtx_) {
193 send_config->rtp.ulpfec.red_rtx_payload_type =
194 test::CallTest::kRtxRedPayloadType;
195 }
stefan4fbd1452015-09-28 03:57:14 -0700196 }
197
198 size_t i = 0;
199 for (VideoReceiveStream::Config& recv_config : *receive_configs) {
200 recv_config.rtp.remb = remb;
stefan43edf0f2015-11-20 18:05:48 -0800201 recv_config.rtp.transport_cc = transport_cc;
stefan4fbd1452015-09-28 03:57:14 -0700202 recv_config.rtp.extensions = send_config->rtp.extensions;
203
stefanff483612015-12-21 03:14:00 -0800204 recv_config.rtp.remote_ssrc = video_ssrcs_[i];
stefan4fbd1452015-09-28 03:57:14 -0700205 recv_config.rtp.nack.rtp_history_ms = send_config->rtp.nack.rtp_history_ms;
206
207 if (red_) {
nisse3b3622f2017-09-26 02:49:21 -0700208 recv_config.rtp.red_payload_type =
brandtrb5f2c3f2016-10-04 23:28:39 -0700209 send_config->rtp.ulpfec.red_payload_type;
nisse3b3622f2017-09-26 02:49:21 -0700210 recv_config.rtp.ulpfec_payload_type =
brandtrb5f2c3f2016-10-04 23:28:39 -0700211 send_config->rtp.ulpfec.ulpfec_payload_type;
brandtrfbfb5362016-11-17 04:18:37 -0800212 if (rtx_) {
nisseca5706d2017-09-11 02:32:16 -0700213 recv_config.rtp.rtx_associated_payload_types
214 [send_config->rtp.ulpfec.red_rtx_payload_type] =
215 send_config->rtp.ulpfec.red_payload_type;
brandtrfbfb5362016-11-17 04:18:37 -0800216 }
stefan4fbd1452015-09-28 03:57:14 -0700217 }
218
219 if (rtx_) {
brandtr14742122017-01-27 04:53:07 -0800220 recv_config.rtp.rtx_ssrc = video_rtx_ssrcs_[i];
221 recv_config.rtp
nisse26e3abb2017-08-25 04:44:25 -0700222 .rtx_associated_payload_types[send_config->rtp.rtx.payload_type] =
Niels Möller259a4972018-04-05 15:36:51 +0200223 send_config->rtp.payload_type;
stefan4fbd1452015-09-28 03:57:14 -0700224 }
225 ++i;
226 }
philipel5ef2bc12017-02-21 07:28:31 -0800227
228 RTC_DCHECK_LE(num_flexfec_streams_, 1);
229 if (num_flexfec_streams_ == 1) {
230 send_config->rtp.flexfec.payload_type = test::CallTest::kFlexfecPayloadType;
231 send_config->rtp.flexfec.ssrc = test::CallTest::kFlexfecSendSsrc;
232 send_config->rtp.flexfec.protected_media_ssrcs = {video_ssrcs_[0]};
233 }
stefan@webrtc.org3d7da882014-07-08 13:59:46 +0000234}
235
Stefan Holmerff2a6352016-01-14 10:00:21 +0100236void RampUpTester::ModifyAudioConfigs(
237 AudioSendStream::Config* send_config,
238 std::vector<AudioReceiveStream::Config>* receive_configs) {
239 if (num_audio_streams_ == 0)
240 return;
241
isheriff6f8d6862016-05-26 11:24:55 -0700242 EXPECT_NE(RtpExtension::kTimestampOffsetUri, extension_type_)
Stefan Holmerff2a6352016-01-14 10:00:21 +0100243 << "Audio BWE not supported with toffset.";
philipel5ef2bc12017-02-21 07:28:31 -0800244 EXPECT_NE(RtpExtension::kAbsSendTimeUri, extension_type_)
245 << "Audio BWE not supported with abs-send-time.";
Stefan Holmerff2a6352016-01-14 10:00:21 +0100246
247 send_config->rtp.ssrc = audio_ssrcs_[0];
248 send_config->rtp.extensions.clear();
249
minyue10cbb462016-11-07 09:29:22 -0800250 send_config->min_bitrate_bps = 6000;
251 send_config->max_bitrate_bps = 60000;
mflodman86cc6ff2016-07-26 04:44:06 -0700252
Stefan Holmerff2a6352016-01-14 10:00:21 +0100253 bool transport_cc = false;
philipel5ef2bc12017-02-21 07:28:31 -0800254 if (extension_type_ == RtpExtension::kTransportSequenceNumberUri) {
Stefan Holmerff2a6352016-01-14 10:00:21 +0100255 transport_cc = true;
256 send_config->rtp.extensions.push_back(RtpExtension(
257 extension_type_.c_str(), kTransportSequenceNumberExtensionId));
258 }
259
260 for (AudioReceiveStream::Config& recv_config : *receive_configs) {
Stefan Holmerff2a6352016-01-14 10:00:21 +0100261 recv_config.rtp.transport_cc = transport_cc;
262 recv_config.rtp.extensions = send_config->rtp.extensions;
263 recv_config.rtp.remote_ssrc = send_config->rtp.ssrc;
264 }
265}
266
philipel5ef2bc12017-02-21 07:28:31 -0800267void RampUpTester::ModifyFlexfecConfigs(
268 std::vector<FlexfecReceiveStream::Config>* receive_configs) {
269 if (num_flexfec_streams_ == 0)
270 return;
271 RTC_DCHECK_EQ(1, num_flexfec_streams_);
272 (*receive_configs)[0].payload_type = test::CallTest::kFlexfecPayloadType;
273 (*receive_configs)[0].remote_ssrc = test::CallTest::kFlexfecSendSsrc;
274 (*receive_configs)[0].protected_media_ssrcs = {video_ssrcs_[0]};
275 (*receive_configs)[0].local_ssrc = video_ssrcs_[0];
276 if (extension_type_ == RtpExtension::kAbsSendTimeUri) {
277 (*receive_configs)[0].transport_cc = false;
278 (*receive_configs)[0].rtp_header_extensions.push_back(
279 RtpExtension(extension_type_.c_str(), kAbsSendTimeExtensionId));
280 } else if (extension_type_ == RtpExtension::kTransportSequenceNumberUri) {
281 (*receive_configs)[0].transport_cc = true;
282 (*receive_configs)[0].rtp_header_extensions.push_back(RtpExtension(
283 extension_type_.c_str(), kTransportSequenceNumberExtensionId));
284 }
285}
286
stefan4fbd1452015-09-28 03:57:14 -0700287void RampUpTester::OnCallsCreated(Call* sender_call, Call* receiver_call) {
288 sender_call_ = sender_call;
stefan@webrtc.org3d7da882014-07-08 13:59:46 +0000289}
290
tommi0f8b4032017-02-22 11:22:05 -0800291void RampUpTester::BitrateStatsPollingThread(void* obj) {
292 static_cast<RampUpTester*>(obj)->PollStats();
pbos@webrtc.org32452b22014-10-22 12:15:24 +0000293}
stefan@webrtc.org3d7da882014-07-08 13:59:46 +0000294
tommi0f8b4032017-02-22 11:22:05 -0800295void RampUpTester::PollStats() {
296 do {
297 if (sender_call_) {
298 Call::Stats stats = sender_call_->GetStats();
stefan4fbd1452015-09-28 03:57:14 -0700299
tommi0f8b4032017-02-22 11:22:05 -0800300 EXPECT_GE(stats.send_bandwidth_bps, start_bitrate_bps_);
301 EXPECT_GE(expected_bitrate_bps_, 0);
302 if (stats.send_bandwidth_bps >= expected_bitrate_bps_ &&
303 (min_run_time_ms_ == -1 ||
304 clock_->TimeInMilliseconds() - test_start_ms_ >= min_run_time_ms_)) {
305 ramp_up_finished_ms_ = clock_->TimeInMilliseconds();
306 observation_complete_.Set();
307 }
stefan4fbd1452015-09-28 03:57:14 -0700308 }
tommi0f8b4032017-02-22 11:22:05 -0800309 } while (!stop_event_.Wait(kPollIntervalMs));
Erik Språng468e62a2015-07-06 10:50:47 +0200310}
311
stefan4fbd1452015-09-28 03:57:14 -0700312void RampUpTester::ReportResult(const std::string& measurement,
313 size_t value,
314 const std::string& units) const {
stefan@webrtc.org3d7da882014-07-08 13:59:46 +0000315 webrtc::test::PrintResult(
316 measurement, "",
stefanff483612015-12-21 03:14:00 -0800317 ::testing::UnitTest::GetInstance()->current_test_info()->name(), value,
318 units, false);
stefan@webrtc.org3d7da882014-07-08 13:59:46 +0000319}
320
stefan092508a2015-09-29 02:26:42 -0700321void RampUpTester::AccumulateStats(const VideoSendStream::StreamStats& stream,
322 size_t* total_packets_sent,
323 size_t* total_sent,
324 size_t* padding_sent,
325 size_t* media_sent) const {
stefan4fbd1452015-09-28 03:57:14 -0700326 *total_packets_sent += stream.rtp_stats.transmitted.packets +
327 stream.rtp_stats.retransmitted.packets +
328 stream.rtp_stats.fec.packets;
329 *total_sent += stream.rtp_stats.transmitted.TotalBytes() +
330 stream.rtp_stats.retransmitted.TotalBytes() +
331 stream.rtp_stats.fec.TotalBytes();
332 *padding_sent += stream.rtp_stats.transmitted.padding_bytes +
333 stream.rtp_stats.retransmitted.padding_bytes +
334 stream.rtp_stats.fec.padding_bytes;
335 *media_sent += stream.rtp_stats.MediaPayloadBytes();
stefan@webrtc.org3d7da882014-07-08 13:59:46 +0000336}
337
stefan4fbd1452015-09-28 03:57:14 -0700338void RampUpTester::TriggerTestDone() {
Erik Språngf3a7c9d2015-10-05 14:03:22 +0200339 RTC_DCHECK_GE(test_start_ms_, 0);
340
Stefan Holmerff2a6352016-01-14 10:00:21 +0100341 // TODO(holmer): Add audio send stats here too when those APIs are available.
mflodman86cc6ff2016-07-26 04:44:06 -0700342 if (!send_stream_)
343 return;
344
stefan4fbd1452015-09-28 03:57:14 -0700345 VideoSendStream::Stats send_stats = send_stream_->GetStats();
346
347 size_t total_packets_sent = 0;
348 size_t total_sent = 0;
349 size_t padding_sent = 0;
350 size_t media_sent = 0;
stefanff483612015-12-21 03:14:00 -0800351 for (uint32_t ssrc : video_ssrcs_) {
stefan092508a2015-09-29 02:26:42 -0700352 AccumulateStats(send_stats.substreams[ssrc], &total_packets_sent,
353 &total_sent, &padding_sent, &media_sent);
stefan4fbd1452015-09-28 03:57:14 -0700354 }
355
356 size_t rtx_total_packets_sent = 0;
357 size_t rtx_total_sent = 0;
358 size_t rtx_padding_sent = 0;
359 size_t rtx_media_sent = 0;
stefanff483612015-12-21 03:14:00 -0800360 for (uint32_t rtx_ssrc : video_rtx_ssrcs_) {
stefan092508a2015-09-29 02:26:42 -0700361 AccumulateStats(send_stats.substreams[rtx_ssrc], &rtx_total_packets_sent,
362 &rtx_total_sent, &rtx_padding_sent, &rtx_media_sent);
stefan4fbd1452015-09-28 03:57:14 -0700363 }
364
stefan5a2c5062017-01-27 06:43:18 -0800365 if (report_perf_stats_) {
stefan5a2c5062017-01-27 06:43:18 -0800366 ReportResult("ramp-up-media-sent", media_sent, "bytes");
367 ReportResult("ramp-up-padding-sent", padding_sent, "bytes");
stefan5a2c5062017-01-27 06:43:18 -0800368 ReportResult("ramp-up-rtx-media-sent", rtx_media_sent, "bytes");
369 ReportResult("ramp-up-rtx-padding-sent", rtx_padding_sent, "bytes");
370 if (ramp_up_finished_ms_ >= 0) {
371 ReportResult("ramp-up-time", ramp_up_finished_ms_ - test_start_ms_,
372 "milliseconds");
373 }
374 ReportResult("ramp-up-average-network-latency",
375 send_transport_->GetAverageDelayMs(), "milliseconds");
stefan4fbd1452015-09-28 03:57:14 -0700376 }
377}
378
379void RampUpTester::PerformTest() {
Erik Språngf3a7c9d2015-10-05 14:03:22 +0200380 test_start_ms_ = clock_->TimeInMilliseconds();
Peter Boström8c38e8b2015-11-26 17:45:47 +0100381 poller_thread_.Start();
Peter Boström5811a392015-12-10 13:02:50 +0100382 EXPECT_TRUE(Wait()) << "Timed out while waiting for ramp-up to complete.";
stefan4fbd1452015-09-28 03:57:14 -0700383 TriggerTestDone();
tommi0f8b4032017-02-22 11:22:05 -0800384 stop_event_.Set();
Peter Boström8c38e8b2015-11-26 17:45:47 +0100385 poller_thread_.Stop();
stefan4fbd1452015-09-28 03:57:14 -0700386}
387
Stefan Holmerff2a6352016-01-14 10:00:21 +0100388RampUpDownUpTester::RampUpDownUpTester(size_t num_video_streams,
389 size_t num_audio_streams,
philipel5ef2bc12017-02-21 07:28:31 -0800390 size_t num_flexfec_streams,
stefan4fbd1452015-09-28 03:57:14 -0700391 unsigned int start_bitrate_bps,
392 const std::string& extension_type,
393 bool rtx,
philipel5ef2bc12017-02-21 07:28:31 -0800394 bool red,
stefan45b5fe52017-03-09 06:27:02 -0800395 const std::vector<int>& loss_rates,
396 bool report_perf_stats)
Stefan Holmerff2a6352016-01-14 10:00:21 +0100397 : RampUpTester(num_video_streams,
398 num_audio_streams,
philipel5ef2bc12017-02-21 07:28:31 -0800399 num_flexfec_streams,
Stefan Holmerff2a6352016-01-14 10:00:21 +0100400 start_bitrate_bps,
stefan5a2c5062017-01-27 06:43:18 -0800401 0,
Stefan Holmerff2a6352016-01-14 10:00:21 +0100402 extension_type,
403 rtx,
stefan5a2c5062017-01-27 06:43:18 -0800404 red,
stefan45b5fe52017-03-09 06:27:02 -0800405 report_perf_stats),
stefan76d9c9c2017-04-01 06:51:09 -0700406 link_rates_({4 * GetExpectedHighBitrate() / (3 * 1000),
407 kLowBandwidthLimitBps / 1000,
408 4 * GetExpectedHighBitrate() / (3 * 1000), 0}),
stefan@webrtc.org3d7da882014-07-08 13:59:46 +0000409 test_state_(kFirstRampup),
philipel5ef2bc12017-02-21 07:28:31 -0800410 next_state_(kTransitionToNextState),
stefan@webrtc.org3d7da882014-07-08 13:59:46 +0000411 state_start_ms_(clock_->TimeInMilliseconds()),
stefan4fbd1452015-09-28 03:57:14 -0700412 interval_start_ms_(clock_->TimeInMilliseconds()),
philipel5ef2bc12017-02-21 07:28:31 -0800413 sent_bytes_(0),
414 loss_rates_(loss_rates) {
415 forward_transport_config_.link_capacity_kbps = link_rates_[test_state_];
416 forward_transport_config_.queue_delay_ms = 100;
417 forward_transport_config_.loss_percent = loss_rates_[test_state_];
stefan@webrtc.org3d7da882014-07-08 13:59:46 +0000418}
419
stefan4fbd1452015-09-28 03:57:14 -0700420RampUpDownUpTester::~RampUpDownUpTester() {}
stefan@webrtc.org3d7da882014-07-08 13:59:46 +0000421
tommi0f8b4032017-02-22 11:22:05 -0800422void RampUpDownUpTester::PollStats() {
423 do {
Alex Narest78609d52017-10-20 10:37:47 +0200424 int transmit_bitrate_bps = 0;
425 bool suspended = false;
426 if (num_video_streams_ > 0) {
tommi0f8b4032017-02-22 11:22:05 -0800427 webrtc::VideoSendStream::Stats stats = send_stream_->GetStats();
tommi0f8b4032017-02-22 11:22:05 -0800428 for (auto it : stats.substreams) {
429 transmit_bitrate_bps += it.second.total_bitrate_bps;
430 }
Alex Narest78609d52017-10-20 10:37:47 +0200431 suspended = stats.suspended;
432 }
433 if (num_audio_streams_ > 0 && sender_call_ != nullptr) {
tommi0f8b4032017-02-22 11:22:05 -0800434 // An audio send stream doesn't have bitrate stats, so the call send BW is
435 // currently used instead.
Alex Narest78609d52017-10-20 10:37:47 +0200436 transmit_bitrate_bps = sender_call_->GetStats().send_bandwidth_bps;
mflodman@webrtc.orgeb16b812014-06-16 08:57:39 +0000437 }
Alex Narest78609d52017-10-20 10:37:47 +0200438 EvolveTestState(transmit_bitrate_bps, suspended);
tommi0f8b4032017-02-22 11:22:05 -0800439 } while (!stop_event_.Wait(kPollIntervalMs));
stefan@webrtc.org3d7da882014-07-08 13:59:46 +0000440}
stefan@webrtc.org7e9315b2013-12-04 10:24:26 +0000441
stefan4fbd1452015-09-28 03:57:14 -0700442Call::Config RampUpDownUpTester::GetReceiverCallConfig() {
skvlad11a9cbf2016-10-07 11:53:05 -0700443 Call::Config config(&event_log_);
stefan4fbd1452015-09-28 03:57:14 -0700444 config.bitrate_config.min_bitrate_bps = 10000;
445 return config;
stefan@webrtc.org3d7da882014-07-08 13:59:46 +0000446}
stefan@webrtc.org7e9315b2013-12-04 10:24:26 +0000447
stefan4fbd1452015-09-28 03:57:14 -0700448std::string RampUpDownUpTester::GetModifierString() const {
stefan@webrtc.org3d7da882014-07-08 13:59:46 +0000449 std::string str("_");
stefanff483612015-12-21 03:14:00 -0800450 if (num_video_streams_ > 0) {
Jonas Olsson0a713b62018-04-04 15:49:32 +0200451 str += rtc::ToString(num_video_streams_);
stefanff483612015-12-21 03:14:00 -0800452 str += "stream";
453 str += (num_video_streams_ > 1 ? "s" : "");
454 str += "_";
455 }
456 if (num_audio_streams_ > 0) {
Jonas Olsson0a713b62018-04-04 15:49:32 +0200457 str += rtc::ToString(num_audio_streams_);
stefanff483612015-12-21 03:14:00 -0800458 str += "stream";
459 str += (num_audio_streams_ > 1 ? "s" : "");
460 str += "_";
461 }
stefan4fbd1452015-09-28 03:57:14 -0700462 str += (rtx_ ? "" : "no");
Edward Lemurdd3987f2018-01-04 19:44:54 +0100463 str += "rtx_";
464 str += (red_ ? "" : "no");
465 str += "red";
stefan@webrtc.org3d7da882014-07-08 13:59:46 +0000466 return str;
467}
andresp@webrtc.orgc1480792014-03-20 03:23:55 +0000468
stefandb752f92016-12-05 08:23:40 -0800469int RampUpDownUpTester::GetExpectedHighBitrate() const {
stefan38d8b3c2017-01-09 04:19:24 -0800470 int expected_bitrate_bps = 0;
471 if (num_audio_streams_ > 0)
472 expected_bitrate_bps += kExpectedHighAudioBitrateBps;
473 if (num_video_streams_ > 0)
474 expected_bitrate_bps += kExpectedHighVideoBitrateBps;
475 return expected_bitrate_bps;
476}
477
philipel5ef2bc12017-02-21 07:28:31 -0800478size_t RampUpDownUpTester::GetFecBytes() const {
479 size_t flex_fec_bytes = 0;
480 if (num_flexfec_streams_ > 0) {
481 webrtc::VideoSendStream::Stats stats = send_stream_->GetStats();
482 for (const auto& kv : stats.substreams)
483 flex_fec_bytes += kv.second.rtp_stats.fec.TotalBytes();
484 }
485 return flex_fec_bytes;
486}
487
488bool RampUpDownUpTester::ExpectingFec() const {
489 return num_flexfec_streams_ > 0 && forward_transport_config_.loss_percent > 0;
490}
491
stefan4fbd1452015-09-28 03:57:14 -0700492void RampUpDownUpTester::EvolveTestState(int bitrate_bps, bool suspended) {
stefan@webrtc.org3d7da882014-07-08 13:59:46 +0000493 int64_t now = clock_->TimeInMilliseconds();
stefan@webrtc.org3d7da882014-07-08 13:59:46 +0000494 switch (test_state_) {
philipel5ef2bc12017-02-21 07:28:31 -0800495 case kFirstRampup:
stefan4fbd1452015-09-28 03:57:14 -0700496 EXPECT_FALSE(suspended);
stefandb752f92016-12-05 08:23:40 -0800497 if (bitrate_bps >= GetExpectedHighBitrate()) {
stefan45b5fe52017-03-09 06:27:02 -0800498 if (report_perf_stats_) {
499 webrtc::test::PrintResult("ramp_up_down_up", GetModifierString(),
500 "first_rampup", now - state_start_ms_, "ms",
501 false);
502 }
philipel5ef2bc12017-02-21 07:28:31 -0800503 // Apply loss during the transition between states if FEC is enabled.
504 forward_transport_config_.loss_percent = loss_rates_[test_state_];
505 test_state_ = kTransitionToNextState;
506 next_state_ = kLowRate;
andresp@webrtc.orgc1480792014-03-20 03:23:55 +0000507 }
stefan@webrtc.org3d7da882014-07-08 13:59:46 +0000508 break;
stefan@webrtc.org3d7da882014-07-08 13:59:46 +0000509 case kLowRate: {
mflodman86cc6ff2016-07-26 04:44:06 -0700510 // Audio streams are never suspended.
511 bool check_suspend_state = num_video_streams_ > 0;
Sebastian Jansson12fb1702018-02-23 11:34:18 +0100512 if (bitrate_bps < kLowBandwidthLimitBps + kLowBitrateMarginBps &&
mflodman86cc6ff2016-07-26 04:44:06 -0700513 suspended == check_suspend_state) {
stefan45b5fe52017-03-09 06:27:02 -0800514 if (report_perf_stats_) {
515 webrtc::test::PrintResult("ramp_up_down_up", GetModifierString(),
516 "rampdown", now - state_start_ms_, "ms",
517 false);
518 }
philipel5ef2bc12017-02-21 07:28:31 -0800519 // Apply loss during the transition between states if FEC is enabled.
520 forward_transport_config_.loss_percent = loss_rates_[test_state_];
521 test_state_ = kTransitionToNextState;
522 next_state_ = kSecondRampup;
stefan@webrtc.org3d7da882014-07-08 13:59:46 +0000523 }
524 break;
525 }
philipel5ef2bc12017-02-21 07:28:31 -0800526 case kSecondRampup:
stefandb752f92016-12-05 08:23:40 -0800527 if (bitrate_bps >= GetExpectedHighBitrate() && !suspended) {
stefan45b5fe52017-03-09 06:27:02 -0800528 if (report_perf_stats_) {
529 webrtc::test::PrintResult("ramp_up_down_up", GetModifierString(),
530 "second_rampup", now - state_start_ms_,
531 "ms", false);
532 ReportResult("ramp-up-down-up-average-network-latency",
533 send_transport_->GetAverageDelayMs(), "milliseconds");
534 }
philipel5ef2bc12017-02-21 07:28:31 -0800535 // Apply loss during the transition between states if FEC is enabled.
536 forward_transport_config_.loss_percent = loss_rates_[test_state_];
537 test_state_ = kTransitionToNextState;
538 next_state_ = kTestEnd;
stefan@webrtc.org3d7da882014-07-08 13:59:46 +0000539 }
540 break;
philipel5ef2bc12017-02-21 07:28:31 -0800541 case kTestEnd:
542 observation_complete_.Set();
543 break;
544 case kTransitionToNextState:
545 if (!ExpectingFec() || GetFecBytes() > 0) {
546 test_state_ = next_state_;
547 forward_transport_config_.link_capacity_kbps = link_rates_[test_state_];
548 // No loss while ramping up and down as it may affect the BWE
549 // negatively, making the test flaky.
550 forward_transport_config_.loss_percent = 0;
551 state_start_ms_ = now;
552 interval_start_ms_ = now;
553 sent_bytes_ = 0;
554 send_transport_->SetConfig(forward_transport_config_);
555 }
556 break;
stefan@webrtc.org3d7da882014-07-08 13:59:46 +0000557 }
558}
pbos@webrtc.orgf577ae92014-03-19 08:43:57 +0000559
stefan4fbd1452015-09-28 03:57:14 -0700560class RampUpTest : public test::CallTest {
Erik Språng6b8d3552015-09-24 15:06:57 +0200561 public:
stefan4fbd1452015-09-28 03:57:14 -0700562 RampUpTest() {}
Erik Språng6b8d3552015-09-24 15:06:57 +0200563
stefan4fbd1452015-09-28 03:57:14 -0700564 virtual ~RampUpTest() {
stefanff483612015-12-21 03:14:00 -0800565 EXPECT_EQ(nullptr, video_send_stream_);
566 EXPECT_TRUE(video_receive_streams_.empty());
Erik Språng6b8d3552015-09-24 15:06:57 +0200567 }
Erik Språng6b8d3552015-09-24 15:06:57 +0200568};
569
Stefan Holmerff2a6352016-01-14 10:00:21 +0100570static const uint32_t kStartBitrateBps = 60000;
571
stefan38d8b3c2017-01-09 04:19:24 -0800572TEST_F(RampUpTest, UpDownUpAbsSendTimeSimulcastRedRtx) {
philipel5ef2bc12017-02-21 07:28:31 -0800573 std::vector<int> loss_rates = {0, 0, 0, 0};
574 RampUpDownUpTester test(3, 0, 0, kStartBitrateBps,
stefan45b5fe52017-03-09 06:27:02 -0800575 RtpExtension::kAbsSendTimeUri, true, true, loss_rates,
576 true);
stefane74eef12016-01-08 06:47:13 -0800577 RunBaseTest(&test);
Shao Changbine62202f2015-04-21 20:24:50 +0800578}
Stefan Holmerff2a6352016-01-14 10:00:21 +0100579
Taylor Brandstetterba6b5032018-02-16 09:59:56 -0800580// TODO(bugs.webrtc.org/8878)
581#if defined(WEBRTC_MAC)
582#define MAYBE_UpDownUpTransportSequenceNumberRtx \
583 DISABLED_UpDownUpTransportSequenceNumberRtx
584#else
585#define MAYBE_UpDownUpTransportSequenceNumberRtx \
586 UpDownUpTransportSequenceNumberRtx
587#endif
588TEST_F(RampUpTest, MAYBE_UpDownUpTransportSequenceNumberRtx) {
philipel5ef2bc12017-02-21 07:28:31 -0800589 std::vector<int> loss_rates = {0, 0, 0, 0};
590 RampUpDownUpTester test(3, 0, 0, kStartBitrateBps,
isheriff6f8d6862016-05-26 11:24:55 -0700591 RtpExtension::kTransportSequenceNumberUri, true,
stefan45b5fe52017-03-09 06:27:02 -0800592 false, loss_rates, true);
philipel5ef2bc12017-02-21 07:28:31 -0800593 RunBaseTest(&test);
594}
595
stefan45b5fe52017-03-09 06:27:02 -0800596// TODO(holmer): Tests which don't report perf stats should be moved to a
597// different executable since they per definition are not perf tests.
Taylor Brandstetter05d98222017-10-10 12:12:53 -0700598// This test is disabled because it crashes on Linux, and is flaky on other
599// platforms. See: crbug.com/webrtc/7919
600TEST_F(RampUpTest, DISABLED_UpDownUpTransportSequenceNumberPacketLoss) {
philipel5ef2bc12017-02-21 07:28:31 -0800601 std::vector<int> loss_rates = {20, 0, 0, 0};
602 RampUpDownUpTester test(1, 0, 1, kStartBitrateBps,
603 RtpExtension::kTransportSequenceNumberUri, true,
stefan45b5fe52017-03-09 06:27:02 -0800604 false, loss_rates, false);
Stefan Holmerff2a6352016-01-14 10:00:21 +0100605 RunBaseTest(&test);
606}
607
Taylor Brandstetterab86e7f2018-02-12 15:16:02 -0800608// TODO(bugs.webrtc.org/8878)
609#if defined(WEBRTC_MAC)
610#define MAYBE_UpDownUpAudioVideoTransportSequenceNumberRtx \
611 DISABLED_UpDownUpAudioVideoTransportSequenceNumberRtx
612#else
613#define MAYBE_UpDownUpAudioVideoTransportSequenceNumberRtx \
614 UpDownUpAudioVideoTransportSequenceNumberRtx
615#endif
616TEST_F(RampUpTest, MAYBE_UpDownUpAudioVideoTransportSequenceNumberRtx) {
philipel5ef2bc12017-02-21 07:28:31 -0800617 std::vector<int> loss_rates = {0, 0, 0, 0};
618 RampUpDownUpTester test(3, 1, 0, kStartBitrateBps,
isheriff6f8d6862016-05-26 11:24:55 -0700619 RtpExtension::kTransportSequenceNumberUri, true,
stefanff2ebf52017-03-13 01:27:03 -0700620 false, loss_rates, false);
Stefan Holmerff2a6352016-01-14 10:00:21 +0100621 RunBaseTest(&test);
622}
623
stefan38d8b3c2017-01-09 04:19:24 -0800624TEST_F(RampUpTest, UpDownUpAudioTransportSequenceNumberRtx) {
philipel5ef2bc12017-02-21 07:28:31 -0800625 std::vector<int> loss_rates = {0, 0, 0, 0};
626 RampUpDownUpTester test(0, 1, 0, kStartBitrateBps,
mflodman86cc6ff2016-07-26 04:44:06 -0700627 RtpExtension::kTransportSequenceNumberUri, true,
stefanff2ebf52017-03-13 01:27:03 -0700628 false, loss_rates, false);
mflodman86cc6ff2016-07-26 04:44:06 -0700629 RunBaseTest(&test);
630}
631
stefan38d8b3c2017-01-09 04:19:24 -0800632TEST_F(RampUpTest, TOffsetSimulcastRedRtx) {
philipel5ef2bc12017-02-21 07:28:31 -0800633 RampUpTester test(3, 0, 0, 0, 0, RtpExtension::kTimestampOffsetUri, true,
634 true, true);
stefan38d8b3c2017-01-09 04:19:24 -0800635 RunBaseTest(&test);
636}
637
638TEST_F(RampUpTest, AbsSendTime) {
philipel5ef2bc12017-02-21 07:28:31 -0800639 RampUpTester test(1, 0, 0, 0, 0, RtpExtension::kAbsSendTimeUri, false, false,
stefanff2ebf52017-03-13 01:27:03 -0700640 false);
stefane74eef12016-01-08 06:47:13 -0800641 RunBaseTest(&test);
pbos@webrtc.org85bd53e2014-12-10 10:36:20 +0000642}
643
stefan38d8b3c2017-01-09 04:19:24 -0800644TEST_F(RampUpTest, AbsSendTimeSimulcastRedRtx) {
philipel5ef2bc12017-02-21 07:28:31 -0800645 RampUpTester test(3, 0, 0, 0, 0, RtpExtension::kAbsSendTimeUri, true, true,
stefan5a2c5062017-01-27 06:43:18 -0800646 true);
stefane74eef12016-01-08 06:47:13 -0800647 RunBaseTest(&test);
pbos@webrtc.org85bd53e2014-12-10 10:36:20 +0000648}
649
stefan38d8b3c2017-01-09 04:19:24 -0800650TEST_F(RampUpTest, TransportSequenceNumber) {
philipel5ef2bc12017-02-21 07:28:31 -0800651 RampUpTester test(1, 0, 0, 0, 0, RtpExtension::kTransportSequenceNumberUri,
stefanff2ebf52017-03-13 01:27:03 -0700652 false, false, false);
stefane74eef12016-01-08 06:47:13 -0800653 RunBaseTest(&test);
Erik Språng6b8d3552015-09-24 15:06:57 +0200654}
655
656TEST_F(RampUpTest, TransportSequenceNumberSimulcast) {
philipel5ef2bc12017-02-21 07:28:31 -0800657 RampUpTester test(3, 0, 0, 0, 0, RtpExtension::kTransportSequenceNumberUri,
stefanff2ebf52017-03-13 01:27:03 -0700658 false, false, false);
stefane74eef12016-01-08 06:47:13 -0800659 RunBaseTest(&test);
Erik Språng6b8d3552015-09-24 15:06:57 +0200660}
661
stefan38d8b3c2017-01-09 04:19:24 -0800662TEST_F(RampUpTest, TransportSequenceNumberSimulcastRedRtx) {
philipel5ef2bc12017-02-21 07:28:31 -0800663 RampUpTester test(3, 0, 0, 0, 0, RtpExtension::kTransportSequenceNumberUri,
stefan5a2c5062017-01-27 06:43:18 -0800664 true, true, true);
665 RunBaseTest(&test);
666}
667
Taylor Brandstetterab86e7f2018-02-12 15:16:02 -0800668// TODO(bugs.webrtc.org/8878)
669#if defined(WEBRTC_MAC)
670#define MAYBE_AudioTransportSequenceNumber DISABLED_AudioTransportSequenceNumber
671#else
672#define MAYBE_AudioTransportSequenceNumber AudioTransportSequenceNumber
673#endif
674TEST_F(RampUpTest, MAYBE_AudioTransportSequenceNumber) {
philipel5ef2bc12017-02-21 07:28:31 -0800675 RampUpTester test(0, 1, 0, 300000, 10000,
stefan5a2c5062017-01-27 06:43:18 -0800676 RtpExtension::kTransportSequenceNumberUri, false, false,
677 false);
stefane74eef12016-01-08 06:47:13 -0800678 RunBaseTest(&test);
Erik Språng6b8d3552015-09-24 15:06:57 +0200679}
pbos@webrtc.org744fbc72013-09-10 09:26:25 +0000680} // namespace webrtc