blob: 17b729629a7a0857d9318e1f833b8348a3affdda [file] [log] [blame]
pbos@webrtc.org744fbc72013-09-10 09:26:25 +00001/*
2 * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
pbos@webrtc.org744fbc72013-09-10 09:26:25 +000010
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020011#include "call/rampup_tests.h"
stefanff483612015-12-21 03:14:00 -080012
Artem Titov4e199e92018-08-20 13:30:39 +020013#include "call/fake_network_pipe.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020014#include "rtc_base/checks.h"
15#include "rtc_base/logging.h"
16#include "rtc_base/platform_thread.h"
Jonas Olsson0a713b62018-04-04 15:49:32 +020017#include "rtc_base/stringencode.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020018#include "test/encoder_settings.h"
19#include "test/gtest.h"
20#include "test/testsupport/perf_test.h"
pbos@webrtc.org744fbc72013-09-10 09:26:25 +000021
22namespace webrtc {
pbos@webrtc.org29023282013-09-11 10:14:56 +000023namespace {
stefan@webrtc.org3d7da882014-07-08 13:59:46 +000024
Stefan Holmer723dff12015-10-05 14:59:41 +020025static const int64_t kPollIntervalMs = 20;
philipel5ef2bc12017-02-21 07:28:31 -080026static const int kExpectedHighVideoBitrateBps = 80000;
stefandb752f92016-12-05 08:23:40 -080027static const int kExpectedHighAudioBitrateBps = 30000;
28static const int kLowBandwidthLimitBps = 20000;
Sebastian Jansson12fb1702018-02-23 11:34:18 +010029// Set target detected bitrate to slightly larger than the target bitrate to
30// avoid flakiness.
31static const int kLowBitrateMarginBps = 2000;
pbos@webrtc.org29023282013-09-11 10:14:56 +000032
stefanff483612015-12-21 03:14:00 -080033std::vector<uint32_t> GenerateSsrcs(size_t num_streams, uint32_t ssrc_offset) {
stefan@webrtc.org3d7da882014-07-08 13:59:46 +000034 std::vector<uint32_t> ssrcs;
35 for (size_t i = 0; i != num_streams; ++i)
36 ssrcs.push_back(static_cast<uint32_t>(ssrc_offset + i));
37 return ssrcs;
38}
mflodman@webrtc.orgeb16b812014-06-16 08:57:39 +000039} // namespace
henrik.lundin@webrtc.org845862f2014-03-06 07:19:28 +000040
stefanff483612015-12-21 03:14:00 -080041RampUpTester::RampUpTester(size_t num_video_streams,
42 size_t num_audio_streams,
philipel5ef2bc12017-02-21 07:28:31 -080043 size_t num_flexfec_streams,
stefan4fbd1452015-09-28 03:57:14 -070044 unsigned int start_bitrate_bps,
stefan5a2c5062017-01-27 06:43:18 -080045 int64_t min_run_time_ms,
stefan4fbd1452015-09-28 03:57:14 -070046 const std::string& extension_type,
47 bool rtx,
stefan5a2c5062017-01-27 06:43:18 -080048 bool red,
49 bool report_perf_stats)
stefan4fbd1452015-09-28 03:57:14 -070050 : EndToEndTest(test::CallTest::kLongTimeoutMs),
tommi0f8b4032017-02-22 11:22:05 -080051 stop_event_(false, false),
stefan4fbd1452015-09-28 03:57:14 -070052 clock_(Clock::GetRealTimeClock()),
stefanff483612015-12-21 03:14:00 -080053 num_video_streams_(num_video_streams),
54 num_audio_streams_(num_audio_streams),
philipel5ef2bc12017-02-21 07:28:31 -080055 num_flexfec_streams_(num_flexfec_streams),
stefan4fbd1452015-09-28 03:57:14 -070056 rtx_(rtx),
57 red_(red),
stefan45b5fe52017-03-09 06:27:02 -080058 report_perf_stats_(report_perf_stats),
mflodman86cc6ff2016-07-26 04:44:06 -070059 sender_call_(nullptr),
stefan4fbd1452015-09-28 03:57:14 -070060 send_stream_(nullptr),
sprang4847ae62017-06-27 07:06:52 -070061 send_transport_(nullptr),
Artem Titov631cafa2018-08-21 21:01:00 +020062 send_simulated_network_(nullptr),
stefan4fbd1452015-09-28 03:57:14 -070063 start_bitrate_bps_(start_bitrate_bps),
stefan5a2c5062017-01-27 06:43:18 -080064 min_run_time_ms_(min_run_time_ms),
stefan@webrtc.org3d7da882014-07-08 13:59:46 +000065 expected_bitrate_bps_(0),
Erik Språngf3a7c9d2015-10-05 14:03:22 +020066 test_start_ms_(-1),
stefan4fbd1452015-09-28 03:57:14 -070067 ramp_up_finished_ms_(-1),
68 extension_type_(extension_type),
stefanff483612015-12-21 03:14:00 -080069 video_ssrcs_(GenerateSsrcs(num_video_streams_, 100)),
70 video_rtx_ssrcs_(GenerateSsrcs(num_video_streams_, 200)),
71 audio_ssrcs_(GenerateSsrcs(num_audio_streams_, 300)),
Peter Boström8c38e8b2015-11-26 17:45:47 +010072 poller_thread_(&BitrateStatsPollingThread,
73 this,
mflodman86cc6ff2016-07-26 04:44:06 -070074 "BitrateStatsPollingThread") {
philipel5ef2bc12017-02-21 07:28:31 -080075 if (red_)
76 EXPECT_EQ(0u, num_flexfec_streams_);
Stefan Holmerff2a6352016-01-14 10:00:21 +010077 EXPECT_LE(num_audio_streams_, 1u);
stefan4fbd1452015-09-28 03:57:14 -070078}
79
Yves Gerey665174f2018-06-19 15:03:05 +020080RampUpTester::~RampUpTester() {}
stefan4fbd1452015-09-28 03:57:14 -070081
Sebastian Jansson72582242018-07-13 13:19:42 +020082void RampUpTester::ModifySenderCallConfig(Call::Config* config) {
stefan4fbd1452015-09-28 03:57:14 -070083 if (start_bitrate_bps_ != 0) {
Sebastian Jansson72582242018-07-13 13:19:42 +020084 config->bitrate_config.start_bitrate_bps = start_bitrate_bps_;
stefan4fbd1452015-09-28 03:57:14 -070085 }
Sebastian Jansson72582242018-07-13 13:19:42 +020086 config->bitrate_config.min_bitrate_bps = 10000;
stefan4fbd1452015-09-28 03:57:14 -070087}
88
stefanff483612015-12-21 03:14:00 -080089void RampUpTester::OnVideoStreamsCreated(
stefan4fbd1452015-09-28 03:57:14 -070090 VideoSendStream* send_stream,
91 const std::vector<VideoReceiveStream*>& receive_streams) {
92 send_stream_ = send_stream;
93}
94
eladalon413ee9a2017-08-22 04:02:52 -070095test::PacketTransport* RampUpTester::CreateSendTransport(
96 test::SingleThreadedTaskQueueForTesting* task_queue,
97 Call* sender_call) {
Artem Titov631cafa2018-08-21 21:01:00 +020098 auto network = absl::make_unique<SimulatedNetwork>(forward_transport_config_);
99 send_simulated_network_ = network.get();
minyue20c84cc2017-04-10 16:57:57 -0700100 send_transport_ = new test::PacketTransport(
eladalon413ee9a2017-08-22 04:02:52 -0700101 task_queue, sender_call, this, test::PacketTransport::kSender,
Artem Titov4e199e92018-08-20 13:30:39 +0200102 test::CallTest::payload_type_map_,
Artem Titov631cafa2018-08-21 21:01:00 +0200103 absl::make_unique<FakeNetworkPipe>(Clock::GetRealTimeClock(),
104 std::move(network)));
stefane74eef12016-01-08 06:47:13 -0800105 return send_transport_;
stefanf116bd02015-10-27 08:29:42 -0700106}
107
Stefan Holmerd20e6512016-01-12 15:51:22 +0100108size_t RampUpTester::GetNumVideoStreams() const {
109 return num_video_streams_;
110}
111
Stefan Holmerff2a6352016-01-14 10:00:21 +0100112size_t RampUpTester::GetNumAudioStreams() const {
113 return num_audio_streams_;
114}
115
philipel5ef2bc12017-02-21 07:28:31 -0800116size_t RampUpTester::GetNumFlexfecStreams() const {
117 return num_flexfec_streams_;
118}
119
perkjfa10b552016-10-02 23:45:26 -0700120class RampUpTester::VideoStreamFactory
121 : public VideoEncoderConfig::VideoStreamFactoryInterface {
122 public:
123 VideoStreamFactory() {}
124
125 private:
126 std::vector<VideoStream> CreateEncoderStreams(
127 int width,
128 int height,
129 const VideoEncoderConfig& encoder_config) override {
130 std::vector<VideoStream> streams =
131 test::CreateVideoStreams(width, height, encoder_config);
132 if (encoder_config.number_of_streams == 1) {
133 streams[0].target_bitrate_bps = streams[0].max_bitrate_bps = 2000000;
134 }
135 return streams;
136 }
137};
138
stefanff483612015-12-21 03:14:00 -0800139void RampUpTester::ModifyVideoConfigs(
stefan4fbd1452015-09-28 03:57:14 -0700140 VideoSendStream::Config* send_config,
141 std::vector<VideoReceiveStream::Config>* receive_configs,
142 VideoEncoderConfig* encoder_config) {
143 send_config->suspend_below_min_bitrate = true;
perkjfa10b552016-10-02 23:45:26 -0700144 encoder_config->number_of_streams = num_video_streams_;
145 encoder_config->max_bitrate_bps = 2000000;
146 encoder_config->video_stream_factory =
147 new rtc::RefCountedObject<RampUpTester::VideoStreamFactory>();
stefanff483612015-12-21 03:14:00 -0800148 if (num_video_streams_ == 1) {
stefan4fbd1452015-09-28 03:57:14 -0700149 // For single stream rampup until 1mbps
150 expected_bitrate_bps_ = kSingleStreamTargetBps;
151 } else {
Ilya Nikolaevskiyb0588e62018-08-27 14:12:27 +0200152 // To ensure simulcast rate allocation.
153 send_config->rtp.payload_name = "VP8";
154 encoder_config->codec_type = kVideoCodecVP8;
perkjfa10b552016-10-02 23:45:26 -0700155 std::vector<VideoStream> streams = test::CreateVideoStreams(
156 test::CallTest::kDefaultWidth, test::CallTest::kDefaultHeight,
157 *encoder_config);
Ilya Nikolaevskiyb0588e62018-08-27 14:12:27 +0200158 // For multi stream rampup until all streams are being sent. That means
159 // enough bitrate to send all the target streams plus the min bitrate of
160 // the last one.
perkjfa10b552016-10-02 23:45:26 -0700161 expected_bitrate_bps_ = streams.back().min_bitrate_bps;
162 for (size_t i = 0; i < streams.size() - 1; ++i) {
163 expected_bitrate_bps_ += streams[i].target_bitrate_bps;
stefan@webrtc.org7e9315b2013-12-04 10:24:26 +0000164 }
stefan@webrtc.org3d7da882014-07-08 13:59:46 +0000165 }
stefan@webrtc.org7e9315b2013-12-04 10:24:26 +0000166
stefan4fbd1452015-09-28 03:57:14 -0700167 send_config->rtp.extensions.clear();
168
169 bool remb;
stefan43edf0f2015-11-20 18:05:48 -0800170 bool transport_cc;
isheriff6f8d6862016-05-26 11:24:55 -0700171 if (extension_type_ == RtpExtension::kAbsSendTimeUri) {
stefan4fbd1452015-09-28 03:57:14 -0700172 remb = true;
stefan43edf0f2015-11-20 18:05:48 -0800173 transport_cc = false;
stefan4fbd1452015-09-28 03:57:14 -0700174 send_config->rtp.extensions.push_back(
175 RtpExtension(extension_type_.c_str(), kAbsSendTimeExtensionId));
isheriff6f8d6862016-05-26 11:24:55 -0700176 } else if (extension_type_ == RtpExtension::kTransportSequenceNumberUri) {
stefan4fbd1452015-09-28 03:57:14 -0700177 remb = false;
stefan43edf0f2015-11-20 18:05:48 -0800178 transport_cc = true;
stefan4fbd1452015-09-28 03:57:14 -0700179 send_config->rtp.extensions.push_back(RtpExtension(
180 extension_type_.c_str(), kTransportSequenceNumberExtensionId));
stefan@webrtc.org3d7da882014-07-08 13:59:46 +0000181 } else {
stefan4fbd1452015-09-28 03:57:14 -0700182 remb = true;
stefan43edf0f2015-11-20 18:05:48 -0800183 transport_cc = false;
stefan4fbd1452015-09-28 03:57:14 -0700184 send_config->rtp.extensions.push_back(RtpExtension(
185 extension_type_.c_str(), kTransmissionTimeOffsetExtensionId));
stefan@webrtc.org3d7da882014-07-08 13:59:46 +0000186 }
stefan4fbd1452015-09-28 03:57:14 -0700187
188 send_config->rtp.nack.rtp_history_ms = test::CallTest::kNackRtpHistoryMs;
stefanff483612015-12-21 03:14:00 -0800189 send_config->rtp.ssrcs = video_ssrcs_;
stefan4fbd1452015-09-28 03:57:14 -0700190 if (rtx_) {
191 send_config->rtp.rtx.payload_type = test::CallTest::kSendRtxPayloadType;
stefanff483612015-12-21 03:14:00 -0800192 send_config->rtp.rtx.ssrcs = video_rtx_ssrcs_;
stefan4fbd1452015-09-28 03:57:14 -0700193 }
194 if (red_) {
brandtrb5f2c3f2016-10-04 23:28:39 -0700195 send_config->rtp.ulpfec.ulpfec_payload_type =
stefan4fbd1452015-09-28 03:57:14 -0700196 test::CallTest::kUlpfecPayloadType;
brandtrb5f2c3f2016-10-04 23:28:39 -0700197 send_config->rtp.ulpfec.red_payload_type = test::CallTest::kRedPayloadType;
brandtrfbfb5362016-11-17 04:18:37 -0800198 if (rtx_) {
199 send_config->rtp.ulpfec.red_rtx_payload_type =
200 test::CallTest::kRtxRedPayloadType;
201 }
stefan4fbd1452015-09-28 03:57:14 -0700202 }
203
204 size_t i = 0;
205 for (VideoReceiveStream::Config& recv_config : *receive_configs) {
206 recv_config.rtp.remb = remb;
stefan43edf0f2015-11-20 18:05:48 -0800207 recv_config.rtp.transport_cc = transport_cc;
stefan4fbd1452015-09-28 03:57:14 -0700208 recv_config.rtp.extensions = send_config->rtp.extensions;
Ilya Nikolaevskiyb0588e62018-08-27 14:12:27 +0200209 recv_config.decoders.reserve(1);
210 recv_config.decoders[0].payload_type = send_config->rtp.payload_type;
211 recv_config.decoders[0].payload_name = send_config->rtp.payload_name;
stefan4fbd1452015-09-28 03:57:14 -0700212
stefanff483612015-12-21 03:14:00 -0800213 recv_config.rtp.remote_ssrc = video_ssrcs_[i];
stefan4fbd1452015-09-28 03:57:14 -0700214 recv_config.rtp.nack.rtp_history_ms = send_config->rtp.nack.rtp_history_ms;
215
216 if (red_) {
nisse3b3622f2017-09-26 02:49:21 -0700217 recv_config.rtp.red_payload_type =
brandtrb5f2c3f2016-10-04 23:28:39 -0700218 send_config->rtp.ulpfec.red_payload_type;
nisse3b3622f2017-09-26 02:49:21 -0700219 recv_config.rtp.ulpfec_payload_type =
brandtrb5f2c3f2016-10-04 23:28:39 -0700220 send_config->rtp.ulpfec.ulpfec_payload_type;
brandtrfbfb5362016-11-17 04:18:37 -0800221 if (rtx_) {
nisseca5706d2017-09-11 02:32:16 -0700222 recv_config.rtp.rtx_associated_payload_types
223 [send_config->rtp.ulpfec.red_rtx_payload_type] =
224 send_config->rtp.ulpfec.red_payload_type;
brandtrfbfb5362016-11-17 04:18:37 -0800225 }
stefan4fbd1452015-09-28 03:57:14 -0700226 }
227
228 if (rtx_) {
brandtr14742122017-01-27 04:53:07 -0800229 recv_config.rtp.rtx_ssrc = video_rtx_ssrcs_[i];
230 recv_config.rtp
nisse26e3abb2017-08-25 04:44:25 -0700231 .rtx_associated_payload_types[send_config->rtp.rtx.payload_type] =
Niels Möller259a4972018-04-05 15:36:51 +0200232 send_config->rtp.payload_type;
stefan4fbd1452015-09-28 03:57:14 -0700233 }
234 ++i;
235 }
philipel5ef2bc12017-02-21 07:28:31 -0800236
237 RTC_DCHECK_LE(num_flexfec_streams_, 1);
238 if (num_flexfec_streams_ == 1) {
239 send_config->rtp.flexfec.payload_type = test::CallTest::kFlexfecPayloadType;
240 send_config->rtp.flexfec.ssrc = test::CallTest::kFlexfecSendSsrc;
241 send_config->rtp.flexfec.protected_media_ssrcs = {video_ssrcs_[0]};
242 }
stefan@webrtc.org3d7da882014-07-08 13:59:46 +0000243}
244
Stefan Holmerff2a6352016-01-14 10:00:21 +0100245void RampUpTester::ModifyAudioConfigs(
246 AudioSendStream::Config* send_config,
247 std::vector<AudioReceiveStream::Config>* receive_configs) {
248 if (num_audio_streams_ == 0)
249 return;
250
isheriff6f8d6862016-05-26 11:24:55 -0700251 EXPECT_NE(RtpExtension::kTimestampOffsetUri, extension_type_)
Stefan Holmerff2a6352016-01-14 10:00:21 +0100252 << "Audio BWE not supported with toffset.";
philipel5ef2bc12017-02-21 07:28:31 -0800253 EXPECT_NE(RtpExtension::kAbsSendTimeUri, extension_type_)
254 << "Audio BWE not supported with abs-send-time.";
Stefan Holmerff2a6352016-01-14 10:00:21 +0100255
256 send_config->rtp.ssrc = audio_ssrcs_[0];
257 send_config->rtp.extensions.clear();
258
minyue10cbb462016-11-07 09:29:22 -0800259 send_config->min_bitrate_bps = 6000;
260 send_config->max_bitrate_bps = 60000;
mflodman86cc6ff2016-07-26 04:44:06 -0700261
Stefan Holmerff2a6352016-01-14 10:00:21 +0100262 bool transport_cc = false;
philipel5ef2bc12017-02-21 07:28:31 -0800263 if (extension_type_ == RtpExtension::kTransportSequenceNumberUri) {
Stefan Holmerff2a6352016-01-14 10:00:21 +0100264 transport_cc = true;
265 send_config->rtp.extensions.push_back(RtpExtension(
266 extension_type_.c_str(), kTransportSequenceNumberExtensionId));
267 }
268
269 for (AudioReceiveStream::Config& recv_config : *receive_configs) {
Stefan Holmerff2a6352016-01-14 10:00:21 +0100270 recv_config.rtp.transport_cc = transport_cc;
271 recv_config.rtp.extensions = send_config->rtp.extensions;
272 recv_config.rtp.remote_ssrc = send_config->rtp.ssrc;
273 }
274}
275
philipel5ef2bc12017-02-21 07:28:31 -0800276void RampUpTester::ModifyFlexfecConfigs(
277 std::vector<FlexfecReceiveStream::Config>* receive_configs) {
278 if (num_flexfec_streams_ == 0)
279 return;
280 RTC_DCHECK_EQ(1, num_flexfec_streams_);
281 (*receive_configs)[0].payload_type = test::CallTest::kFlexfecPayloadType;
282 (*receive_configs)[0].remote_ssrc = test::CallTest::kFlexfecSendSsrc;
283 (*receive_configs)[0].protected_media_ssrcs = {video_ssrcs_[0]};
284 (*receive_configs)[0].local_ssrc = video_ssrcs_[0];
285 if (extension_type_ == RtpExtension::kAbsSendTimeUri) {
286 (*receive_configs)[0].transport_cc = false;
287 (*receive_configs)[0].rtp_header_extensions.push_back(
288 RtpExtension(extension_type_.c_str(), kAbsSendTimeExtensionId));
289 } else if (extension_type_ == RtpExtension::kTransportSequenceNumberUri) {
290 (*receive_configs)[0].transport_cc = true;
291 (*receive_configs)[0].rtp_header_extensions.push_back(RtpExtension(
292 extension_type_.c_str(), kTransportSequenceNumberExtensionId));
293 }
294}
295
stefan4fbd1452015-09-28 03:57:14 -0700296void RampUpTester::OnCallsCreated(Call* sender_call, Call* receiver_call) {
297 sender_call_ = sender_call;
stefan@webrtc.org3d7da882014-07-08 13:59:46 +0000298}
299
tommi0f8b4032017-02-22 11:22:05 -0800300void RampUpTester::BitrateStatsPollingThread(void* obj) {
301 static_cast<RampUpTester*>(obj)->PollStats();
pbos@webrtc.org32452b22014-10-22 12:15:24 +0000302}
stefan@webrtc.org3d7da882014-07-08 13:59:46 +0000303
tommi0f8b4032017-02-22 11:22:05 -0800304void RampUpTester::PollStats() {
305 do {
306 if (sender_call_) {
307 Call::Stats stats = sender_call_->GetStats();
stefan4fbd1452015-09-28 03:57:14 -0700308
tommi0f8b4032017-02-22 11:22:05 -0800309 EXPECT_GE(expected_bitrate_bps_, 0);
310 if (stats.send_bandwidth_bps >= expected_bitrate_bps_ &&
311 (min_run_time_ms_ == -1 ||
312 clock_->TimeInMilliseconds() - test_start_ms_ >= min_run_time_ms_)) {
313 ramp_up_finished_ms_ = clock_->TimeInMilliseconds();
314 observation_complete_.Set();
315 }
stefan4fbd1452015-09-28 03:57:14 -0700316 }
tommi0f8b4032017-02-22 11:22:05 -0800317 } while (!stop_event_.Wait(kPollIntervalMs));
Erik Språng468e62a2015-07-06 10:50:47 +0200318}
319
stefan4fbd1452015-09-28 03:57:14 -0700320void RampUpTester::ReportResult(const std::string& measurement,
321 size_t value,
322 const std::string& units) const {
stefan@webrtc.org3d7da882014-07-08 13:59:46 +0000323 webrtc::test::PrintResult(
324 measurement, "",
stefanff483612015-12-21 03:14:00 -0800325 ::testing::UnitTest::GetInstance()->current_test_info()->name(), value,
326 units, false);
stefan@webrtc.org3d7da882014-07-08 13:59:46 +0000327}
328
stefan092508a2015-09-29 02:26:42 -0700329void RampUpTester::AccumulateStats(const VideoSendStream::StreamStats& stream,
330 size_t* total_packets_sent,
331 size_t* total_sent,
332 size_t* padding_sent,
333 size_t* media_sent) const {
stefan4fbd1452015-09-28 03:57:14 -0700334 *total_packets_sent += stream.rtp_stats.transmitted.packets +
335 stream.rtp_stats.retransmitted.packets +
336 stream.rtp_stats.fec.packets;
337 *total_sent += stream.rtp_stats.transmitted.TotalBytes() +
338 stream.rtp_stats.retransmitted.TotalBytes() +
339 stream.rtp_stats.fec.TotalBytes();
340 *padding_sent += stream.rtp_stats.transmitted.padding_bytes +
341 stream.rtp_stats.retransmitted.padding_bytes +
342 stream.rtp_stats.fec.padding_bytes;
343 *media_sent += stream.rtp_stats.MediaPayloadBytes();
stefan@webrtc.org3d7da882014-07-08 13:59:46 +0000344}
345
stefan4fbd1452015-09-28 03:57:14 -0700346void RampUpTester::TriggerTestDone() {
Erik Språngf3a7c9d2015-10-05 14:03:22 +0200347 RTC_DCHECK_GE(test_start_ms_, 0);
348
Stefan Holmerff2a6352016-01-14 10:00:21 +0100349 // TODO(holmer): Add audio send stats here too when those APIs are available.
mflodman86cc6ff2016-07-26 04:44:06 -0700350 if (!send_stream_)
351 return;
352
stefan4fbd1452015-09-28 03:57:14 -0700353 VideoSendStream::Stats send_stats = send_stream_->GetStats();
354
355 size_t total_packets_sent = 0;
356 size_t total_sent = 0;
357 size_t padding_sent = 0;
358 size_t media_sent = 0;
stefanff483612015-12-21 03:14:00 -0800359 for (uint32_t ssrc : video_ssrcs_) {
stefan092508a2015-09-29 02:26:42 -0700360 AccumulateStats(send_stats.substreams[ssrc], &total_packets_sent,
361 &total_sent, &padding_sent, &media_sent);
stefan4fbd1452015-09-28 03:57:14 -0700362 }
363
364 size_t rtx_total_packets_sent = 0;
365 size_t rtx_total_sent = 0;
366 size_t rtx_padding_sent = 0;
367 size_t rtx_media_sent = 0;
stefanff483612015-12-21 03:14:00 -0800368 for (uint32_t rtx_ssrc : video_rtx_ssrcs_) {
stefan092508a2015-09-29 02:26:42 -0700369 AccumulateStats(send_stats.substreams[rtx_ssrc], &rtx_total_packets_sent,
370 &rtx_total_sent, &rtx_padding_sent, &rtx_media_sent);
stefan4fbd1452015-09-28 03:57:14 -0700371 }
372
stefan5a2c5062017-01-27 06:43:18 -0800373 if (report_perf_stats_) {
stefan5a2c5062017-01-27 06:43:18 -0800374 ReportResult("ramp-up-media-sent", media_sent, "bytes");
375 ReportResult("ramp-up-padding-sent", padding_sent, "bytes");
stefan5a2c5062017-01-27 06:43:18 -0800376 ReportResult("ramp-up-rtx-media-sent", rtx_media_sent, "bytes");
377 ReportResult("ramp-up-rtx-padding-sent", rtx_padding_sent, "bytes");
378 if (ramp_up_finished_ms_ >= 0) {
379 ReportResult("ramp-up-time", ramp_up_finished_ms_ - test_start_ms_,
380 "milliseconds");
381 }
382 ReportResult("ramp-up-average-network-latency",
383 send_transport_->GetAverageDelayMs(), "milliseconds");
stefan4fbd1452015-09-28 03:57:14 -0700384 }
385}
386
387void RampUpTester::PerformTest() {
Erik Språngf3a7c9d2015-10-05 14:03:22 +0200388 test_start_ms_ = clock_->TimeInMilliseconds();
Peter Boström8c38e8b2015-11-26 17:45:47 +0100389 poller_thread_.Start();
Peter Boström5811a392015-12-10 13:02:50 +0100390 EXPECT_TRUE(Wait()) << "Timed out while waiting for ramp-up to complete.";
stefan4fbd1452015-09-28 03:57:14 -0700391 TriggerTestDone();
tommi0f8b4032017-02-22 11:22:05 -0800392 stop_event_.Set();
Peter Boström8c38e8b2015-11-26 17:45:47 +0100393 poller_thread_.Stop();
stefan4fbd1452015-09-28 03:57:14 -0700394}
395
Stefan Holmerff2a6352016-01-14 10:00:21 +0100396RampUpDownUpTester::RampUpDownUpTester(size_t num_video_streams,
397 size_t num_audio_streams,
philipel5ef2bc12017-02-21 07:28:31 -0800398 size_t num_flexfec_streams,
stefan4fbd1452015-09-28 03:57:14 -0700399 unsigned int start_bitrate_bps,
400 const std::string& extension_type,
401 bool rtx,
philipel5ef2bc12017-02-21 07:28:31 -0800402 bool red,
stefan45b5fe52017-03-09 06:27:02 -0800403 const std::vector<int>& loss_rates,
404 bool report_perf_stats)
Stefan Holmerff2a6352016-01-14 10:00:21 +0100405 : RampUpTester(num_video_streams,
406 num_audio_streams,
philipel5ef2bc12017-02-21 07:28:31 -0800407 num_flexfec_streams,
Stefan Holmerff2a6352016-01-14 10:00:21 +0100408 start_bitrate_bps,
stefan5a2c5062017-01-27 06:43:18 -0800409 0,
Stefan Holmerff2a6352016-01-14 10:00:21 +0100410 extension_type,
411 rtx,
stefan5a2c5062017-01-27 06:43:18 -0800412 red,
stefan45b5fe52017-03-09 06:27:02 -0800413 report_perf_stats),
stefan76d9c9c2017-04-01 06:51:09 -0700414 link_rates_({4 * GetExpectedHighBitrate() / (3 * 1000),
415 kLowBandwidthLimitBps / 1000,
416 4 * GetExpectedHighBitrate() / (3 * 1000), 0}),
stefan@webrtc.org3d7da882014-07-08 13:59:46 +0000417 test_state_(kFirstRampup),
philipel5ef2bc12017-02-21 07:28:31 -0800418 next_state_(kTransitionToNextState),
stefan@webrtc.org3d7da882014-07-08 13:59:46 +0000419 state_start_ms_(clock_->TimeInMilliseconds()),
stefan4fbd1452015-09-28 03:57:14 -0700420 interval_start_ms_(clock_->TimeInMilliseconds()),
philipel5ef2bc12017-02-21 07:28:31 -0800421 sent_bytes_(0),
422 loss_rates_(loss_rates) {
423 forward_transport_config_.link_capacity_kbps = link_rates_[test_state_];
424 forward_transport_config_.queue_delay_ms = 100;
425 forward_transport_config_.loss_percent = loss_rates_[test_state_];
stefan@webrtc.org3d7da882014-07-08 13:59:46 +0000426}
427
stefan4fbd1452015-09-28 03:57:14 -0700428RampUpDownUpTester::~RampUpDownUpTester() {}
stefan@webrtc.org3d7da882014-07-08 13:59:46 +0000429
tommi0f8b4032017-02-22 11:22:05 -0800430void RampUpDownUpTester::PollStats() {
431 do {
Alex Narest78609d52017-10-20 10:37:47 +0200432 int transmit_bitrate_bps = 0;
433 bool suspended = false;
434 if (num_video_streams_ > 0) {
tommi0f8b4032017-02-22 11:22:05 -0800435 webrtc::VideoSendStream::Stats stats = send_stream_->GetStats();
tommi0f8b4032017-02-22 11:22:05 -0800436 for (auto it : stats.substreams) {
437 transmit_bitrate_bps += it.second.total_bitrate_bps;
438 }
Alex Narest78609d52017-10-20 10:37:47 +0200439 suspended = stats.suspended;
440 }
441 if (num_audio_streams_ > 0 && sender_call_ != nullptr) {
tommi0f8b4032017-02-22 11:22:05 -0800442 // An audio send stream doesn't have bitrate stats, so the call send BW is
443 // currently used instead.
Alex Narest78609d52017-10-20 10:37:47 +0200444 transmit_bitrate_bps = sender_call_->GetStats().send_bandwidth_bps;
mflodman@webrtc.orgeb16b812014-06-16 08:57:39 +0000445 }
Alex Narest78609d52017-10-20 10:37:47 +0200446 EvolveTestState(transmit_bitrate_bps, suspended);
tommi0f8b4032017-02-22 11:22:05 -0800447 } while (!stop_event_.Wait(kPollIntervalMs));
stefan@webrtc.org3d7da882014-07-08 13:59:46 +0000448}
stefan@webrtc.org7e9315b2013-12-04 10:24:26 +0000449
Sebastian Jansson72582242018-07-13 13:19:42 +0200450void RampUpDownUpTester::ModifyReceiverCallConfig(Call::Config* config) {
451 config->bitrate_config.min_bitrate_bps = 10000;
stefan@webrtc.org3d7da882014-07-08 13:59:46 +0000452}
stefan@webrtc.org7e9315b2013-12-04 10:24:26 +0000453
stefan4fbd1452015-09-28 03:57:14 -0700454std::string RampUpDownUpTester::GetModifierString() const {
stefan@webrtc.org3d7da882014-07-08 13:59:46 +0000455 std::string str("_");
stefanff483612015-12-21 03:14:00 -0800456 if (num_video_streams_ > 0) {
Jonas Olsson0a713b62018-04-04 15:49:32 +0200457 str += rtc::ToString(num_video_streams_);
stefanff483612015-12-21 03:14:00 -0800458 str += "stream";
459 str += (num_video_streams_ > 1 ? "s" : "");
460 str += "_";
461 }
462 if (num_audio_streams_ > 0) {
Jonas Olsson0a713b62018-04-04 15:49:32 +0200463 str += rtc::ToString(num_audio_streams_);
stefanff483612015-12-21 03:14:00 -0800464 str += "stream";
465 str += (num_audio_streams_ > 1 ? "s" : "");
466 str += "_";
467 }
stefan4fbd1452015-09-28 03:57:14 -0700468 str += (rtx_ ? "" : "no");
Edward Lemurdd3987f2018-01-04 19:44:54 +0100469 str += "rtx_";
470 str += (red_ ? "" : "no");
471 str += "red";
stefan@webrtc.org3d7da882014-07-08 13:59:46 +0000472 return str;
473}
andresp@webrtc.orgc1480792014-03-20 03:23:55 +0000474
stefandb752f92016-12-05 08:23:40 -0800475int RampUpDownUpTester::GetExpectedHighBitrate() const {
stefan38d8b3c2017-01-09 04:19:24 -0800476 int expected_bitrate_bps = 0;
477 if (num_audio_streams_ > 0)
478 expected_bitrate_bps += kExpectedHighAudioBitrateBps;
479 if (num_video_streams_ > 0)
480 expected_bitrate_bps += kExpectedHighVideoBitrateBps;
481 return expected_bitrate_bps;
482}
483
philipel5ef2bc12017-02-21 07:28:31 -0800484size_t RampUpDownUpTester::GetFecBytes() const {
485 size_t flex_fec_bytes = 0;
486 if (num_flexfec_streams_ > 0) {
487 webrtc::VideoSendStream::Stats stats = send_stream_->GetStats();
488 for (const auto& kv : stats.substreams)
489 flex_fec_bytes += kv.second.rtp_stats.fec.TotalBytes();
490 }
491 return flex_fec_bytes;
492}
493
494bool RampUpDownUpTester::ExpectingFec() const {
495 return num_flexfec_streams_ > 0 && forward_transport_config_.loss_percent > 0;
496}
497
stefan4fbd1452015-09-28 03:57:14 -0700498void RampUpDownUpTester::EvolveTestState(int bitrate_bps, bool suspended) {
stefan@webrtc.org3d7da882014-07-08 13:59:46 +0000499 int64_t now = clock_->TimeInMilliseconds();
stefan@webrtc.org3d7da882014-07-08 13:59:46 +0000500 switch (test_state_) {
philipel5ef2bc12017-02-21 07:28:31 -0800501 case kFirstRampup:
stefan4fbd1452015-09-28 03:57:14 -0700502 EXPECT_FALSE(suspended);
stefandb752f92016-12-05 08:23:40 -0800503 if (bitrate_bps >= GetExpectedHighBitrate()) {
stefan45b5fe52017-03-09 06:27:02 -0800504 if (report_perf_stats_) {
505 webrtc::test::PrintResult("ramp_up_down_up", GetModifierString(),
506 "first_rampup", now - state_start_ms_, "ms",
507 false);
508 }
philipel5ef2bc12017-02-21 07:28:31 -0800509 // Apply loss during the transition between states if FEC is enabled.
510 forward_transport_config_.loss_percent = loss_rates_[test_state_];
511 test_state_ = kTransitionToNextState;
512 next_state_ = kLowRate;
andresp@webrtc.orgc1480792014-03-20 03:23:55 +0000513 }
stefan@webrtc.org3d7da882014-07-08 13:59:46 +0000514 break;
stefan@webrtc.org3d7da882014-07-08 13:59:46 +0000515 case kLowRate: {
mflodman86cc6ff2016-07-26 04:44:06 -0700516 // Audio streams are never suspended.
517 bool check_suspend_state = num_video_streams_ > 0;
Sebastian Jansson12fb1702018-02-23 11:34:18 +0100518 if (bitrate_bps < kLowBandwidthLimitBps + kLowBitrateMarginBps &&
mflodman86cc6ff2016-07-26 04:44:06 -0700519 suspended == check_suspend_state) {
stefan45b5fe52017-03-09 06:27:02 -0800520 if (report_perf_stats_) {
521 webrtc::test::PrintResult("ramp_up_down_up", GetModifierString(),
522 "rampdown", now - state_start_ms_, "ms",
523 false);
524 }
philipel5ef2bc12017-02-21 07:28:31 -0800525 // Apply loss during the transition between states if FEC is enabled.
526 forward_transport_config_.loss_percent = loss_rates_[test_state_];
527 test_state_ = kTransitionToNextState;
528 next_state_ = kSecondRampup;
stefan@webrtc.org3d7da882014-07-08 13:59:46 +0000529 }
530 break;
531 }
philipel5ef2bc12017-02-21 07:28:31 -0800532 case kSecondRampup:
stefandb752f92016-12-05 08:23:40 -0800533 if (bitrate_bps >= GetExpectedHighBitrate() && !suspended) {
stefan45b5fe52017-03-09 06:27:02 -0800534 if (report_perf_stats_) {
535 webrtc::test::PrintResult("ramp_up_down_up", GetModifierString(),
536 "second_rampup", now - state_start_ms_,
537 "ms", false);
538 ReportResult("ramp-up-down-up-average-network-latency",
539 send_transport_->GetAverageDelayMs(), "milliseconds");
540 }
philipel5ef2bc12017-02-21 07:28:31 -0800541 // Apply loss during the transition between states if FEC is enabled.
542 forward_transport_config_.loss_percent = loss_rates_[test_state_];
543 test_state_ = kTransitionToNextState;
544 next_state_ = kTestEnd;
stefan@webrtc.org3d7da882014-07-08 13:59:46 +0000545 }
546 break;
philipel5ef2bc12017-02-21 07:28:31 -0800547 case kTestEnd:
548 observation_complete_.Set();
549 break;
550 case kTransitionToNextState:
551 if (!ExpectingFec() || GetFecBytes() > 0) {
552 test_state_ = next_state_;
553 forward_transport_config_.link_capacity_kbps = link_rates_[test_state_];
554 // No loss while ramping up and down as it may affect the BWE
555 // negatively, making the test flaky.
556 forward_transport_config_.loss_percent = 0;
557 state_start_ms_ = now;
558 interval_start_ms_ = now;
559 sent_bytes_ = 0;
Artem Titov631cafa2018-08-21 21:01:00 +0200560 send_simulated_network_->SetConfig(forward_transport_config_);
philipel5ef2bc12017-02-21 07:28:31 -0800561 }
562 break;
stefan@webrtc.org3d7da882014-07-08 13:59:46 +0000563 }
564}
pbos@webrtc.orgf577ae92014-03-19 08:43:57 +0000565
stefan4fbd1452015-09-28 03:57:14 -0700566class RampUpTest : public test::CallTest {
Erik Språng6b8d3552015-09-24 15:06:57 +0200567 public:
stefan4fbd1452015-09-28 03:57:14 -0700568 RampUpTest() {}
Erik Språng6b8d3552015-09-24 15:06:57 +0200569};
570
Stefan Holmerff2a6352016-01-14 10:00:21 +0100571static const uint32_t kStartBitrateBps = 60000;
572
stefan38d8b3c2017-01-09 04:19:24 -0800573TEST_F(RampUpTest, UpDownUpAbsSendTimeSimulcastRedRtx) {
philipel5ef2bc12017-02-21 07:28:31 -0800574 std::vector<int> loss_rates = {0, 0, 0, 0};
575 RampUpDownUpTester test(3, 0, 0, kStartBitrateBps,
stefan45b5fe52017-03-09 06:27:02 -0800576 RtpExtension::kAbsSendTimeUri, true, true, loss_rates,
577 true);
stefane74eef12016-01-08 06:47:13 -0800578 RunBaseTest(&test);
Shao Changbine62202f2015-04-21 20:24:50 +0800579}
Stefan Holmerff2a6352016-01-14 10:00:21 +0100580
Taylor Brandstetterba6b5032018-02-16 09:59:56 -0800581// TODO(bugs.webrtc.org/8878)
582#if defined(WEBRTC_MAC)
583#define MAYBE_UpDownUpTransportSequenceNumberRtx \
584 DISABLED_UpDownUpTransportSequenceNumberRtx
585#else
586#define MAYBE_UpDownUpTransportSequenceNumberRtx \
587 UpDownUpTransportSequenceNumberRtx
588#endif
589TEST_F(RampUpTest, MAYBE_UpDownUpTransportSequenceNumberRtx) {
philipel5ef2bc12017-02-21 07:28:31 -0800590 std::vector<int> loss_rates = {0, 0, 0, 0};
591 RampUpDownUpTester test(3, 0, 0, kStartBitrateBps,
isheriff6f8d6862016-05-26 11:24:55 -0700592 RtpExtension::kTransportSequenceNumberUri, true,
stefan45b5fe52017-03-09 06:27:02 -0800593 false, loss_rates, true);
philipel5ef2bc12017-02-21 07:28:31 -0800594 RunBaseTest(&test);
595}
596
stefan45b5fe52017-03-09 06:27:02 -0800597// TODO(holmer): Tests which don't report perf stats should be moved to a
598// different executable since they per definition are not perf tests.
Taylor Brandstetter05d98222017-10-10 12:12:53 -0700599// This test is disabled because it crashes on Linux, and is flaky on other
600// platforms. See: crbug.com/webrtc/7919
601TEST_F(RampUpTest, DISABLED_UpDownUpTransportSequenceNumberPacketLoss) {
philipel5ef2bc12017-02-21 07:28:31 -0800602 std::vector<int> loss_rates = {20, 0, 0, 0};
603 RampUpDownUpTester test(1, 0, 1, kStartBitrateBps,
604 RtpExtension::kTransportSequenceNumberUri, true,
stefan45b5fe52017-03-09 06:27:02 -0800605 false, loss_rates, false);
Stefan Holmerff2a6352016-01-14 10:00:21 +0100606 RunBaseTest(&test);
607}
608
Taylor Brandstetterab86e7f2018-02-12 15:16:02 -0800609// TODO(bugs.webrtc.org/8878)
610#if defined(WEBRTC_MAC)
611#define MAYBE_UpDownUpAudioVideoTransportSequenceNumberRtx \
612 DISABLED_UpDownUpAudioVideoTransportSequenceNumberRtx
613#else
614#define MAYBE_UpDownUpAudioVideoTransportSequenceNumberRtx \
615 UpDownUpAudioVideoTransportSequenceNumberRtx
616#endif
617TEST_F(RampUpTest, MAYBE_UpDownUpAudioVideoTransportSequenceNumberRtx) {
philipel5ef2bc12017-02-21 07:28:31 -0800618 std::vector<int> loss_rates = {0, 0, 0, 0};
619 RampUpDownUpTester test(3, 1, 0, kStartBitrateBps,
isheriff6f8d6862016-05-26 11:24:55 -0700620 RtpExtension::kTransportSequenceNumberUri, true,
stefanff2ebf52017-03-13 01:27:03 -0700621 false, loss_rates, false);
Stefan Holmerff2a6352016-01-14 10:00:21 +0100622 RunBaseTest(&test);
623}
624
stefan38d8b3c2017-01-09 04:19:24 -0800625TEST_F(RampUpTest, UpDownUpAudioTransportSequenceNumberRtx) {
philipel5ef2bc12017-02-21 07:28:31 -0800626 std::vector<int> loss_rates = {0, 0, 0, 0};
627 RampUpDownUpTester test(0, 1, 0, kStartBitrateBps,
mflodman86cc6ff2016-07-26 04:44:06 -0700628 RtpExtension::kTransportSequenceNumberUri, true,
stefanff2ebf52017-03-13 01:27:03 -0700629 false, loss_rates, false);
mflodman86cc6ff2016-07-26 04:44:06 -0700630 RunBaseTest(&test);
631}
632
stefan38d8b3c2017-01-09 04:19:24 -0800633TEST_F(RampUpTest, TOffsetSimulcastRedRtx) {
philipel5ef2bc12017-02-21 07:28:31 -0800634 RampUpTester test(3, 0, 0, 0, 0, RtpExtension::kTimestampOffsetUri, true,
635 true, true);
stefan38d8b3c2017-01-09 04:19:24 -0800636 RunBaseTest(&test);
637}
638
639TEST_F(RampUpTest, AbsSendTime) {
philipel5ef2bc12017-02-21 07:28:31 -0800640 RampUpTester test(1, 0, 0, 0, 0, RtpExtension::kAbsSendTimeUri, false, false,
stefanff2ebf52017-03-13 01:27:03 -0700641 false);
stefane74eef12016-01-08 06:47:13 -0800642 RunBaseTest(&test);
pbos@webrtc.org85bd53e2014-12-10 10:36:20 +0000643}
644
stefan38d8b3c2017-01-09 04:19:24 -0800645TEST_F(RampUpTest, AbsSendTimeSimulcastRedRtx) {
philipel5ef2bc12017-02-21 07:28:31 -0800646 RampUpTester test(3, 0, 0, 0, 0, RtpExtension::kAbsSendTimeUri, true, true,
stefan5a2c5062017-01-27 06:43:18 -0800647 true);
stefane74eef12016-01-08 06:47:13 -0800648 RunBaseTest(&test);
pbos@webrtc.org85bd53e2014-12-10 10:36:20 +0000649}
650
stefan38d8b3c2017-01-09 04:19:24 -0800651TEST_F(RampUpTest, TransportSequenceNumber) {
philipel5ef2bc12017-02-21 07:28:31 -0800652 RampUpTester test(1, 0, 0, 0, 0, RtpExtension::kTransportSequenceNumberUri,
stefanff2ebf52017-03-13 01:27:03 -0700653 false, false, false);
stefane74eef12016-01-08 06:47:13 -0800654 RunBaseTest(&test);
Erik Språng6b8d3552015-09-24 15:06:57 +0200655}
656
657TEST_F(RampUpTest, TransportSequenceNumberSimulcast) {
philipel5ef2bc12017-02-21 07:28:31 -0800658 RampUpTester test(3, 0, 0, 0, 0, RtpExtension::kTransportSequenceNumberUri,
stefanff2ebf52017-03-13 01:27:03 -0700659 false, false, false);
stefane74eef12016-01-08 06:47:13 -0800660 RunBaseTest(&test);
Erik Språng6b8d3552015-09-24 15:06:57 +0200661}
662
stefan38d8b3c2017-01-09 04:19:24 -0800663TEST_F(RampUpTest, TransportSequenceNumberSimulcastRedRtx) {
philipel5ef2bc12017-02-21 07:28:31 -0800664 RampUpTester test(3, 0, 0, 0, 0, RtpExtension::kTransportSequenceNumberUri,
stefan5a2c5062017-01-27 06:43:18 -0800665 true, true, true);
666 RunBaseTest(&test);
667}
668
Taylor Brandstetterab86e7f2018-02-12 15:16:02 -0800669// TODO(bugs.webrtc.org/8878)
670#if defined(WEBRTC_MAC)
671#define MAYBE_AudioTransportSequenceNumber DISABLED_AudioTransportSequenceNumber
672#else
673#define MAYBE_AudioTransportSequenceNumber AudioTransportSequenceNumber
674#endif
675TEST_F(RampUpTest, MAYBE_AudioTransportSequenceNumber) {
philipel5ef2bc12017-02-21 07:28:31 -0800676 RampUpTester test(0, 1, 0, 300000, 10000,
stefan5a2c5062017-01-27 06:43:18 -0800677 RtpExtension::kTransportSequenceNumberUri, false, false,
678 false);
stefane74eef12016-01-08 06:47:13 -0800679 RunBaseTest(&test);
Erik Språng6b8d3552015-09-24 15:06:57 +0200680}
pbos@webrtc.org744fbc72013-09-10 09:26:25 +0000681} // namespace webrtc