- Removes voe_conference_test.
- Adds a new AudioStatsTest, with better coverage of the same features, based on call_test.
- Adds an AudioEndToEndTest utility, which AudioStatsTest and LowBandwidthAudioTest uses.
BUG=webrtc:4690
R=kwiberg@webrtc.org
Review-Url: https://codereview.webrtc.org/3008273002 .
Cr-Commit-Position: refs/heads/master@{#19833}
diff --git a/webrtc/audio/BUILD.gn b/webrtc/audio/BUILD.gn
index 42b74ff..890de51 100644
--- a/webrtc/audio/BUILD.gn
+++ b/webrtc/audio/BUILD.gn
@@ -58,6 +58,27 @@
]
}
if (rtc_include_tests) {
+ rtc_source_set("audio_end_to_end_test") {
+ testonly = true
+
+ sources = [
+ "test/audio_end_to_end_test.cc",
+ "test/audio_end_to_end_test.h",
+ ]
+ deps = [
+ ":audio",
+ "../system_wrappers:system_wrappers",
+ "../test:fake_audio_device",
+ "../test:test_common",
+ "../test:test_support",
+ ]
+
+ if (!build_with_chromium && is_clang) {
+ # Suppress warnings from the Chromium Clang plugin (bugs.webrtc.org/163).
+ suppressed_configs += [ "//build/config/clang:find_bad_constructs" ]
+ }
+ }
+
rtc_source_set("audio_tests") {
testonly = true
@@ -80,6 +101,7 @@
]
deps = [
":audio",
+ ":audio_end_to_end_test",
"../api:mock_audio_mixer",
"../call:rtp_receiver",
"../modules/audio_device:mock_audio_device",
@@ -96,6 +118,11 @@
"//testing/gtest",
]
+ if (!rtc_use_memcheck) {
+ # This test is timing dependent, which rules out running on memcheck bots.
+ sources += [ "test/audio_stats_test.cc" ]
+ }
+
if (!build_with_chromium && is_clang) {
# Suppress warnings from the Chromium Clang plugin (bugs.webrtc.org/163).
suppressed_configs += [ "//build/config/clang:find_bad_constructs" ]
@@ -108,10 +135,10 @@
sources = [
"test/low_bandwidth_audio_test.cc",
- "test/low_bandwidth_audio_test.h",
]
deps = [
+ ":audio_end_to_end_test",
"../common_audio",
"../rtc_base:rtc_base_approved",
"../system_wrappers",
diff --git a/webrtc/audio/test/audio_end_to_end_test.cc b/webrtc/audio/test/audio_end_to_end_test.cc
new file mode 100644
index 0000000..5d4cbf0
--- /dev/null
+++ b/webrtc/audio/test/audio_end_to_end_test.cc
@@ -0,0 +1,105 @@
+/*
+ * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include <algorithm>
+
+#include "webrtc/audio/test/audio_end_to_end_test.h"
+#include "webrtc/system_wrappers/include/sleep.h"
+#include "webrtc/test/fake_audio_device.h"
+#include "webrtc/test/gtest.h"
+
+namespace webrtc {
+namespace test {
+namespace {
+// Wait half a second between stopping sending and stopping receiving audio.
+constexpr int kExtraRecordTimeMs = 500;
+
+constexpr int kSampleRate = 48000;
+} // namespace
+
+AudioEndToEndTest::AudioEndToEndTest()
+ : EndToEndTest(CallTest::kDefaultTimeoutMs) {}
+
+FakeNetworkPipe::Config AudioEndToEndTest::GetNetworkPipeConfig() const {
+ return FakeNetworkPipe::Config();
+}
+
+size_t AudioEndToEndTest::GetNumVideoStreams() const {
+ return 0;
+}
+
+size_t AudioEndToEndTest::GetNumAudioStreams() const {
+ return 1;
+}
+
+size_t AudioEndToEndTest::GetNumFlexfecStreams() const {
+ return 0;
+}
+
+std::unique_ptr<test::FakeAudioDevice::Capturer>
+ AudioEndToEndTest::CreateCapturer() {
+ return test::FakeAudioDevice::CreatePulsedNoiseCapturer(32000, kSampleRate);
+}
+
+std::unique_ptr<test::FakeAudioDevice::Renderer>
+ AudioEndToEndTest::CreateRenderer() {
+ return test::FakeAudioDevice::CreateDiscardRenderer(kSampleRate);
+}
+
+void AudioEndToEndTest::OnFakeAudioDevicesCreated(
+ test::FakeAudioDevice* send_audio_device,
+ test::FakeAudioDevice* recv_audio_device) {
+ send_audio_device_ = send_audio_device;
+}
+
+test::PacketTransport* AudioEndToEndTest::CreateSendTransport(
+ SingleThreadedTaskQueueForTesting* task_queue,
+ Call* sender_call) {
+ return new test::PacketTransport(
+ task_queue, sender_call, this, test::PacketTransport::kSender,
+ test::CallTest::payload_type_map_, GetNetworkPipeConfig());
+}
+
+test::PacketTransport* AudioEndToEndTest::CreateReceiveTransport(
+ SingleThreadedTaskQueueForTesting* task_queue) {
+ return new test::PacketTransport(
+ task_queue, nullptr, this, test::PacketTransport::kReceiver,
+ test::CallTest::payload_type_map_, GetNetworkPipeConfig());
+}
+
+void AudioEndToEndTest::ModifyAudioConfigs(
+ AudioSendStream::Config* send_config,
+ std::vector<AudioReceiveStream::Config>* receive_configs) {
+ // Large bitrate by default.
+ const webrtc::SdpAudioFormat kDefaultFormat("opus", 48000, 2,
+ {{"stereo", "1"}});
+ send_config->send_codec_spec =
+ rtc::Optional<AudioSendStream::Config::SendCodecSpec>(
+ {test::CallTest::kAudioSendPayloadType, kDefaultFormat});
+}
+
+void AudioEndToEndTest::OnAudioStreamsCreated(
+ AudioSendStream* send_stream,
+ const std::vector<AudioReceiveStream*>& receive_streams) {
+ ASSERT_NE(nullptr, send_stream);
+ ASSERT_EQ(1u, receive_streams.size());
+ ASSERT_NE(nullptr, receive_streams[0]);
+ send_stream_ = send_stream;
+ receive_stream_ = receive_streams[0];
+}
+
+void AudioEndToEndTest::PerformTest() {
+ // Wait until the input audio file is done...
+ send_audio_device_->WaitForRecordingEnd();
+ // and some extra time to account for network delay.
+ SleepMs(GetNetworkPipeConfig().queue_delay_ms + kExtraRecordTimeMs);
+}
+} // namespace test
+} // namespace webrtc
diff --git a/webrtc/audio/test/low_bandwidth_audio_test.h b/webrtc/audio/test/audio_end_to_end_test.h
similarity index 64%
rename from webrtc/audio/test/low_bandwidth_audio_test.h
rename to webrtc/audio/test/audio_end_to_end_test.h
index ae75707..d14b7a1 100644
--- a/webrtc/audio/test/low_bandwidth_audio_test.h
+++ b/webrtc/audio/test/audio_end_to_end_test.h
@@ -7,28 +7,28 @@
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
-#ifndef WEBRTC_AUDIO_TEST_LOW_BANDWIDTH_AUDIO_TEST_H_
-#define WEBRTC_AUDIO_TEST_LOW_BANDWIDTH_AUDIO_TEST_H_
+#ifndef WEBRTC_AUDIO_TEST_AUDIO_END_TO_END_TEST_H_
+#define WEBRTC_AUDIO_TEST_AUDIO_END_TO_END_TEST_H_
#include <memory>
#include <string>
#include <vector>
#include "webrtc/test/call_test.h"
-#include "webrtc/test/fake_audio_device.h"
namespace webrtc {
namespace test {
-class AudioQualityTest : public test::EndToEndTest {
+class AudioEndToEndTest : public test::EndToEndTest {
public:
- AudioQualityTest();
+ AudioEndToEndTest();
protected:
- virtual std::string AudioInputFile();
- virtual std::string AudioOutputFile();
+ test::FakeAudioDevice* send_audio_device() { return send_audio_device_; }
+ const AudioSendStream* send_stream() const { return send_stream_; }
+ const AudioReceiveStream* receive_stream() const { return receive_stream_; }
- virtual FakeNetworkPipe::Config GetNetworkPipeConfig();
+ virtual FakeNetworkPipe::Config GetNetworkPipeConfig() const;
size_t GetNumVideoStreams() const override;
size_t GetNumAudioStreams() const override;
@@ -50,15 +50,19 @@
void ModifyAudioConfigs(
AudioSendStream::Config* send_config,
std::vector<AudioReceiveStream::Config>* receive_configs) override;
+ void OnAudioStreamsCreated(
+ AudioSendStream* send_stream,
+ const std::vector<AudioReceiveStream*>& receive_streams) override;
void PerformTest() override;
- void OnTestFinished() override;
private:
- test::FakeAudioDevice* send_audio_device_;
+ test::FakeAudioDevice* send_audio_device_ = nullptr;
+ AudioSendStream* send_stream_ = nullptr;
+ AudioReceiveStream* receive_stream_ = nullptr;
};
} // namespace test
} // namespace webrtc
-#endif // WEBRTC_AUDIO_TEST_LOW_BANDWIDTH_AUDIO_TEST_H_
+#endif // WEBRTC_AUDIO_TEST_AUDIO_END_TO_END_TEST_H_
diff --git a/webrtc/audio/test/audio_stats_test.cc b/webrtc/audio/test/audio_stats_test.cc
new file mode 100644
index 0000000..57dfbed
--- /dev/null
+++ b/webrtc/audio/test/audio_stats_test.cc
@@ -0,0 +1,118 @@
+/*
+ * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "webrtc/audio/test/audio_end_to_end_test.h"
+#include "webrtc/rtc_base/safe_compare.h"
+#include "webrtc/system_wrappers/include/sleep.h"
+#include "webrtc/test/gtest.h"
+
+namespace webrtc {
+namespace test {
+namespace {
+
+bool IsNear(int reference, int v) {
+ // Margin is 10%.
+ const int error = reference / 10 + 1;
+ return std::abs(reference - v) <= error;
+}
+
+class NoLossTest : public AudioEndToEndTest {
+ public:
+ const int kTestDurationMs = 8000;
+ const int kBytesSent = 69351;
+ const int32_t kPacketsSent = 400;
+ const int64_t kRttMs = 100;
+
+ NoLossTest() = default;
+
+ FakeNetworkPipe::Config GetNetworkPipeConfig() const override {
+ FakeNetworkPipe::Config pipe_config;
+ pipe_config.queue_delay_ms = kRttMs / 2;
+ return pipe_config;
+ }
+
+ void PerformTest() override {
+ SleepMs(kTestDurationMs);
+ send_audio_device()->StopRecording();
+ AudioEndToEndTest::PerformTest();
+ }
+
+ void OnStreamsStopped() override {
+ AudioSendStream::Stats send_stats = send_stream()->GetStats();
+ EXPECT_PRED2(IsNear, kBytesSent, send_stats.bytes_sent);
+ EXPECT_PRED2(IsNear, kPacketsSent, send_stats.packets_sent);
+ EXPECT_EQ(0, send_stats.packets_lost);
+ EXPECT_EQ(0.0f, send_stats.fraction_lost);
+ EXPECT_EQ("opus", send_stats.codec_name);
+ // send_stats.jitter_ms
+ EXPECT_PRED2(IsNear, kRttMs, send_stats.rtt_ms);
+ // Send level is 0 because it is cleared in TransmitMixer::StopSend().
+ EXPECT_EQ(0, send_stats.audio_level);
+ // send_stats.total_input_energy
+ // send_stats.total_input_duration
+ EXPECT_EQ(-1.0f, send_stats.aec_quality_min);
+ EXPECT_EQ(-1, send_stats.echo_delay_median_ms);
+ EXPECT_EQ(-1, send_stats.echo_delay_std_ms);
+ EXPECT_EQ(-100, send_stats.echo_return_loss);
+ EXPECT_EQ(-100, send_stats.echo_return_loss_enhancement);
+ EXPECT_EQ(0.0f, send_stats.residual_echo_likelihood);
+ EXPECT_EQ(0.0f, send_stats.residual_echo_likelihood_recent_max);
+ EXPECT_EQ(false, send_stats.typing_noise_detected);
+
+ AudioReceiveStream::Stats recv_stats = receive_stream()->GetStats();
+ EXPECT_PRED2(IsNear, kBytesSent, recv_stats.bytes_rcvd);
+ EXPECT_PRED2(IsNear, kPacketsSent, recv_stats.packets_rcvd);
+ EXPECT_EQ(0u, recv_stats.packets_lost);
+ EXPECT_EQ(0.0f, recv_stats.fraction_lost);
+ EXPECT_EQ("opus", send_stats.codec_name);
+ // recv_stats.jitter_ms
+ // recv_stats.jitter_buffer_ms
+ EXPECT_EQ(20u, recv_stats.jitter_buffer_preferred_ms);
+ // recv_stats.delay_estimate_ms
+ // Receive level is 0 because it is cleared in Channel::StopPlayout().
+ EXPECT_EQ(0, recv_stats.audio_level);
+ // recv_stats.total_output_energy
+ // recv_stats.total_samples_received
+ // recv_stats.total_output_duration
+ // recv_stats.concealed_samples
+ // recv_stats.expand_rate
+ // recv_stats.speech_expand_rate
+ EXPECT_EQ(0.0, recv_stats.secondary_decoded_rate);
+ EXPECT_EQ(0.0, recv_stats.secondary_discarded_rate);
+ EXPECT_EQ(0.0, recv_stats.accelerate_rate);
+ EXPECT_EQ(0.0, recv_stats.preemptive_expand_rate);
+ EXPECT_EQ(0, recv_stats.decoding_calls_to_silence_generator);
+ // recv_stats.decoding_calls_to_neteq
+ // recv_stats.decoding_normal
+ // recv_stats.decoding_plc
+ EXPECT_EQ(0, recv_stats.decoding_cng);
+ // recv_stats.decoding_plc_cng
+ // recv_stats.decoding_muted_output
+ // Capture start time is -1 because we do not have an associated send stream
+ // on the receiver side.
+ EXPECT_EQ(-1, recv_stats.capture_start_ntp_time_ms);
+
+ // Match these stats between caller and receiver.
+ EXPECT_EQ(send_stats.local_ssrc, recv_stats.remote_ssrc);
+ EXPECT_EQ(*send_stats.codec_payload_type, *recv_stats.codec_payload_type);
+ EXPECT_TRUE(rtc::SafeEq(send_stats.ext_seqnum, recv_stats.ext_seqnum));
+ }
+};
+} // namespace
+
+using AudioStatsTest = CallTest;
+
+TEST_F(AudioStatsTest, NoLoss) {
+ NoLossTest test;
+ RunBaseTest(&test);
+}
+
+} // namespace test
+} // namespace webrtc
diff --git a/webrtc/audio/test/low_bandwidth_audio_test.cc b/webrtc/audio/test/low_bandwidth_audio_test.cc
index ea0cdf0..8bbadfb 100644
--- a/webrtc/audio/test/low_bandwidth_audio_test.cc
+++ b/webrtc/audio/test/low_bandwidth_audio_test.cc
@@ -8,16 +8,11 @@
* be found in the AUTHORS file in the root of the source tree.
*/
-#include <algorithm>
-
-#include "webrtc/audio/test/low_bandwidth_audio_test.h"
-#include "webrtc/common_audio/wav_file.h"
+#include "webrtc/audio/test/audio_end_to_end_test.h"
#include "webrtc/rtc_base/flags.h"
#include "webrtc/system_wrappers/include/sleep.h"
-#include "webrtc/test/gtest.h"
#include "webrtc/test/testsupport/fileutils.h"
-
DEFINE_int(sample_rate_hz, 16000,
"Sample rate (Hz) of the produced audio files.");
@@ -25,122 +20,59 @@
"Don't do the full audio recording. "
"Used to quickly check that the test runs without crashing.");
+namespace webrtc {
+namespace test {
namespace {
-// Wait half a second between stopping sending and stopping receiving audio.
-constexpr int kExtraRecordTimeMs = 500;
-
std::string FileSampleRateSuffix() {
return std::to_string(FLAG_sample_rate_hz / 1000);
}
-} // namespace
+class AudioQualityTest : public AudioEndToEndTest {
+ public:
+ AudioQualityTest() = default;
-namespace webrtc {
-namespace test {
-
-AudioQualityTest::AudioQualityTest()
- : EndToEndTest(CallTest::kDefaultTimeoutMs) {}
-
-size_t AudioQualityTest::GetNumVideoStreams() const {
- return 0;
-}
-size_t AudioQualityTest::GetNumAudioStreams() const {
- return 1;
-}
-size_t AudioQualityTest::GetNumFlexfecStreams() const {
- return 0;
-}
-
-std::string AudioQualityTest::AudioInputFile() {
- return test::ResourcePath("voice_engine/audio_tiny" + FileSampleRateSuffix(),
- "wav");
-}
-
-std::string AudioQualityTest::AudioOutputFile() {
- const ::testing::TestInfo* const test_info =
- ::testing::UnitTest::GetInstance()->current_test_info();
- return webrtc::test::OutputPath() + "LowBandwidth_" + test_info->name() +
- "_" + FileSampleRateSuffix() + ".wav";
-}
-
-std::unique_ptr<test::FakeAudioDevice::Capturer>
- AudioQualityTest::CreateCapturer() {
- return test::FakeAudioDevice::CreateWavFileReader(AudioInputFile());
-}
-
-std::unique_ptr<test::FakeAudioDevice::Renderer>
- AudioQualityTest::CreateRenderer() {
- return test::FakeAudioDevice::CreateBoundedWavFileWriter(
- AudioOutputFile(), FLAG_sample_rate_hz);
-}
-
-void AudioQualityTest::OnFakeAudioDevicesCreated(
- test::FakeAudioDevice* send_audio_device,
- test::FakeAudioDevice* recv_audio_device) {
- send_audio_device_ = send_audio_device;
-}
-
-FakeNetworkPipe::Config AudioQualityTest::GetNetworkPipeConfig() {
- return FakeNetworkPipe::Config();
-}
-
-test::PacketTransport* AudioQualityTest::CreateSendTransport(
- SingleThreadedTaskQueueForTesting* task_queue,
- Call* sender_call) {
- return new test::PacketTransport(
- task_queue, sender_call, this, test::PacketTransport::kSender,
- test::CallTest::payload_type_map_, GetNetworkPipeConfig());
-}
-
-test::PacketTransport* AudioQualityTest::CreateReceiveTransport(
- SingleThreadedTaskQueueForTesting* task_queue) {
- return new test::PacketTransport(
- task_queue, nullptr, this, test::PacketTransport::kReceiver,
- test::CallTest::payload_type_map_, GetNetworkPipeConfig());
-}
-
-void AudioQualityTest::ModifyAudioConfigs(
- AudioSendStream::Config* send_config,
- std::vector<AudioReceiveStream::Config>* receive_configs) {
- // Large bitrate by default.
- const webrtc::SdpAudioFormat kDefaultFormat("OPUS", 48000, 2,
- {{"stereo", "1"}});
- send_config->send_codec_spec =
- rtc::Optional<AudioSendStream::Config::SendCodecSpec>(
- {test::CallTest::kAudioSendPayloadType, kDefaultFormat});
-}
-
-void AudioQualityTest::PerformTest() {
- if (FLAG_quick) {
- // Let the recording run for a small amount of time to check if it works.
- SleepMs(1000);
- } else {
- // Wait until the input audio file is done...
- send_audio_device_->WaitForRecordingEnd();
- // and some extra time to account for network delay.
- SleepMs(GetNetworkPipeConfig().queue_delay_ms + kExtraRecordTimeMs);
+ private:
+ std::string AudioInputFile() const {
+ return test::ResourcePath(
+ "voice_engine/audio_tiny" + FileSampleRateSuffix(), "wav");
}
-}
-void AudioQualityTest::OnTestFinished() {
- const ::testing::TestInfo* const test_info =
- ::testing::UnitTest::GetInstance()->current_test_info();
+ std::string AudioOutputFile() const {
+ const ::testing::TestInfo* const test_info =
+ ::testing::UnitTest::GetInstance()->current_test_info();
+ return webrtc::test::OutputPath() + "LowBandwidth_" + test_info->name() +
+ "_" + FileSampleRateSuffix() + ".wav";
+ }
- // Output information about the input and output audio files so that further
- // processing can be done by an external process.
- printf("TEST %s %s %s\n", test_info->name(),
- AudioInputFile().c_str(), AudioOutputFile().c_str());
-}
+ std::unique_ptr<test::FakeAudioDevice::Capturer> CreateCapturer() override {
+ return test::FakeAudioDevice::CreateWavFileReader(AudioInputFile());
+ }
+ std::unique_ptr<test::FakeAudioDevice::Renderer> CreateRenderer() override {
+ return test::FakeAudioDevice::CreateBoundedWavFileWriter(
+ AudioOutputFile(), FLAG_sample_rate_hz);
+ }
-using LowBandwidthAudioTest = CallTest;
+ void PerformTest() override {
+ if (FLAG_quick) {
+ // Let the recording run for a small amount of time to check if it works.
+ SleepMs(1000);
+ } else {
+ AudioEndToEndTest::PerformTest();
+ }
+ }
-TEST_F(LowBandwidthAudioTest, GoodNetworkHighBitrate) {
- AudioQualityTest test;
- RunBaseTest(&test);
-}
+ void OnStreamsStopped() override {
+ const ::testing::TestInfo* const test_info =
+ ::testing::UnitTest::GetInstance()->current_test_info();
+ // Output information about the input and output audio files so that further
+ // processing can be done by an external process.
+ printf("TEST %s %s %s\n", test_info->name(),
+ AudioInputFile().c_str(), AudioOutputFile().c_str());
+ }
+};
class Mobile2GNetworkTest : public AudioQualityTest {
void ModifyAudioConfigs(AudioSendStream::Config* send_config,
@@ -156,7 +88,7 @@
{"stereo", "1"}}}});
}
- FakeNetworkPipe::Config GetNetworkPipeConfig() override {
+ FakeNetworkPipe::Config GetNetworkPipeConfig() const override {
FakeNetworkPipe::Config pipe_config;
pipe_config.link_capacity_kbps = 12;
pipe_config.queue_length_packets = 1500;
@@ -164,11 +96,18 @@
return pipe_config;
}
};
+} // namespace
+
+using LowBandwidthAudioTest = CallTest;
+
+TEST_F(LowBandwidthAudioTest, GoodNetworkHighBitrate) {
+ AudioQualityTest test;
+ RunBaseTest(&test);
+}
TEST_F(LowBandwidthAudioTest, Mobile2GNetwork) {
Mobile2GNetworkTest test;
RunBaseTest(&test);
}
-
} // namespace test
} // namespace webrtc