NetEq: Implement logging of Delayed Packet Outage Events
Measures the duration of each packet loss concealment (a.k.a. expand)
event that is not followed by a merge operation.
Having decoded and played packet m−1, the next expected packet is
m. If packet m arrives after some time of packet loss concealment, we
have a delayed packet outage event. However, if instead packet n>m
arrives, we have a lost packet outage event. In NetEq, the two outage
types results in different operations. Both types start with expand
operations to generate audio to play while the buffer is empty. When a
lost packet outage happens, the expand operation(s) are followed by
one merge operation. For delayed packet outages, merge is not done,
and the expand operations are immediately followed by normal
operations.
This change also includes unit tests for the new statistics.
BUG=webrtc:4915, chromium:488124
R=minyue@webrtc.org
Review URL: https://codereview.webrtc.org/1290113002 .
Cr-Commit-Position: refs/heads/master@{#9725}
diff --git a/webrtc/modules/audio_coding/neteq/expand.cc b/webrtc/modules/audio_coding/neteq/expand.cc
index ae12e50..d01465a 100644
--- a/webrtc/modules/audio_coding/neteq/expand.cc
+++ b/webrtc/modules/audio_coding/neteq/expand.cc
@@ -16,10 +16,12 @@
#include <algorithm> // min, max
#include <limits> // numeric_limits<T>
+#include "webrtc/base/safe_conversions.h"
#include "webrtc/common_audio/signal_processing/include/signal_processing_library.h"
#include "webrtc/modules/audio_coding/neteq/background_noise.h"
#include "webrtc/modules/audio_coding/neteq/dsp_helper.h"
#include "webrtc/modules/audio_coding/neteq/random_vector.h"
+#include "webrtc/modules/audio_coding/neteq/statistics_calculator.h"
#include "webrtc/modules/audio_coding/neteq/sync_buffer.h"
namespace webrtc {
@@ -27,6 +29,7 @@
Expand::Expand(BackgroundNoise* background_noise,
SyncBuffer* sync_buffer,
RandomVector* random_vector,
+ StatisticsCalculator* statistics,
int fs,
size_t num_channels)
: random_vector_(random_vector),
@@ -36,10 +39,12 @@
num_channels_(num_channels),
consecutive_expands_(0),
background_noise_(background_noise),
+ statistics_(statistics),
overlap_length_(5 * fs / 8000),
lag_index_direction_(0),
current_lag_index_(0),
stop_muting_(false),
+ expand_duration_samples_(0),
channel_parameters_(new ChannelParameters[num_channels_]) {
assert(fs == 8000 || fs == 16000 || fs == 32000 || fs == 48000);
assert(fs <= kMaxSampleRate); // Should not be possible.
@@ -78,6 +83,7 @@
// Perform initial setup if this is the first expansion since last reset.
AnalyzeSignal(random_vector);
first_expand_ = false;
+ expand_duration_samples_ = 0;
} else {
// This is not the first expansion, parameters are already estimated.
// Extract a noise segment.
@@ -298,6 +304,10 @@
// Increase call number and cap it.
consecutive_expands_ = consecutive_expands_ >= kMaxConsecutiveExpands ?
kMaxConsecutiveExpands : consecutive_expands_ + 1;
+ expand_duration_samples_ += output->Size();
+ // Clamp the duration counter at 2 seconds.
+ expand_duration_samples_ =
+ std::min(expand_duration_samples_, rtc::checked_cast<size_t>(fs_hz_ * 2));
return 0;
}
@@ -305,6 +315,8 @@
current_lag_index_ = 0;
lag_index_direction_ = 0;
stop_muting_ = true; // Do not mute signal any more.
+ statistics_->LogDelayedPacketOutageEvent(
+ rtc::checked_cast<int>(expand_duration_samples_) / (fs_hz_ / 1000));
}
void Expand::SetParametersForMergeAfterExpand() {
@@ -833,10 +845,11 @@
Expand* ExpandFactory::Create(BackgroundNoise* background_noise,
SyncBuffer* sync_buffer,
RandomVector* random_vector,
+ StatisticsCalculator* statistics,
int fs,
size_t num_channels) const {
- return new Expand(background_noise, sync_buffer, random_vector, fs,
- num_channels);
+ return new Expand(background_noise, sync_buffer, random_vector, statistics,
+ fs, num_channels);
}
// TODO(turajs): This can be moved to BackgroundNoise class.
diff --git a/webrtc/modules/audio_coding/neteq/expand.h b/webrtc/modules/audio_coding/neteq/expand.h
index 5fb117d..3fbafdb 100644
--- a/webrtc/modules/audio_coding/neteq/expand.h
+++ b/webrtc/modules/audio_coding/neteq/expand.h
@@ -23,6 +23,7 @@
// Forward declarations.
class BackgroundNoise;
class RandomVector;
+class StatisticsCalculator;
class SyncBuffer;
// This class handles extrapolation of audio data from the sync_buffer to
@@ -34,6 +35,7 @@
Expand(BackgroundNoise* background_noise,
SyncBuffer* sync_buffer,
RandomVector* random_vector,
+ StatisticsCalculator* statistics,
int fs,
size_t num_channels);
@@ -86,8 +88,8 @@
// necessary to produce concealment data.
void AnalyzeSignal(int16_t* random_vector);
- RandomVector* random_vector_;
- SyncBuffer* sync_buffer_;
+ RandomVector* const random_vector_;
+ SyncBuffer* const sync_buffer_;
bool first_expand_;
const int fs_hz_;
const size_t num_channels_;
@@ -127,13 +129,15 @@
void UpdateLagIndex();
- BackgroundNoise* background_noise_;
+ BackgroundNoise* const background_noise_;
+ StatisticsCalculator* const statistics_;
const size_t overlap_length_;
int16_t max_lag_;
size_t expand_lags_[kNumLags];
int lag_index_direction_;
int current_lag_index_;
bool stop_muting_;
+ size_t expand_duration_samples_;
rtc::scoped_ptr<ChannelParameters[]> channel_parameters_;
DISALLOW_COPY_AND_ASSIGN(Expand);
@@ -146,6 +150,7 @@
virtual Expand* Create(BackgroundNoise* background_noise,
SyncBuffer* sync_buffer,
RandomVector* random_vector,
+ StatisticsCalculator* statistics,
int fs,
size_t num_channels) const;
};
diff --git a/webrtc/modules/audio_coding/neteq/expand_unittest.cc b/webrtc/modules/audio_coding/neteq/expand_unittest.cc
index 68b4f60..1441704 100644
--- a/webrtc/modules/audio_coding/neteq/expand_unittest.cc
+++ b/webrtc/modules/audio_coding/neteq/expand_unittest.cc
@@ -13,9 +13,14 @@
#include "webrtc/modules/audio_coding/neteq/expand.h"
#include "testing/gtest/include/gtest/gtest.h"
+#include "webrtc/base/safe_conversions.h"
+#include "webrtc/common_audio/signal_processing/include/signal_processing_library.h"
#include "webrtc/modules/audio_coding/neteq/background_noise.h"
#include "webrtc/modules/audio_coding/neteq/random_vector.h"
+#include "webrtc/modules/audio_coding/neteq/statistics_calculator.h"
#include "webrtc/modules/audio_coding/neteq/sync_buffer.h"
+#include "webrtc/modules/audio_coding/neteq/tools/resample_input_audio_file.h"
+#include "webrtc/test/testsupport/fileutils.h"
namespace webrtc {
@@ -25,7 +30,8 @@
BackgroundNoise bgn(channels);
SyncBuffer sync_buffer(1, 1000);
RandomVector random_vector;
- Expand expand(&bgn, &sync_buffer, &random_vector, fs, channels);
+ StatisticsCalculator statistics;
+ Expand expand(&bgn, &sync_buffer, &random_vector, &statistics, fs, channels);
}
TEST(Expand, CreateUsingFactory) {
@@ -34,13 +40,135 @@
BackgroundNoise bgn(channels);
SyncBuffer sync_buffer(1, 1000);
RandomVector random_vector;
+ StatisticsCalculator statistics;
ExpandFactory expand_factory;
- Expand* expand =
- expand_factory.Create(&bgn, &sync_buffer, &random_vector, fs, channels);
+ Expand* expand = expand_factory.Create(&bgn, &sync_buffer, &random_vector,
+ &statistics, fs, channels);
EXPECT_TRUE(expand != NULL);
delete expand;
}
+namespace {
+class FakeStatisticsCalculator : public StatisticsCalculator {
+ public:
+ void LogDelayedPacketOutageEvent(int outage_duration_ms) override {
+ last_outage_duration_ms_ = outage_duration_ms;
+ }
+
+ int last_outage_duration_ms() const { return last_outage_duration_ms_; }
+
+ private:
+ int last_outage_duration_ms_ = 0;
+};
+
+// This is the same size that is given to the SyncBuffer object in NetEq.
+const size_t kNetEqSyncBufferLengthMs = 720;
+} // namespace
+
+class ExpandTest : public ::testing::Test {
+ protected:
+ ExpandTest()
+ : input_file_(test::ResourcePath("audio_coding/testfile32kHz", "pcm"),
+ 32000),
+ test_sample_rate_hz_(32000),
+ num_channels_(1),
+ background_noise_(num_channels_),
+ sync_buffer_(num_channels_,
+ kNetEqSyncBufferLengthMs * test_sample_rate_hz_ / 1000),
+ expand_(&background_noise_,
+ &sync_buffer_,
+ &random_vector_,
+ &statistics_,
+ test_sample_rate_hz_,
+ num_channels_) {
+ WebRtcSpl_Init();
+ input_file_.set_output_rate_hz(test_sample_rate_hz_);
+ }
+
+ void SetUp() override {
+ // Fast-forward the input file until there is speech (about 1.1 second into
+ // the file).
+ const size_t speech_start_samples =
+ static_cast<size_t>(test_sample_rate_hz_ * 1.1f);
+ ASSERT_TRUE(input_file_.Seek(speech_start_samples));
+
+ // Pre-load the sync buffer with speech data.
+ ASSERT_TRUE(
+ input_file_.Read(sync_buffer_.Size(), &sync_buffer_.Channel(0)[0]));
+ ASSERT_EQ(1u, num_channels_) << "Fix: Must populate all channels.";
+ }
+
+ test::ResampleInputAudioFile input_file_;
+ int test_sample_rate_hz_;
+ size_t num_channels_;
+ BackgroundNoise background_noise_;
+ SyncBuffer sync_buffer_;
+ RandomVector random_vector_;
+ FakeStatisticsCalculator statistics_;
+ Expand expand_;
+};
+
+// This test calls the expand object to produce concealment data a few times,
+// and then ends by calling SetParametersForNormalAfterExpand. This simulates
+// the situation where the packet next up for decoding was just delayed, not
+// lost.
+TEST_F(ExpandTest, DelayedPacketOutage) {
+ AudioMultiVector output(num_channels_);
+ size_t sum_output_len_samples = 0;
+ for (int i = 0; i < 10; ++i) {
+ EXPECT_EQ(0, expand_.Process(&output));
+ EXPECT_GT(output.Size(), 0u);
+ sum_output_len_samples += output.Size();
+ EXPECT_EQ(0, statistics_.last_outage_duration_ms());
+ }
+ expand_.SetParametersForNormalAfterExpand();
+ // Convert |sum_output_len_samples| to milliseconds.
+ EXPECT_EQ(rtc::checked_cast<int>(sum_output_len_samples /
+ (test_sample_rate_hz_ / 1000)),
+ statistics_.last_outage_duration_ms());
+}
+
+// This test is similar to DelayedPacketOutage, but ends by calling
+// SetParametersForMergeAfterExpand. This simulates the situation where the
+// packet next up for decoding was actually lost (or at least a later packet
+// arrived before it).
+TEST_F(ExpandTest, LostPacketOutage) {
+ AudioMultiVector output(num_channels_);
+ size_t sum_output_len_samples = 0;
+ for (int i = 0; i < 10; ++i) {
+ EXPECT_EQ(0, expand_.Process(&output));
+ EXPECT_GT(output.Size(), 0u);
+ sum_output_len_samples += output.Size();
+ EXPECT_EQ(0, statistics_.last_outage_duration_ms());
+ }
+ expand_.SetParametersForMergeAfterExpand();
+ EXPECT_EQ(0, statistics_.last_outage_duration_ms());
+}
+
+// This test is similar to the DelayedPacketOutage test above, but with the
+// difference that Expand::Reset() is called after 5 calls to Expand::Process().
+// This should reset the statistics, and will in the end lead to an outage of
+// 5 periods instead of 10.
+TEST_F(ExpandTest, CheckOutageStatsAfterReset) {
+ AudioMultiVector output(num_channels_);
+ size_t sum_output_len_samples = 0;
+ for (int i = 0; i < 10; ++i) {
+ EXPECT_EQ(0, expand_.Process(&output));
+ EXPECT_GT(output.Size(), 0u);
+ sum_output_len_samples += output.Size();
+ if (i == 5) {
+ expand_.Reset();
+ sum_output_len_samples = 0;
+ }
+ EXPECT_EQ(0, statistics_.last_outage_duration_ms());
+ }
+ expand_.SetParametersForNormalAfterExpand();
+ // Convert |sum_output_len_samples| to milliseconds.
+ EXPECT_EQ(rtc::checked_cast<int>(sum_output_len_samples /
+ (test_sample_rate_hz_ / 1000)),
+ statistics_.last_outage_duration_ms());
+}
+
// TODO(hlundin): Write more tests.
} // namespace webrtc
diff --git a/webrtc/modules/audio_coding/neteq/merge_unittest.cc b/webrtc/modules/audio_coding/neteq/merge_unittest.cc
index bdcbbb8..ddb0e16 100644
--- a/webrtc/modules/audio_coding/neteq/merge_unittest.cc
+++ b/webrtc/modules/audio_coding/neteq/merge_unittest.cc
@@ -18,6 +18,7 @@
#include "webrtc/modules/audio_coding/neteq/background_noise.h"
#include "webrtc/modules/audio_coding/neteq/expand.h"
#include "webrtc/modules/audio_coding/neteq/random_vector.h"
+#include "webrtc/modules/audio_coding/neteq/statistics_calculator.h"
#include "webrtc/modules/audio_coding/neteq/sync_buffer.h"
namespace webrtc {
@@ -28,7 +29,8 @@
BackgroundNoise bgn(channels);
SyncBuffer sync_buffer(1, 1000);
RandomVector random_vector;
- Expand expand(&bgn, &sync_buffer, &random_vector, fs, channels);
+ StatisticsCalculator statistics;
+ Expand expand(&bgn, &sync_buffer, &random_vector, &statistics, fs, channels);
Merge merge(fs, channels, &expand, &sync_buffer);
}
diff --git a/webrtc/modules/audio_coding/neteq/mock/mock_expand.h b/webrtc/modules/audio_coding/neteq/mock/mock_expand.h
index 45e3239..f5ca077 100644
--- a/webrtc/modules/audio_coding/neteq/mock/mock_expand.h
+++ b/webrtc/modules/audio_coding/neteq/mock/mock_expand.h
@@ -22,10 +22,15 @@
MockExpand(BackgroundNoise* background_noise,
SyncBuffer* sync_buffer,
RandomVector* random_vector,
+ StatisticsCalculator* statistics,
int fs,
size_t num_channels)
- : Expand(background_noise, sync_buffer, random_vector, fs, num_channels) {
- }
+ : Expand(background_noise,
+ sync_buffer,
+ random_vector,
+ statistics,
+ fs,
+ num_channels) {}
virtual ~MockExpand() { Die(); }
MOCK_METHOD0(Die, void());
MOCK_METHOD0(Reset,
@@ -46,10 +51,11 @@
class MockExpandFactory : public ExpandFactory {
public:
- MOCK_CONST_METHOD5(Create,
+ MOCK_CONST_METHOD6(Create,
Expand*(BackgroundNoise* background_noise,
SyncBuffer* sync_buffer,
RandomVector* random_vector,
+ StatisticsCalculator* statistics,
int fs,
size_t num_channels));
};
diff --git a/webrtc/modules/audio_coding/neteq/neteq_impl.cc b/webrtc/modules/audio_coding/neteq/neteq_impl.cc
index 3b81999..636ae87 100644
--- a/webrtc/modules/audio_coding/neteq/neteq_impl.cc
+++ b/webrtc/modules/audio_coding/neteq/neteq_impl.cc
@@ -1874,7 +1874,7 @@
// Delete objects and create new ones.
expand_.reset(expand_factory_->Create(background_noise_.get(),
sync_buffer_.get(), &random_vector_,
- fs_hz, channels));
+ &stats_, fs_hz, channels));
merge_.reset(new Merge(fs_hz, channels, expand_.get(), sync_buffer_.get()));
}
diff --git a/webrtc/modules/audio_coding/neteq/normal_unittest.cc b/webrtc/modules/audio_coding/neteq/normal_unittest.cc
index 796409b..1ac32f4 100644
--- a/webrtc/modules/audio_coding/neteq/normal_unittest.cc
+++ b/webrtc/modules/audio_coding/neteq/normal_unittest.cc
@@ -23,6 +23,7 @@
#include "webrtc/modules/audio_coding/neteq/mock/mock_decoder_database.h"
#include "webrtc/modules/audio_coding/neteq/mock/mock_expand.h"
#include "webrtc/modules/audio_coding/neteq/random_vector.h"
+#include "webrtc/modules/audio_coding/neteq/statistics_calculator.h"
#include "webrtc/modules/audio_coding/neteq/sync_buffer.h"
using ::testing::_;
@@ -36,7 +37,8 @@
BackgroundNoise bgn(channels);
SyncBuffer sync_buffer(1, 1000);
RandomVector random_vector;
- Expand expand(&bgn, &sync_buffer, &random_vector, fs, channels);
+ StatisticsCalculator statistics;
+ Expand expand(&bgn, &sync_buffer, &random_vector, &statistics, fs, channels);
Normal normal(fs, &db, bgn, &expand);
EXPECT_CALL(db, Die()); // Called when |db| goes out of scope.
}
@@ -49,7 +51,9 @@
BackgroundNoise bgn(channels);
SyncBuffer sync_buffer(1, 1000);
RandomVector random_vector;
- MockExpand expand(&bgn, &sync_buffer, &random_vector, fs, channels);
+ StatisticsCalculator statistics;
+ MockExpand expand(&bgn, &sync_buffer, &random_vector, &statistics, fs,
+ channels);
Normal normal(fs, &db, bgn, &expand);
int16_t input[1000] = {0};
@@ -93,7 +97,9 @@
BackgroundNoise bgn(channels);
SyncBuffer sync_buffer(channels, 1000);
RandomVector random_vector;
- MockExpand expand(&bgn, &sync_buffer, &random_vector, fs, channels);
+ StatisticsCalculator statistics;
+ MockExpand expand(&bgn, &sync_buffer, &random_vector, &statistics, fs,
+ channels);
Normal normal(fs, &db, bgn, &expand);
int16_t input[1000] = {0};
diff --git a/webrtc/modules/audio_coding/neteq/statistics_calculator.cc b/webrtc/modules/audio_coding/neteq/statistics_calculator.cc
index ce800dd..37a0d50 100644
--- a/webrtc/modules/audio_coding/neteq/statistics_calculator.cc
+++ b/webrtc/modules/audio_coding/neteq/statistics_calculator.cc
@@ -15,6 +15,7 @@
#include "webrtc/modules/audio_coding/neteq/decision_logic.h"
#include "webrtc/modules/audio_coding/neteq/delay_manager.h"
+#include "webrtc/system_wrappers/interface/metrics.h"
namespace webrtc {
@@ -96,6 +97,12 @@
secondary_decoded_samples_ += num_samples;
}
+void StatisticsCalculator::LogDelayedPacketOutageEvent(int outage_duration_ms) {
+ RTC_HISTOGRAM_COUNTS("WebRTC.Audio.DelayedPacketOutageEventMs",
+ outage_duration_ms, 1 /* min */, 2000 /* max */,
+ 100 /* bucket count */);
+}
+
void StatisticsCalculator::StoreWaitingTime(int waiting_time_ms) {
assert(next_waiting_time_index_ < kLenWaitingTimes);
waiting_times_[next_waiting_time_index_] = waiting_time_ms;
diff --git a/webrtc/modules/audio_coding/neteq/statistics_calculator.h b/webrtc/modules/audio_coding/neteq/statistics_calculator.h
index a2cd9be..513322f 100644
--- a/webrtc/modules/audio_coding/neteq/statistics_calculator.h
+++ b/webrtc/modules/audio_coding/neteq/statistics_calculator.h
@@ -73,6 +73,11 @@
// Reports that |num_samples| samples were decoded from secondary packets.
void SecondaryDecodedSamples(int num_samples);
+ // Logs a delayed packet outage event of |outage_duration_ms|. A delayed
+ // packet outage event is defined as an expand period caused not by an actual
+ // packet loss, but by a delayed packet.
+ virtual void LogDelayedPacketOutageEvent(int outage_duration_ms);
+
// Returns the current network statistics in |stats|. The current sample rate
// is |fs_hz|, the total number of samples in packet buffer and sync buffer
// yet to play out is |num_samples_in_buffers|, and the number of samples per
diff --git a/webrtc/modules/audio_coding/neteq/tools/input_audio_file.cc b/webrtc/modules/audio_coding/neteq/tools/input_audio_file.cc
index 6bbb328..e2ec419 100644
--- a/webrtc/modules/audio_coding/neteq/tools/input_audio_file.cc
+++ b/webrtc/modules/audio_coding/neteq/tools/input_audio_file.cc
@@ -10,6 +10,8 @@
#include "webrtc/modules/audio_coding/neteq/tools/input_audio_file.h"
+#include "webrtc/base/checks.h"
+
namespace webrtc {
namespace test {
@@ -37,6 +39,25 @@
return true;
}
+bool InputAudioFile::Seek(int samples) {
+ if (!fp_) {
+ return false;
+ }
+ // Find file boundaries.
+ const long current_pos = ftell(fp_);
+ CHECK_NE(EOF, current_pos) << "Error returned when getting file position.";
+ CHECK_EQ(0, fseek(fp_, 0, SEEK_END)); // Move to end of file.
+ const long file_size = ftell(fp_);
+ CHECK_NE(EOF, file_size) << "Error returned when getting file position.";
+ // Find new position.
+ long new_pos = current_pos + sizeof(int16_t) * samples; // Samples to bytes.
+ CHECK_GE(new_pos, 0) << "Trying to move to before the beginning of the file";
+ new_pos = new_pos % file_size; // Wrap around the end of the file.
+ // Move to new position relative to the beginning of the file.
+ CHECK_EQ(0, fseek(fp_, new_pos, SEEK_SET));
+ return true;
+}
+
void InputAudioFile::DuplicateInterleaved(const int16_t* source, size_t samples,
size_t channels,
int16_t* destination) {
diff --git a/webrtc/modules/audio_coding/neteq/tools/input_audio_file.h b/webrtc/modules/audio_coding/neteq/tools/input_audio_file.h
index 075b5d3..fae5573 100644
--- a/webrtc/modules/audio_coding/neteq/tools/input_audio_file.h
+++ b/webrtc/modules/audio_coding/neteq/tools/input_audio_file.h
@@ -34,6 +34,12 @@
// The output |destination| must have the capacity to hold |samples| elements.
virtual bool Read(size_t samples, int16_t* destination);
+ // Fast-forwards (|samples| > 0) or -backwards (|samples| < 0) the file by the
+ // indicated number of samples. Just like Read(), Seek() starts over at the
+ // beginning of the file if the end is reached. However, seeking backwards
+ // past the beginning of the file is not possible.
+ virtual bool Seek(int samples);
+
// Creates a multi-channel signal from a mono signal. Each sample is repeated
// |channels| times to create an interleaved multi-channel signal where all
// channels are identical. The output |destination| must have the capacity to