Hooking up audio network adaptor to VoE.
BUG=webrtc:6303
Review-Url: https://codereview.webrtc.org/2390883004
Cr-Commit-Position: refs/heads/master@{#14611}
diff --git a/webrtc/modules/audio_coding/codecs/audio_encoder.cc b/webrtc/modules/audio_coding/codecs/audio_encoder.cc
index 1216484..956c4e0 100644
--- a/webrtc/modules/audio_coding/codecs/audio_encoder.cc
+++ b/webrtc/modules/audio_coding/codecs/audio_encoder.cc
@@ -77,9 +77,13 @@
void AudioEncoder::OnReceivedUplinkBandwidth(int uplink_bandwidth_bps) {}
void AudioEncoder::OnReceivedUplinkPacketLossFraction(
- float uplink_packet_loss_fraction) {}
+ float uplink_packet_loss_fraction) {
+ SetProjectedPacketLossRate(uplink_packet_loss_fraction);
+}
-void AudioEncoder::OnReceivedTargetAudioBitrate(int target_audio_bitrate_bps) {}
+void AudioEncoder::OnReceivedTargetAudioBitrate(int target_audio_bitrate_bps) {
+ SetTargetBitrate(target_audio_bitrate_bps);
+}
void AudioEncoder::OnReceivedRtt(int rtt_ms) {}
diff --git a/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.cc b/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.cc
index ae9dae2..29f85ac 100644
--- a/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.cc
+++ b/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.cc
@@ -13,6 +13,7 @@
#include <algorithm>
#include "webrtc/base/checks.h"
+#include "webrtc/base/exp_filter.h"
#include "webrtc/base/safe_conversions.h"
#include "webrtc/common_types.h"
#include "webrtc/modules/audio_coding/audio_network_adaptor/audio_network_adaptor_impl.h"
@@ -24,11 +25,15 @@
namespace {
-const int kSampleRateHz = 48000;
-const int kMinBitrateBps = 500;
-const int kMaxBitrateBps = 512000;
+constexpr int kSampleRateHz = 48000;
+constexpr int kMinBitrateBps = 500;
+constexpr int kMaxBitrateBps = 512000;
constexpr int kSupportedFrameLengths[] = {20, 60};
+// PacketLossFractionSmoother uses an exponential filter with a time constant
+// of -1.0 / ln(0.9999) = 10000 ms.
+constexpr float kAlphaForPacketLossFractionSmoother = 0.9999f;
+
AudioEncoderOpus::Config CreateConfig(const CodecInst& codec_inst) {
AudioEncoderOpus::Config config;
config.frame_size_ms = rtc::CheckedDivExact(codec_inst.pacsize, 48);
@@ -82,6 +87,35 @@
} // namespace
+class AudioEncoderOpus::PacketLossFractionSmoother {
+ public:
+ explicit PacketLossFractionSmoother(const Clock* clock)
+ : clock_(clock),
+ last_sample_time_ms_(clock_->TimeInMilliseconds()),
+ smoother_(kAlphaForPacketLossFractionSmoother) {}
+
+ // Gets the smoothed packet loss fraction.
+ float GetAverage() const {
+ float value = smoother_.filtered();
+ return (value == rtc::ExpFilter::kValueUndefined) ? 0.0f : value;
+ }
+
+ // Add new observation to the packet loss fraction smoother.
+ void AddSample(float packet_loss_fraction) {
+ int64_t now_ms = clock_->TimeInMilliseconds();
+ smoother_.Apply(static_cast<float>(now_ms - last_sample_time_ms_),
+ packet_loss_fraction);
+ last_sample_time_ms_ = now_ms;
+ }
+
+ private:
+ const Clock* const clock_;
+ int64_t last_sample_time_ms_;
+
+ // An exponential filter is used to smooth the packet loss fraction.
+ rtc::ExpFilter smoother_;
+};
+
AudioEncoderOpus::Config::Config() = default;
AudioEncoderOpus::Config::Config(const Config&) = default;
AudioEncoderOpus::Config::~Config() = default;
@@ -113,9 +147,11 @@
AudioNetworkAdaptorCreator&& audio_network_adaptor_creator)
: packet_loss_rate_(0.0),
inst_(nullptr),
+ packet_loss_fraction_smoother_(new PacketLossFractionSmoother(
+ config.clock ? config.clock : Clock::GetRealTimeClock())),
audio_network_adaptor_creator_(
audio_network_adaptor_creator
- ? audio_network_adaptor_creator
+ ? std::move(audio_network_adaptor_creator)
: [this](const std::string& config_string, const Clock* clock) {
return DefaultAudioNetworkAdaptorCreator(config_string,
clock);
@@ -234,8 +270,11 @@
void AudioEncoderOpus::OnReceivedUplinkPacketLossFraction(
float uplink_packet_loss_fraction) {
- if (!audio_network_adaptor_)
- return;
+ if (!audio_network_adaptor_) {
+ packet_loss_fraction_smoother_->AddSample(uplink_packet_loss_fraction);
+ float average_fraction_loss = packet_loss_fraction_smoother_->GetAverage();
+ return SetProjectedPacketLossRate(average_fraction_loss);
+ }
audio_network_adaptor_->SetUplinkPacketLossFraction(
uplink_packet_loss_fraction);
ApplyAudioNetworkAdaptor();
@@ -244,7 +283,7 @@
void AudioEncoderOpus::OnReceivedTargetAudioBitrate(
int target_audio_bitrate_bps) {
if (!audio_network_adaptor_)
- return;
+ return SetTargetBitrate(target_audio_bitrate_bps);
audio_network_adaptor_->SetTargetAudioBitrate(target_audio_bitrate_bps);
ApplyAudioNetworkAdaptor();
}
diff --git a/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.h b/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.h
index 150a841..342668e 100644
--- a/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.h
+++ b/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.h
@@ -49,6 +49,7 @@
int max_playback_rate_hz = 48000;
int complexity = kDefaultComplexity;
bool dtx_enabled = false;
+ const Clock* clock = nullptr;
private:
#if defined(WEBRTC_ANDROID) || defined(WEBRTC_IOS) || defined(WEBRTC_ARCH_ARM)
@@ -115,6 +116,8 @@
rtc::Buffer* encoded) override;
private:
+ class PacketLossFractionSmoother;
+
size_t Num10msFramesPerPacket() const;
size_t SamplesPer10msFrame() const;
size_t SufficientOutputBufferSize() const;
@@ -133,6 +136,7 @@
uint32_t first_timestamp_in_buffer_;
size_t num_channels_to_encode_;
int next_frame_length_ms_;
+ std::unique_ptr<PacketLossFractionSmoother> packet_loss_fraction_smoother_;
AudioNetworkAdaptorCreator audio_network_adaptor_creator_;
std::unique_ptr<AudioNetworkAdaptor> audio_network_adaptor_;
diff --git a/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus_unittest.cc b/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus_unittest.cc
index 3e0e186..6a4c47c 100644
--- a/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus_unittest.cc
+++ b/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus_unittest.cc
@@ -15,6 +15,7 @@
#include "webrtc/modules/audio_coding/audio_network_adaptor/mock/mock_audio_network_adaptor.h"
#include "webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.h"
#include "webrtc/test/gtest.h"
+#include "webrtc/system_wrappers/include/clock.h"
namespace webrtc {
using ::testing::NiceMock;
@@ -23,6 +24,7 @@
namespace {
const CodecInst kDefaultOpusSettings = {105, "opus", 48000, 960, 1, 32000};
+constexpr int64_t kInitialTimeUs = 12345678;
AudioEncoderOpus::Config CreateConfig(const CodecInst& codec_inst) {
AudioEncoderOpus::Config config;
@@ -38,6 +40,7 @@
struct AudioEncoderOpusStates {
std::shared_ptr<MockAudioNetworkAdaptor*> mock_audio_network_adaptor;
std::unique_ptr<AudioEncoderOpus> encoder;
+ std::unique_ptr<SimulatedClock> simulated_clock;
};
AudioEncoderOpusStates CreateCodec(size_t num_channels) {
@@ -63,6 +66,9 @@
CodecInst codec_inst = kDefaultOpusSettings;
codec_inst.channels = num_channels;
auto config = CreateConfig(codec_inst);
+ states.simulated_clock.reset(new SimulatedClock(kInitialTimeUs));
+ config.clock = states.simulated_clock.get();
+
states.encoder.reset(new AudioEncoderOpus(config, std::move(creator)));
return states;
}
@@ -303,4 +309,30 @@
CheckEncoderRuntimeConfig(states.encoder.get(), config);
}
+TEST(AudioEncoderOpusTest,
+ PacketLossFractionSmoothedOnSetUplinkPacketLossFraction) {
+ auto states = CreateCodec(2);
+
+ // The values are carefully chosen so that if no smoothing is made, the test
+ // will fail.
+ constexpr float kPacketLossFraction_1 = 0.02f;
+ constexpr float kPacketLossFraction_2 = 0.198f;
+ // |kSecondSampleTimeMs| is chose to ease the calculation since
+ // 0.9999 ^ 6931 = 0.5.
+ constexpr float kSecondSampleTimeMs = 6931;
+
+ // First time, no filtering.
+ states.encoder->OnReceivedUplinkPacketLossFraction(kPacketLossFraction_1);
+ EXPECT_DOUBLE_EQ(0.01, states.encoder->packet_loss_rate());
+
+ states.simulated_clock->AdvanceTimeMilliseconds(kSecondSampleTimeMs);
+ states.encoder->OnReceivedUplinkPacketLossFraction(kPacketLossFraction_2);
+
+ // Now the output of packet loss fraction smoother should be
+ // (0.02 + 0.198) / 2 = 0.109, which reach the threshold for the optimized
+ // packet loss rate to increase to 0.05. If no smoothing has been made, the
+ // optimized packet loss rate should have been increase to 0.1.
+ EXPECT_DOUBLE_EQ(0.05, states.encoder->packet_loss_rate());
+}
+
} // namespace webrtc
diff --git a/webrtc/modules/audio_coding/include/audio_coding_module.h b/webrtc/modules/audio_coding/include/audio_coding_module.h
index 946ad1d..e62bbd7 100644
--- a/webrtc/modules/audio_coding/include/audio_coding_module.h
+++ b/webrtc/modules/audio_coding/include/audio_coding_module.h
@@ -252,6 +252,9 @@
///////////////////////////////////////////////////////////////////////////
// Sets the bitrate to the specified value in bits/sec. If the value is not
// supported by the codec, it will choose another appropriate value.
+ //
+ // This is only used in test code that rely on old ACM APIs.
+ // TODO(minyue): Remove it when possible.
virtual void SetBitRate(int bitrate_bps) = 0;
// int32_t RegisterTransportCallback()
@@ -371,6 +374,8 @@
// -1 if failed to set packet loss rate,
// 0 if succeeded.
//
+ // This is only used in test code that rely on old ACM APIs.
+ // TODO(minyue): Remove it when possible.
virtual int SetPacketLossRate(int packet_loss_rate) = 0;
///////////////////////////////////////////////////////////////////////////