Hooking up audio network adaptor to VoE.
BUG=webrtc:6303
Review-Url: https://codereview.webrtc.org/2390883004
Cr-Commit-Position: refs/heads/master@{#14611}
diff --git a/webrtc/modules/audio_coding/codecs/audio_encoder.cc b/webrtc/modules/audio_coding/codecs/audio_encoder.cc
index 1216484..956c4e0 100644
--- a/webrtc/modules/audio_coding/codecs/audio_encoder.cc
+++ b/webrtc/modules/audio_coding/codecs/audio_encoder.cc
@@ -77,9 +77,13 @@
void AudioEncoder::OnReceivedUplinkBandwidth(int uplink_bandwidth_bps) {}
void AudioEncoder::OnReceivedUplinkPacketLossFraction(
- float uplink_packet_loss_fraction) {}
+ float uplink_packet_loss_fraction) {
+ SetProjectedPacketLossRate(uplink_packet_loss_fraction);
+}
-void AudioEncoder::OnReceivedTargetAudioBitrate(int target_audio_bitrate_bps) {}
+void AudioEncoder::OnReceivedTargetAudioBitrate(int target_audio_bitrate_bps) {
+ SetTargetBitrate(target_audio_bitrate_bps);
+}
void AudioEncoder::OnReceivedRtt(int rtt_ms) {}
diff --git a/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.cc b/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.cc
index ae9dae2..29f85ac 100644
--- a/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.cc
+++ b/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.cc
@@ -13,6 +13,7 @@
#include <algorithm>
#include "webrtc/base/checks.h"
+#include "webrtc/base/exp_filter.h"
#include "webrtc/base/safe_conversions.h"
#include "webrtc/common_types.h"
#include "webrtc/modules/audio_coding/audio_network_adaptor/audio_network_adaptor_impl.h"
@@ -24,11 +25,15 @@
namespace {
-const int kSampleRateHz = 48000;
-const int kMinBitrateBps = 500;
-const int kMaxBitrateBps = 512000;
+constexpr int kSampleRateHz = 48000;
+constexpr int kMinBitrateBps = 500;
+constexpr int kMaxBitrateBps = 512000;
constexpr int kSupportedFrameLengths[] = {20, 60};
+// PacketLossFractionSmoother uses an exponential filter with a time constant
+// of -1.0 / ln(0.9999) = 10000 ms.
+constexpr float kAlphaForPacketLossFractionSmoother = 0.9999f;
+
AudioEncoderOpus::Config CreateConfig(const CodecInst& codec_inst) {
AudioEncoderOpus::Config config;
config.frame_size_ms = rtc::CheckedDivExact(codec_inst.pacsize, 48);
@@ -82,6 +87,35 @@
} // namespace
+class AudioEncoderOpus::PacketLossFractionSmoother {
+ public:
+ explicit PacketLossFractionSmoother(const Clock* clock)
+ : clock_(clock),
+ last_sample_time_ms_(clock_->TimeInMilliseconds()),
+ smoother_(kAlphaForPacketLossFractionSmoother) {}
+
+ // Gets the smoothed packet loss fraction.
+ float GetAverage() const {
+ float value = smoother_.filtered();
+ return (value == rtc::ExpFilter::kValueUndefined) ? 0.0f : value;
+ }
+
+ // Add new observation to the packet loss fraction smoother.
+ void AddSample(float packet_loss_fraction) {
+ int64_t now_ms = clock_->TimeInMilliseconds();
+ smoother_.Apply(static_cast<float>(now_ms - last_sample_time_ms_),
+ packet_loss_fraction);
+ last_sample_time_ms_ = now_ms;
+ }
+
+ private:
+ const Clock* const clock_;
+ int64_t last_sample_time_ms_;
+
+ // An exponential filter is used to smooth the packet loss fraction.
+ rtc::ExpFilter smoother_;
+};
+
AudioEncoderOpus::Config::Config() = default;
AudioEncoderOpus::Config::Config(const Config&) = default;
AudioEncoderOpus::Config::~Config() = default;
@@ -113,9 +147,11 @@
AudioNetworkAdaptorCreator&& audio_network_adaptor_creator)
: packet_loss_rate_(0.0),
inst_(nullptr),
+ packet_loss_fraction_smoother_(new PacketLossFractionSmoother(
+ config.clock ? config.clock : Clock::GetRealTimeClock())),
audio_network_adaptor_creator_(
audio_network_adaptor_creator
- ? audio_network_adaptor_creator
+ ? std::move(audio_network_adaptor_creator)
: [this](const std::string& config_string, const Clock* clock) {
return DefaultAudioNetworkAdaptorCreator(config_string,
clock);
@@ -234,8 +270,11 @@
void AudioEncoderOpus::OnReceivedUplinkPacketLossFraction(
float uplink_packet_loss_fraction) {
- if (!audio_network_adaptor_)
- return;
+ if (!audio_network_adaptor_) {
+ packet_loss_fraction_smoother_->AddSample(uplink_packet_loss_fraction);
+ float average_fraction_loss = packet_loss_fraction_smoother_->GetAverage();
+ return SetProjectedPacketLossRate(average_fraction_loss);
+ }
audio_network_adaptor_->SetUplinkPacketLossFraction(
uplink_packet_loss_fraction);
ApplyAudioNetworkAdaptor();
@@ -244,7 +283,7 @@
void AudioEncoderOpus::OnReceivedTargetAudioBitrate(
int target_audio_bitrate_bps) {
if (!audio_network_adaptor_)
- return;
+ return SetTargetBitrate(target_audio_bitrate_bps);
audio_network_adaptor_->SetTargetAudioBitrate(target_audio_bitrate_bps);
ApplyAudioNetworkAdaptor();
}
diff --git a/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.h b/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.h
index 150a841..342668e 100644
--- a/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.h
+++ b/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.h
@@ -49,6 +49,7 @@
int max_playback_rate_hz = 48000;
int complexity = kDefaultComplexity;
bool dtx_enabled = false;
+ const Clock* clock = nullptr;
private:
#if defined(WEBRTC_ANDROID) || defined(WEBRTC_IOS) || defined(WEBRTC_ARCH_ARM)
@@ -115,6 +116,8 @@
rtc::Buffer* encoded) override;
private:
+ class PacketLossFractionSmoother;
+
size_t Num10msFramesPerPacket() const;
size_t SamplesPer10msFrame() const;
size_t SufficientOutputBufferSize() const;
@@ -133,6 +136,7 @@
uint32_t first_timestamp_in_buffer_;
size_t num_channels_to_encode_;
int next_frame_length_ms_;
+ std::unique_ptr<PacketLossFractionSmoother> packet_loss_fraction_smoother_;
AudioNetworkAdaptorCreator audio_network_adaptor_creator_;
std::unique_ptr<AudioNetworkAdaptor> audio_network_adaptor_;
diff --git a/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus_unittest.cc b/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus_unittest.cc
index 3e0e186..6a4c47c 100644
--- a/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus_unittest.cc
+++ b/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus_unittest.cc
@@ -15,6 +15,7 @@
#include "webrtc/modules/audio_coding/audio_network_adaptor/mock/mock_audio_network_adaptor.h"
#include "webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.h"
#include "webrtc/test/gtest.h"
+#include "webrtc/system_wrappers/include/clock.h"
namespace webrtc {
using ::testing::NiceMock;
@@ -23,6 +24,7 @@
namespace {
const CodecInst kDefaultOpusSettings = {105, "opus", 48000, 960, 1, 32000};
+constexpr int64_t kInitialTimeUs = 12345678;
AudioEncoderOpus::Config CreateConfig(const CodecInst& codec_inst) {
AudioEncoderOpus::Config config;
@@ -38,6 +40,7 @@
struct AudioEncoderOpusStates {
std::shared_ptr<MockAudioNetworkAdaptor*> mock_audio_network_adaptor;
std::unique_ptr<AudioEncoderOpus> encoder;
+ std::unique_ptr<SimulatedClock> simulated_clock;
};
AudioEncoderOpusStates CreateCodec(size_t num_channels) {
@@ -63,6 +66,9 @@
CodecInst codec_inst = kDefaultOpusSettings;
codec_inst.channels = num_channels;
auto config = CreateConfig(codec_inst);
+ states.simulated_clock.reset(new SimulatedClock(kInitialTimeUs));
+ config.clock = states.simulated_clock.get();
+
states.encoder.reset(new AudioEncoderOpus(config, std::move(creator)));
return states;
}
@@ -303,4 +309,30 @@
CheckEncoderRuntimeConfig(states.encoder.get(), config);
}
+TEST(AudioEncoderOpusTest,
+ PacketLossFractionSmoothedOnSetUplinkPacketLossFraction) {
+ auto states = CreateCodec(2);
+
+ // The values are carefully chosen so that if no smoothing is made, the test
+ // will fail.
+ constexpr float kPacketLossFraction_1 = 0.02f;
+ constexpr float kPacketLossFraction_2 = 0.198f;
+ // |kSecondSampleTimeMs| is chose to ease the calculation since
+ // 0.9999 ^ 6931 = 0.5.
+ constexpr float kSecondSampleTimeMs = 6931;
+
+ // First time, no filtering.
+ states.encoder->OnReceivedUplinkPacketLossFraction(kPacketLossFraction_1);
+ EXPECT_DOUBLE_EQ(0.01, states.encoder->packet_loss_rate());
+
+ states.simulated_clock->AdvanceTimeMilliseconds(kSecondSampleTimeMs);
+ states.encoder->OnReceivedUplinkPacketLossFraction(kPacketLossFraction_2);
+
+ // Now the output of packet loss fraction smoother should be
+ // (0.02 + 0.198) / 2 = 0.109, which reach the threshold for the optimized
+ // packet loss rate to increase to 0.05. If no smoothing has been made, the
+ // optimized packet loss rate should have been increase to 0.1.
+ EXPECT_DOUBLE_EQ(0.05, states.encoder->packet_loss_rate());
+}
+
} // namespace webrtc
diff --git a/webrtc/modules/audio_coding/include/audio_coding_module.h b/webrtc/modules/audio_coding/include/audio_coding_module.h
index 946ad1d..e62bbd7 100644
--- a/webrtc/modules/audio_coding/include/audio_coding_module.h
+++ b/webrtc/modules/audio_coding/include/audio_coding_module.h
@@ -252,6 +252,9 @@
///////////////////////////////////////////////////////////////////////////
// Sets the bitrate to the specified value in bits/sec. If the value is not
// supported by the codec, it will choose another appropriate value.
+ //
+ // This is only used in test code that rely on old ACM APIs.
+ // TODO(minyue): Remove it when possible.
virtual void SetBitRate(int bitrate_bps) = 0;
// int32_t RegisterTransportCallback()
@@ -371,6 +374,8 @@
// -1 if failed to set packet loss rate,
// 0 if succeeded.
//
+ // This is only used in test code that rely on old ACM APIs.
+ // TODO(minyue): Remove it when possible.
virtual int SetPacketLossRate(int packet_loss_rate) = 0;
///////////////////////////////////////////////////////////////////////////
diff --git a/webrtc/voice_engine/BUILD.gn b/webrtc/voice_engine/BUILD.gn
index dd5546b..0b6bb78 100644
--- a/webrtc/voice_engine/BUILD.gn
+++ b/webrtc/voice_engine/BUILD.gn
@@ -30,8 +30,6 @@
"include/voe_volume_control.h",
"monitor_module.cc",
"monitor_module.h",
- "network_predictor.cc",
- "network_predictor.h",
"output_mixer.cc",
"output_mixer.h",
"shared_data.cc",
@@ -206,7 +204,6 @@
sources = [
"channel_unittest.cc",
- "network_predictor_unittest.cc",
"test/channel_transport/udp_socket_manager_unittest.cc",
"test/channel_transport/udp_socket_wrapper_unittest.cc",
"test/channel_transport/udp_transport_unittest.cc",
diff --git a/webrtc/voice_engine/channel.cc b/webrtc/voice_engine/channel.cc
index 9713299..0965963 100644
--- a/webrtc/voice_engine/channel.cc
+++ b/webrtc/voice_engine/channel.cc
@@ -899,7 +899,6 @@
_outputSpeechType(AudioFrame::kNormalSpeech),
restored_packet_in_use_(false),
rtcp_observer_(new VoERtcpObserver(this)),
- network_predictor_(new NetworkPredictor(Clock::GetRealTimeClock())),
associate_send_channel_(ChannelOwner(nullptr)),
pacing_enabled_(config.enable_voice_pacing),
feedback_observer_proxy_(new TransportFeedbackProxy()),
@@ -1331,19 +1330,18 @@
void Channel::SetBitRate(int bitrate_bps) {
WEBRTC_TRACE(kTraceInfo, kTraceVoice, VoEId(_instanceId, _channelId),
"Channel::SetBitRate(bitrate_bps=%d)", bitrate_bps);
- audio_coding_->SetBitRate(bitrate_bps);
+ audio_coding_->ModifyEncoder([&](std::unique_ptr<AudioEncoder>* encoder) {
+ if (*encoder)
+ (*encoder)->OnReceivedTargetAudioBitrate(bitrate_bps);
+ });
retransmission_rate_limiter_->SetMaxRate(bitrate_bps);
}
void Channel::OnIncomingFractionLoss(int fraction_lost) {
- network_predictor_->UpdatePacketLossRate(fraction_lost);
- uint8_t average_fraction_loss = network_predictor_->GetLossRate();
-
- // Normalizes rate to 0 - 100.
- if (audio_coding_->SetPacketLossRate(100 * average_fraction_loss / 255) !=
- 0) {
- assert(false); // This should not happen.
- }
+ audio_coding_->ModifyEncoder([&](std::unique_ptr<AudioEncoder>* encoder) {
+ if (*encoder)
+ (*encoder)->OnReceivedUplinkPacketLossFraction(fraction_lost / 255.0f);
+ });
}
int32_t Channel::SetVADStatus(bool enableVAD,
@@ -1540,6 +1538,34 @@
return success;
}
+bool Channel::EnableAudioNetworkAdaptor(const std::string& config_string) {
+ bool success = false;
+ audio_coding_->ModifyEncoder([&](std::unique_ptr<AudioEncoder>* encoder) {
+ if (*encoder) {
+ success = (*encoder)->EnableAudioNetworkAdaptor(
+ config_string, Clock::GetRealTimeClock());
+ }
+ });
+ return success;
+}
+
+void Channel::DisableAudioNetworkAdaptor() {
+ audio_coding_->ModifyEncoder([&](std::unique_ptr<AudioEncoder>* encoder) {
+ if (*encoder)
+ (*encoder)->DisableAudioNetworkAdaptor();
+ });
+}
+
+void Channel::SetReceiverFrameLengthRange(int min_frame_length_ms,
+ int max_frame_length_ms) {
+ audio_coding_->ModifyEncoder([&](std::unique_ptr<AudioEncoder>* encoder) {
+ if (*encoder) {
+ (*encoder)->SetReceiverFrameLengthRange(min_frame_length_ms,
+ max_frame_length_ms);
+ }
+ });
+}
+
int32_t Channel::RegisterExternalTransport(Transport* transport) {
WEBRTC_TRACE(kTraceInfo, kTraceVoice, VoEId(_instanceId, _channelId),
"Channel::RegisterExternalTransport()");
@@ -1700,6 +1726,12 @@
}
retransmission_rate_limiter_->SetWindowSize(nack_window_ms);
+ // Invoke audio encoders OnReceivedRtt().
+ audio_coding_->ModifyEncoder([&](std::unique_ptr<AudioEncoder>* encoder) {
+ if (*encoder)
+ (*encoder)->OnReceivedRtt(rtt);
+ });
+
uint32_t ntp_secs = 0;
uint32_t ntp_frac = 0;
uint32_t rtp_timestamp = 0;
diff --git a/webrtc/voice_engine/channel.h b/webrtc/voice_engine/channel.h
index f2d48d8..fc123e3 100644
--- a/webrtc/voice_engine/channel.h
+++ b/webrtc/voice_engine/channel.h
@@ -32,7 +32,6 @@
#include "webrtc/voice_engine/include/voe_base.h"
#include "webrtc/voice_engine/include/voe_network.h"
#include "webrtc/voice_engine/level_indicator.h"
-#include "webrtc/voice_engine/network_predictor.h"
#include "webrtc/voice_engine/shared_data.h"
#include "webrtc/voice_engine/voice_engine_defines.h"
@@ -209,6 +208,10 @@
int SetOpusMaxPlaybackRate(int frequency_hz);
int SetOpusDtx(bool enable_dtx);
int GetOpusDtx(bool* enabled);
+ bool EnableAudioNetworkAdaptor(const std::string& config_string);
+ void DisableAudioNetworkAdaptor();
+ void SetReceiverFrameLengthRange(int min_frame_length_ms,
+ int max_frame_length_ms);
// VoENetwork
int32_t RegisterExternalTransport(Transport* transport);
@@ -537,7 +540,6 @@
bool restored_packet_in_use_;
// RtcpBandwidthObserver
std::unique_ptr<VoERtcpObserver> rtcp_observer_;
- std::unique_ptr<NetworkPredictor> network_predictor_;
// An associated send channel.
rtc::CriticalSection assoc_send_channel_lock_;
ChannelOwner associate_send_channel_ GUARDED_BY(assoc_send_channel_lock_);
diff --git a/webrtc/voice_engine/network_predictor.cc b/webrtc/voice_engine/network_predictor.cc
deleted file mode 100644
index f8fa2e3..0000000
--- a/webrtc/voice_engine/network_predictor.cc
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved.
- *
- * Use of this source code is governed by a BSD-style license
- * that can be found in the LICENSE file in the root of the source
- * tree. An additional intellectual property rights grant can be found
- * in the file PATENTS. All contributing project authors may
- * be found in the AUTHORS file in the root of the source tree.
- */
-
-#include "webrtc/voice_engine/network_predictor.h"
-
-namespace webrtc {
-namespace voe {
-
-NetworkPredictor::NetworkPredictor(Clock* clock)
- : clock_(clock),
- last_loss_rate_update_time_ms_(clock_->TimeInMilliseconds()),
- loss_rate_filter_(new rtc::ExpFilter(0.9999f)) {
-}
-
-uint8_t NetworkPredictor::GetLossRate() {
- float value = loss_rate_filter_->filtered();
- return (value == rtc::ExpFilter::kValueUndefined) ? 0 :
- static_cast<uint8_t>(value + 0.5);
-}
-
-void NetworkPredictor::UpdatePacketLossRate(uint8_t loss_rate) {
- int64_t now_ms = clock_->TimeInMilliseconds();
- // Update the recursive average filter.
- loss_rate_filter_->Apply(
- static_cast<float>(now_ms - last_loss_rate_update_time_ms_),
- static_cast<float>(loss_rate));
- last_loss_rate_update_time_ms_ = now_ms;
-}
-
-} // namespace voe
-} // namespace webrtc
diff --git a/webrtc/voice_engine/network_predictor.h b/webrtc/voice_engine/network_predictor.h
deleted file mode 100644
index bf08fe9..0000000
--- a/webrtc/voice_engine/network_predictor.h
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved.
- *
- * Use of this source code is governed by a BSD-style license
- * that can be found in the LICENSE file in the root of the source
- * tree. An additional intellectual property rights grant can be found
- * in the file PATENTS. All contributing project authors may
- * be found in the AUTHORS file in the root of the source tree.
- */
-
-#ifndef WEBRTC_VOICE_ENGINE_NETWORK_PREDICTOR_H_
-#define WEBRTC_VOICE_ENGINE_NETWORK_PREDICTOR_H_
-
-#include <memory>
-
-#include "webrtc/base/exp_filter.h"
-#include "webrtc/system_wrappers/include/clock.h"
-
-namespace webrtc {
-
-namespace voe {
-
-// NetworkPredictor is to predict network conditions e.g., packet loss rate, for
-// sender and/or receiver to cope with changes in the network condition.
-class NetworkPredictor {
- public:
- explicit NetworkPredictor(Clock* clock);
- ~NetworkPredictor() {}
-
- // Gets the predicted packet loss rate.
- uint8_t GetLossRate();
-
- // Updates the packet loss rate predictor, on receiving a new observation of
- // packet loss rate from past. Input packet loss rate should be in the
- // interval [0, 255].
- void UpdatePacketLossRate(uint8_t loss_rate);
-
- private:
- Clock* clock_;
- int64_t last_loss_rate_update_time_ms_;
-
- // An exponential filter is used to predict packet loss rate.
- std::unique_ptr<rtc::ExpFilter> loss_rate_filter_;
-};
-
-} // namespace voe
-} // namespace webrtc
-#endif // WEBRTC_VOICE_ENGINE_NETWORK_PREDICTOR_H_
diff --git a/webrtc/voice_engine/network_predictor_unittest.cc b/webrtc/voice_engine/network_predictor_unittest.cc
deleted file mode 100644
index 9012379..0000000
--- a/webrtc/voice_engine/network_predictor_unittest.cc
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved.
- *
- * Use of this source code is governed by a BSD-style license
- * that can be found in the LICENSE file in the root of the source
- * tree. An additional intellectual property rights grant can be found
- * in the file PATENTS. All contributing project authors may
- * be found in the AUTHORS file in the root of the source tree.
- */
-
-#include <math.h>
-
-#include <memory>
-
-#include "webrtc/system_wrappers/include/clock.h"
-#include "webrtc/test/gtest.h"
-#include "webrtc/voice_engine/network_predictor.h"
-
-namespace webrtc {
-namespace voe {
-
-class TestNetworkPredictor : public ::testing::Test {
- protected:
- TestNetworkPredictor()
- : clock_(0),
- network_predictor_(new NetworkPredictor(&clock_)) {}
- SimulatedClock clock_;
- std::unique_ptr<NetworkPredictor> network_predictor_;
-};
-
-TEST_F(TestNetworkPredictor, TestPacketLossRateFilter) {
- // Test initial packet loss rate estimate is 0.
- EXPECT_EQ(0, network_predictor_->GetLossRate());
- network_predictor_->UpdatePacketLossRate(32);
- // First time, no filtering.
- EXPECT_EQ(32, network_predictor_->GetLossRate());
- clock_.AdvanceTimeMilliseconds(1000);
- network_predictor_->UpdatePacketLossRate(40);
- float exp = pow(0.9999f, 1000);
- float value = 32.0f * exp + (1 - exp) * 40.0f;
- EXPECT_EQ(static_cast<uint8_t>(value + 0.5f),
- network_predictor_->GetLossRate());
-}
-} // namespace voe
-} // namespace webrtc
diff --git a/webrtc/voice_engine/voice_engine.gyp b/webrtc/voice_engine/voice_engine.gyp
index c264fe7..f27db2f 100644
--- a/webrtc/voice_engine/voice_engine.gyp
+++ b/webrtc/voice_engine/voice_engine.gyp
@@ -56,8 +56,6 @@
'channel_proxy.h',
'monitor_module.cc',
'monitor_module.h',
- 'network_predictor.cc',
- 'network_predictor.h',
'output_mixer.cc',
'output_mixer.h',
'shared_data.cc',