Added an empty AudioTransportProxy to AudioState.
All audio in calls is now routed through AudioTransportProxy. The
AudioTransport implemented by VoEBaseImpl is disconnected from
AudioDevice and replaced by an empty proxy layer that forwards calls
to the old Transport. This is a refactoring CL in preparation for
landing https://codereview.webrtc.org/2436033002/, which will connect
the new AudioMixer.
In the planned configuration, the currently empty AudioTransportProxy
will query the new mixer for audio instead of polling data from the
old Transport. Mixed audio will be passed to an AudioProcessing
interface. AudioTransportProxy is initialized with an AudioProcessing*,
which is currently unused.
No presubmit since we implement an interface with non-const references.
NOPRESUBMIT=True
BUG=webrtc:6346
Review-Url: https://codereview.webrtc.org/2454373002
Cr-Commit-Position: refs/heads/master@{#15133}
diff --git a/webrtc/audio/BUILD.gn b/webrtc/audio/BUILD.gn
index 1279005..e1d583f 100644
--- a/webrtc/audio/BUILD.gn
+++ b/webrtc/audio/BUILD.gn
@@ -16,6 +16,8 @@
"audio_send_stream.h",
"audio_state.cc",
"audio_state.h",
+ "audio_transport_proxy.cc",
+ "audio_transport_proxy.h",
"conversion.h",
"scoped_voe_interface.h",
]
@@ -29,6 +31,9 @@
"..:webrtc_common",
"../api:audio_mixer_api",
"../api:call_api",
+ "../base:rtc_base_approved",
+ "../modules/audio_device",
+ "../modules/audio_processing",
"../system_wrappers",
"../voice_engine",
]
@@ -43,6 +48,7 @@
]
deps = [
":audio",
+ "../modules/audio_device:mock_audio_device",
"//testing/gmock",
"//testing/gtest",
]
diff --git a/webrtc/audio/DEPS b/webrtc/audio/DEPS
index 476d463..e53d285 100644
--- a/webrtc/audio/DEPS
+++ b/webrtc/audio/DEPS
@@ -1,9 +1,9 @@
include_rules = [
"+webrtc/base",
- "+webrtc/voice_engine",
- "+webrtc/modules/audio_coding/codecs/mock",
"+webrtc/call",
"+webrtc/logging/rtc_event_log",
+ "+webrtc/modules/audio_coding/codecs/mock",
+ "+webrtc/modules/audio_device",
"+webrtc/modules/audio_processing/include",
"+webrtc/modules/bitrate_controller",
"+webrtc/modules/congestion_controller",
diff --git a/webrtc/audio/audio_receive_stream_unittest.cc b/webrtc/audio/audio_receive_stream_unittest.cc
index 0331d9b..0fc93b8 100644
--- a/webrtc/audio/audio_receive_stream_unittest.cc
+++ b/webrtc/audio/audio_receive_stream_unittest.cc
@@ -78,6 +78,10 @@
RegisterVoiceEngineObserver(_)).WillOnce(Return(0));
EXPECT_CALL(voice_engine_,
DeRegisterVoiceEngineObserver()).WillOnce(Return(0));
+ EXPECT_CALL(voice_engine_, audio_processing());
+ EXPECT_CALL(voice_engine_, audio_device_module());
+ EXPECT_CALL(voice_engine_, audio_transport());
+
AudioState::Config config;
config.voice_engine = &voice_engine_;
audio_state_ = AudioState::Create(config);
diff --git a/webrtc/audio/audio_send_stream_unittest.cc b/webrtc/audio/audio_send_stream_unittest.cc
index 427cda3..7e0ea45 100644
--- a/webrtc/audio/audio_send_stream_unittest.cc
+++ b/webrtc/audio/audio_send_stream_unittest.cc
@@ -75,6 +75,10 @@
RegisterVoiceEngineObserver(_)).WillOnce(Return(0));
EXPECT_CALL(voice_engine_,
DeRegisterVoiceEngineObserver()).WillOnce(Return(0));
+ EXPECT_CALL(voice_engine_, audio_device_module());
+ EXPECT_CALL(voice_engine_, audio_processing());
+ EXPECT_CALL(voice_engine_, audio_transport());
+
AudioState::Config config;
config.voice_engine = &voice_engine_;
audio_state_ = AudioState::Create(config);
diff --git a/webrtc/audio/audio_state.cc b/webrtc/audio/audio_state.cc
index e63f97a..95a90a5 100644
--- a/webrtc/audio/audio_state.cc
+++ b/webrtc/audio/audio_state.cc
@@ -13,16 +13,28 @@
#include "webrtc/base/atomicops.h"
#include "webrtc/base/checks.h"
#include "webrtc/base/logging.h"
+#include "webrtc/modules/audio_device/include/audio_device.h"
#include "webrtc/voice_engine/include/voe_errors.h"
namespace webrtc {
namespace internal {
AudioState::AudioState(const AudioState::Config& config)
- : config_(config), voe_base_(config.voice_engine) {
+ : config_(config),
+ voe_base_(config.voice_engine),
+ audio_transport_proxy_(voe_base_->audio_transport(),
+ voe_base_->audio_processing(),
+ config_.audio_mixer) {
process_thread_checker_.DetachFromThread();
// Only one AudioState should be created per VoiceEngine.
RTC_CHECK(voe_base_->RegisterVoiceEngineObserver(*this) != -1);
+
+ auto* const device = voe_base_->audio_device_module();
+ RTC_DCHECK(device);
+
+ // This is needed for the Chrome implementation of RegisterAudioCallback.
+ device->RegisterAudioCallback(nullptr);
+ device->RegisterAudioCallback(&audio_transport_proxy_);
}
AudioState::~AudioState() {
@@ -35,6 +47,10 @@
return config_.voice_engine;
}
+rtc::scoped_refptr<AudioMixer> AudioState::mixer() {
+ return config_.audio_mixer;
+}
+
bool AudioState::typing_noise_detected() const {
RTC_DCHECK(thread_checker_.CalledOnValidThread());
rtc::CritScope lock(&crit_sect_);
diff --git a/webrtc/audio/audio_state.h b/webrtc/audio/audio_state.h
index 31892d0..307f5ce 100644
--- a/webrtc/audio/audio_state.h
+++ b/webrtc/audio/audio_state.h
@@ -12,6 +12,7 @@
#define WEBRTC_AUDIO_AUDIO_STATE_H_
#include "webrtc/api/call/audio_state.h"
+#include "webrtc/audio/audio_transport_proxy.h"
#include "webrtc/audio/scoped_voe_interface.h"
#include "webrtc/base/constructormagic.h"
#include "webrtc/base/criticalsection.h"
@@ -28,6 +29,8 @@
~AudioState() override;
VoiceEngine* voice_engine();
+
+ rtc::scoped_refptr<AudioMixer> mixer();
bool typing_noise_detected() const;
private:
@@ -53,6 +56,10 @@
// Reference count; implementation copied from rtc::RefCountedObject.
mutable volatile int ref_count_ = 0;
+ // Transports mixed audio from the mixer to the audio device and
+ // recorded audio to the VoE AudioTransport.
+ AudioTransportProxy audio_transport_proxy_;
+
RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(AudioState);
};
} // namespace internal
diff --git a/webrtc/audio/audio_state_unittest.cc b/webrtc/audio/audio_state_unittest.cc
index 86a9176..bd39baa 100644
--- a/webrtc/audio/audio_state_unittest.cc
+++ b/webrtc/audio/audio_state_unittest.cc
@@ -24,6 +24,10 @@
RegisterVoiceEngineObserver(testing::_)).WillOnce(testing::Return(0));
EXPECT_CALL(voice_engine_,
DeRegisterVoiceEngineObserver()).WillOnce(testing::Return(0));
+ EXPECT_CALL(voice_engine_, audio_device_module());
+ EXPECT_CALL(voice_engine_, audio_processing());
+ EXPECT_CALL(voice_engine_, audio_transport());
+
config_.voice_engine = &voice_engine_;
}
AudioState::Config& config() { return config_; }
@@ -76,5 +80,40 @@
voe_observer->CallbackOnError(-1, VE_NOT_INITED);
EXPECT_FALSE(audio_state->typing_noise_detected());
}
+
+// Test that RecordedDataIsAvailable calls get to the original transport.
+TEST(AudioStateTest, RecordedAudioArrivesAtOriginalTransport) {
+ using testing::_;
+
+ ConfigHelper helper;
+ auto& voice_engine = helper.voice_engine();
+ auto device =
+ static_cast<MockAudioDeviceModule*>(voice_engine.audio_device_module());
+
+ AudioTransport* audio_transport_proxy = nullptr;
+ ON_CALL(*device, RegisterAudioCallback(_))
+ .WillByDefault(
+ testing::Invoke([&audio_transport_proxy](AudioTransport* transport) {
+ audio_transport_proxy = transport;
+ return 0;
+ }));
+
+ MockAudioTransport original_audio_transport;
+ ON_CALL(voice_engine, audio_transport())
+ .WillByDefault(testing::Return(&original_audio_transport));
+
+ EXPECT_CALL(voice_engine, audio_device_module());
+ std::unique_ptr<internal::AudioState> audio_state(
+ new internal::AudioState(helper.config()));
+
+ // Setup completed. Ensure call of old transport is forwarded to new.
+ uint32_t new_mic_level;
+ EXPECT_CALL(original_audio_transport,
+ RecordedDataIsAvailable(nullptr, 80, 2, 1, 8000, 0, 0, 0, false,
+ testing::Ref(new_mic_level)));
+
+ audio_transport_proxy->RecordedDataIsAvailable(nullptr, 80, 2, 1, 8000, 0, 0,
+ 0, false, new_mic_level);
+}
} // namespace test
} // namespace webrtc
diff --git a/webrtc/audio/audio_transport_proxy.cc b/webrtc/audio/audio_transport_proxy.cc
new file mode 100644
index 0000000..ed72200
--- /dev/null
+++ b/webrtc/audio/audio_transport_proxy.cc
@@ -0,0 +1,97 @@
+/*
+ * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "webrtc/audio/audio_transport_proxy.h"
+
+namespace webrtc {
+
+AudioTransportProxy::AudioTransportProxy(AudioTransport* voe_audio_transport,
+ AudioProcessing* apm,
+ AudioMixer* mixer)
+ : voe_audio_transport_(voe_audio_transport) {
+ RTC_DCHECK(voe_audio_transport);
+ RTC_DCHECK(apm);
+}
+
+AudioTransportProxy::~AudioTransportProxy() {}
+
+int32_t AudioTransportProxy::RecordedDataIsAvailable(
+ const void* audioSamples,
+ const size_t nSamples,
+ const size_t nBytesPerSample,
+ const size_t nChannels,
+ const uint32_t samplesPerSec,
+ const uint32_t totalDelayMS,
+ const int32_t clockDrift,
+ const uint32_t currentMicLevel,
+ const bool keyPressed,
+ uint32_t& newMicLevel) {
+ // Pass call through to original audio transport instance.
+ return voe_audio_transport_->RecordedDataIsAvailable(
+ audioSamples, nSamples, nBytesPerSample, nChannels, samplesPerSec,
+ totalDelayMS, clockDrift, currentMicLevel, keyPressed, newMicLevel);
+}
+
+int32_t AudioTransportProxy::NeedMorePlayData(const size_t nSamples,
+ const size_t nBytesPerSample,
+ const size_t nChannels,
+ const uint32_t samplesPerSec,
+ void* audioSamples,
+ size_t& nSamplesOut,
+ int64_t* elapsed_time_ms,
+ int64_t* ntp_time_ms) {
+ RTC_DCHECK_EQ(sizeof(int16_t) * nChannels, nBytesPerSample);
+ RTC_DCHECK_GE(nChannels, 1u);
+ RTC_DCHECK_LE(nChannels, 2u);
+ RTC_DCHECK_GE(
+ samplesPerSec,
+ static_cast<uint32_t>(AudioProcessing::NativeRate::kSampleRate8kHz));
+ RTC_DCHECK_EQ(nSamples * 100, samplesPerSec);
+ RTC_DCHECK_LE(nBytesPerSample * nSamples * nChannels,
+ sizeof(AudioFrame::data_));
+
+ // Pass call through to original audio transport instance.
+ return voe_audio_transport_->NeedMorePlayData(
+ nSamples, nBytesPerSample, nChannels, samplesPerSec, audioSamples,
+ nSamplesOut, elapsed_time_ms, ntp_time_ms);
+}
+
+void AudioTransportProxy::PushCaptureData(int voe_channel,
+ const void* audio_data,
+ int bits_per_sample,
+ int sample_rate,
+ size_t number_of_channels,
+ size_t number_of_frames) {
+ // This is part of deprecated VoE interface operating on specific
+ // VoE channels. It should not be used.
+ RTC_NOTREACHED();
+}
+
+void AudioTransportProxy::PullRenderData(int bits_per_sample,
+ int sample_rate,
+ size_t number_of_channels,
+ size_t number_of_frames,
+ void* audio_data,
+ int64_t* elapsed_time_ms,
+ int64_t* ntp_time_ms) {
+ RTC_DCHECK_EQ(static_cast<size_t>(bits_per_sample), 8 * sizeof(int16_t));
+ RTC_DCHECK_GE(number_of_channels, 1u);
+ RTC_DCHECK_LE(number_of_channels, 2u);
+ RTC_DCHECK_GE(static_cast<int>(sample_rate),
+ AudioProcessing::NativeRate::kSampleRate8kHz);
+ RTC_DCHECK_EQ(static_cast<int>(number_of_frames * 100), sample_rate);
+ RTC_DCHECK_LE(bits_per_sample / 8 * number_of_frames * number_of_channels,
+ sizeof(AudioFrame::data_));
+ voe_audio_transport_->PullRenderData(
+ bits_per_sample, sample_rate, number_of_channels, number_of_frames,
+ audio_data, elapsed_time_ms, ntp_time_ms);
+}
+
+} // namespace webrtc
diff --git a/webrtc/audio/audio_transport_proxy.h b/webrtc/audio/audio_transport_proxy.h
new file mode 100644
index 0000000..05e52fc
--- /dev/null
+++ b/webrtc/audio/audio_transport_proxy.h
@@ -0,0 +1,72 @@
+/*
+ * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef WEBRTC_AUDIO_AUDIO_TRANSPORT_PROXY_H_
+#define WEBRTC_AUDIO_AUDIO_TRANSPORT_PROXY_H_
+
+#include "webrtc/api/audio/audio_mixer.h"
+#include "webrtc/base/constructormagic.h"
+#include "webrtc/modules/audio_device/include/audio_device_defines.h"
+#include "webrtc/modules/audio_processing/include/audio_processing.h"
+
+namespace webrtc {
+
+class AudioTransportProxy : public AudioTransport {
+ public:
+ AudioTransportProxy(AudioTransport* voe_audio_transport,
+ AudioProcessing* apm,
+ AudioMixer* mixer);
+
+ ~AudioTransportProxy() override;
+
+ int32_t RecordedDataIsAvailable(const void* audioSamples,
+ const size_t nSamples,
+ const size_t nBytesPerSample,
+ const size_t nChannels,
+ const uint32_t samplesPerSec,
+ const uint32_t totalDelayMS,
+ const int32_t clockDrift,
+ const uint32_t currentMicLevel,
+ const bool keyPressed,
+ uint32_t& newMicLevel) override;
+
+ int32_t NeedMorePlayData(const size_t nSamples,
+ const size_t nBytesPerSample,
+ const size_t nChannels,
+ const uint32_t samplesPerSec,
+ void* audioSamples,
+ size_t& nSamplesOut,
+ int64_t* elapsed_time_ms,
+ int64_t* ntp_time_ms) override;
+
+ void PushCaptureData(int voe_channel,
+ const void* audio_data,
+ int bits_per_sample,
+ int sample_rate,
+ size_t number_of_channels,
+ size_t number_of_frames) override;
+
+ void PullRenderData(int bits_per_sample,
+ int sample_rate,
+ size_t number_of_channels,
+ size_t number_of_frames,
+ void* audio_data,
+ int64_t* elapsed_time_ms,
+ int64_t* ntp_time_ms) override;
+
+ private:
+ AudioTransport* voe_audio_transport_;
+ AudioFrame frame_for_mixing_;
+
+ RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(AudioTransportProxy);
+};
+} // namespace webrtc
+
+#endif // WEBRTC_AUDIO_AUDIO_TRANSPORT_PROXY_H_
diff --git a/webrtc/call/BUILD.gn b/webrtc/call/BUILD.gn
index 98ea9bc..ad20f34 100644
--- a/webrtc/call/BUILD.gn
+++ b/webrtc/call/BUILD.gn
@@ -21,9 +21,12 @@
suppressed_configs += [ "//build/config/clang:find_bad_constructs" ]
}
+ public_deps = [
+ "../api:call_api",
+ ]
+
deps = [
"..:webrtc_common",
- "../api:call_api",
"../audio",
"../base:rtc_task_queue",
"../logging:rtc_event_log_impl",
@@ -46,6 +49,7 @@
]
deps = [
":call",
+ "../modules/audio_device:mock_audio_device",
"//testing/gmock",
"//testing/gtest",
]
diff --git a/webrtc/call/DEPS b/webrtc/call/DEPS
index 4256b58..01ad1d6 100644
--- a/webrtc/call/DEPS
+++ b/webrtc/call/DEPS
@@ -3,6 +3,8 @@
"+webrtc/base",
"+webrtc/logging/rtc_event_log",
"+webrtc/modules/audio_coding",
+ "+webrtc/modules/audio_device",
+ "+webrtc/modules/audio_processing",
"+webrtc/modules/bitrate_controller",
"+webrtc/modules/congestion_controller",
"+webrtc/modules/pacing",
diff --git a/webrtc/call/call_unittest.cc b/webrtc/call/call_unittest.cc
index 2b93deb..d981b0b 100644
--- a/webrtc/call/call_unittest.cc
+++ b/webrtc/call/call_unittest.cc
@@ -26,6 +26,9 @@
: voice_engine_(decoder_factory) {
webrtc::AudioState::Config audio_state_config;
audio_state_config.voice_engine = &voice_engine_;
+ EXPECT_CALL(voice_engine_, audio_device_module());
+ EXPECT_CALL(voice_engine_, audio_processing());
+ EXPECT_CALL(voice_engine_, audio_transport());
webrtc::Call::Config config(&event_log_);
config.audio_state = webrtc::AudioState::Create(audio_state_config);
call_.reset(webrtc::Call::Create(config));
diff --git a/webrtc/test/BUILD.gn b/webrtc/test/BUILD.gn
index 326a3b1..28251e6 100644
--- a/webrtc/test/BUILD.gn
+++ b/webrtc/test/BUILD.gn
@@ -333,6 +333,8 @@
"../audio",
"../base:rtc_base_approved",
"../call",
+ "../modules/audio_device:mock_audio_device",
+ "../modules/audio_processing",
"../modules/media_file",
"../modules/video_capture:video_capture_module",
"../video",
diff --git a/webrtc/test/DEPS b/webrtc/test/DEPS
index ccf57d0..9c41bd4 100644
--- a/webrtc/test/DEPS
+++ b/webrtc/test/DEPS
@@ -6,6 +6,7 @@
"+webrtc/media/base",
"+webrtc/modules/audio_coding",
"+webrtc/modules/audio_device",
+ "+webrtc/modules/audio_processing",
"+webrtc/modules/media_file",
"+webrtc/modules/rtp_rtcp",
"+webrtc/modules/video_capture",
diff --git a/webrtc/test/mock_voice_engine.h b/webrtc/test/mock_voice_engine.h
index 84c1f5a..ced0751 100644
--- a/webrtc/test/mock_voice_engine.h
+++ b/webrtc/test/mock_voice_engine.h
@@ -13,6 +13,9 @@
#include <memory>
+#include "webrtc/modules/audio_device/include/mock_audio_device.h"
+#include "webrtc/modules/audio_device/include/mock_audio_transport.h"
+#include "webrtc/modules/audio_processing/include/mock_audio_processing.h"
#include "webrtc/test/gmock.h"
#include "webrtc/test/mock_voe_channel_proxy.h"
#include "webrtc/voice_engine/voice_engine_impl.h"
@@ -47,6 +50,13 @@
.WillRepeatedly(testing::ReturnRef(decoder_factory_));
return proxy;
}));
+
+ ON_CALL(*this, audio_device_module())
+ .WillByDefault(testing::Return(&mock_audio_device_));
+ ON_CALL(*this, audio_processing())
+ .WillByDefault(testing::Return(&mock_audio_processing_));
+ ON_CALL(*this, audio_transport())
+ .WillByDefault(testing::Return(&mock_audio_transport_));
}
virtual ~MockVoiceEngine() /* override */ {
// Decrease ref count before base class d-tor is called; otherwise it will
@@ -111,6 +121,7 @@
AudioProcessing* audioproc,
const rtc::scoped_refptr<AudioDecoderFactory>& decoder_factory));
MOCK_METHOD0(audio_processing, AudioProcessing*());
+ MOCK_METHOD0(audio_device_module, AudioDeviceModule*());
MOCK_METHOD0(Terminate, int());
MOCK_METHOD0(CreateChannel, int());
MOCK_METHOD1(CreateChannel, int(const ChannelConfig& config));
@@ -330,6 +341,10 @@
// return a dangling reference. Fortunately, this should go away once
// voe::Channel does.
rtc::scoped_refptr<AudioDecoderFactory> decoder_factory_;
+
+ MockAudioDeviceModule mock_audio_device_;
+ MockAudioProcessing mock_audio_processing_;
+ MockAudioTransport mock_audio_transport_;
};
} // namespace test
} // namespace webrtc