Insert audio frame transformer between depacketizer and decoder.
The frame transformer is passed from RTPReceiverInterface through the
library to be eventually set in ChannelReceive, where the frame
transformation will occur in the follow-up CL.
Insertable Streams Web API explainer:
https://github.com/alvestrand/webrtc-media-streams/blob/master/explainer.md
Design doc for WebRTC library changes:
http://doc/1eiLkjNUkRy2FssCPLUp6eH08BZuXXoHfbbBP1ZN7EVk
Bug: webrtc:11380
Change-Id: I5af06d1431047ef50d00e304cf95e92a832b4220
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/171872
Reviewed-by: Magnus Flodman <mflodman@webrtc.org>
Reviewed-by: Tommi <tommi@webrtc.org>
Reviewed-by: Per Åhgren <peah@webrtc.org>
Commit-Queue: Marina Ciocea <marinaciocea@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#30956}
diff --git a/audio/audio_receive_stream.cc b/audio/audio_receive_stream.cc
index ce1b344..6bc0d41 100644
--- a/audio/audio_receive_stream.cc
+++ b/audio/audio_receive_stream.cc
@@ -82,7 +82,8 @@
config.jitter_buffer_max_packets, config.jitter_buffer_fast_accelerate,
config.jitter_buffer_min_delay_ms,
config.jitter_buffer_enable_rtx_handling, config.decoder_factory,
- config.codec_pair_id, config.frame_decryptor, config.crypto_options);
+ config.codec_pair_id, config.frame_decryptor, config.crypto_options,
+ std::move(config.frame_transformer));
}
} // namespace
@@ -409,6 +410,12 @@
channel_receive->SetReceiveCodecs(new_config.decoder_map);
}
+ if (first_time ||
+ old_config.frame_transformer != new_config.frame_transformer) {
+ channel_receive->SetDepacketizerToDecoderFrameTransformer(
+ new_config.frame_transformer);
+ }
+
stream->config_ = new_config;
}
} // namespace internal
diff --git a/audio/audio_receive_stream_unittest.cc b/audio/audio_receive_stream_unittest.cc
index 0b2cae5..186eb1c 100644
--- a/audio/audio_receive_stream_unittest.cc
+++ b/audio/audio_receive_stream_unittest.cc
@@ -100,6 +100,8 @@
.WillRepeatedly(Invoke([](const std::map<int, SdpAudioFormat>& codecs) {
EXPECT_THAT(codecs, ::testing::IsEmpty());
}));
+ EXPECT_CALL(*channel_receive_, SetDepacketizerToDecoderFrameTransformer(_))
+ .Times(1);
stream_config_.rtp.local_ssrc = kLocalSsrc;
stream_config_.rtp.remote_ssrc = kRemoteSsrc;
diff --git a/audio/channel_receive.cc b/audio/channel_receive.cc
index dfc8493..1c88421 100644
--- a/audio/channel_receive.cc
+++ b/audio/channel_receive.cc
@@ -20,6 +20,7 @@
#include <vector>
#include "api/crypto/frame_decryptor_interface.h"
+#include "api/frame_transformer_interface.h"
#include "api/rtc_event_log/rtc_event_log.h"
#include "audio/audio_level.h"
#include "audio/channel_send.h"
@@ -79,22 +80,24 @@
class ChannelReceive : public ChannelReceiveInterface {
public:
// Used for receive streams.
- ChannelReceive(Clock* clock,
- ProcessThread* module_process_thread,
- NetEqFactory* neteq_factory,
- AudioDeviceModule* audio_device_module,
- Transport* rtcp_send_transport,
- RtcEventLog* rtc_event_log,
- uint32_t local_ssrc,
- uint32_t remote_ssrc,
- size_t jitter_buffer_max_packets,
- bool jitter_buffer_fast_playout,
- int jitter_buffer_min_delay_ms,
- bool jitter_buffer_enable_rtx_handling,
- rtc::scoped_refptr<AudioDecoderFactory> decoder_factory,
- absl::optional<AudioCodecPairId> codec_pair_id,
- rtc::scoped_refptr<FrameDecryptorInterface> frame_decryptor,
- const webrtc::CryptoOptions& crypto_options);
+ ChannelReceive(
+ Clock* clock,
+ ProcessThread* module_process_thread,
+ NetEqFactory* neteq_factory,
+ AudioDeviceModule* audio_device_module,
+ Transport* rtcp_send_transport,
+ RtcEventLog* rtc_event_log,
+ uint32_t local_ssrc,
+ uint32_t remote_ssrc,
+ size_t jitter_buffer_max_packets,
+ bool jitter_buffer_fast_playout,
+ int jitter_buffer_min_delay_ms,
+ bool jitter_buffer_enable_rtx_handling,
+ rtc::scoped_refptr<AudioDecoderFactory> decoder_factory,
+ absl::optional<AudioCodecPairId> codec_pair_id,
+ rtc::scoped_refptr<FrameDecryptorInterface> frame_decryptor,
+ const webrtc::CryptoOptions& crypto_options,
+ rtc::scoped_refptr<FrameTransformerInterface> frame_transformer);
~ChannelReceive() override;
void SetSink(AudioSinkInterface* sink) override;
@@ -161,6 +164,12 @@
// Used for obtaining RTT for a receive-only channel.
void SetAssociatedSendChannel(const ChannelSendInterface* channel) override;
+ // Sets a frame transformer between the depacketizer and the decoder, to
+ // transform the received frames before decoding them.
+ void SetDepacketizerToDecoderFrameTransformer(
+ rtc::scoped_refptr<webrtc::FrameTransformerInterface> frame_transformer)
+ override;
+
private:
void ReceivePacket(const uint8_t* packet,
size_t packet_length,
@@ -262,6 +271,8 @@
webrtc::CryptoOptions crypto_options_;
webrtc::AbsoluteCaptureTimeReceiver absolute_capture_time_receiver_;
+
+ rtc::scoped_refptr<FrameTransformerInterface> frame_transformer_;
};
void ChannelReceive::OnReceivedPayloadData(
@@ -422,7 +433,8 @@
rtc::scoped_refptr<AudioDecoderFactory> decoder_factory,
absl::optional<AudioCodecPairId> codec_pair_id,
rtc::scoped_refptr<FrameDecryptorInterface> frame_decryptor,
- const webrtc::CryptoOptions& crypto_options)
+ const webrtc::CryptoOptions& crypto_options,
+ rtc::scoped_refptr<FrameTransformerInterface> frame_transformer)
: event_log_(rtc_event_log),
rtp_receive_statistics_(ReceiveStatistics::Create(clock)),
remote_ssrc_(remote_ssrc),
@@ -444,7 +456,8 @@
associated_send_channel_(nullptr),
frame_decryptor_(frame_decryptor),
crypto_options_(crypto_options),
- absolute_capture_time_receiver_(clock) {
+ absolute_capture_time_receiver_(clock),
+ frame_transformer_(std::move(frame_transformer)) {
// TODO(nisse): Use _moduleProcessThreadPtr instead?
module_process_thread_checker_.Detach();
@@ -742,6 +755,12 @@
associated_send_channel_ = channel;
}
+void ChannelReceive::SetDepacketizerToDecoderFrameTransformer(
+ rtc::scoped_refptr<webrtc::FrameTransformerInterface> frame_transformer) {
+ RTC_DCHECK(worker_thread_checker_.IsCurrent());
+ frame_transformer_ = std::move(frame_transformer);
+}
+
NetworkStatistics ChannelReceive::GetNetworkStatistics() const {
RTC_DCHECK(worker_thread_checker_.IsCurrent());
NetworkStatistics stats;
@@ -927,13 +946,15 @@
rtc::scoped_refptr<AudioDecoderFactory> decoder_factory,
absl::optional<AudioCodecPairId> codec_pair_id,
rtc::scoped_refptr<FrameDecryptorInterface> frame_decryptor,
- const webrtc::CryptoOptions& crypto_options) {
+ const webrtc::CryptoOptions& crypto_options,
+ rtc::scoped_refptr<FrameTransformerInterface> frame_transformer) {
return std::make_unique<ChannelReceive>(
clock, module_process_thread, neteq_factory, audio_device_module,
rtcp_send_transport, rtc_event_log, local_ssrc, remote_ssrc,
jitter_buffer_max_packets, jitter_buffer_fast_playout,
jitter_buffer_min_delay_ms, jitter_buffer_enable_rtx_handling,
- decoder_factory, codec_pair_id, frame_decryptor, crypto_options);
+ decoder_factory, codec_pair_id, frame_decryptor, crypto_options,
+ std::move(frame_transformer));
}
} // namespace voe
diff --git a/audio/channel_receive.h b/audio/channel_receive.h
index 034ac7b..bc02ff3 100644
--- a/audio/channel_receive.h
+++ b/audio/channel_receive.h
@@ -22,6 +22,7 @@
#include "api/call/audio_sink.h"
#include "api/call/transport.h"
#include "api/crypto/crypto_options.h"
+#include "api/frame_transformer_interface.h"
#include "api/neteq/neteq_factory.h"
#include "api/transport/rtp/rtp_source.h"
#include "call/rtp_packet_sink_interface.h"
@@ -137,6 +138,12 @@
// Used for obtaining RTT for a receive-only channel.
virtual void SetAssociatedSendChannel(
const ChannelSendInterface* channel) = 0;
+
+ // Sets a frame transformer between the depacketizer and the decoder, to
+ // transform the received frames before decoding them.
+ virtual void SetDepacketizerToDecoderFrameTransformer(
+ rtc::scoped_refptr<webrtc::FrameTransformerInterface>
+ frame_transformer) = 0;
};
std::unique_ptr<ChannelReceiveInterface> CreateChannelReceive(
@@ -155,7 +162,8 @@
rtc::scoped_refptr<AudioDecoderFactory> decoder_factory,
absl::optional<AudioCodecPairId> codec_pair_id,
rtc::scoped_refptr<FrameDecryptorInterface> frame_decryptor,
- const webrtc::CryptoOptions& crypto_options);
+ const webrtc::CryptoOptions& crypto_options,
+ rtc::scoped_refptr<FrameTransformerInterface> frame_transformer);
} // namespace voe
} // namespace webrtc
diff --git a/audio/mock_voe_channel_proxy.h b/audio/mock_voe_channel_proxy.h
index 9a013ff..38ad208 100644
--- a/audio/mock_voe_channel_proxy.h
+++ b/audio/mock_voe_channel_proxy.h
@@ -66,6 +66,9 @@
MOCK_CONST_METHOD0(GetSources, std::vector<RtpSource>());
MOCK_METHOD0(StartPlayout, void());
MOCK_METHOD0(StopPlayout, void());
+ MOCK_METHOD1(SetDepacketizerToDecoderFrameTransformer,
+ void(rtc::scoped_refptr<webrtc::FrameTransformerInterface>
+ frame_transformer));
};
class MockChannelSend : public voe::ChannelSendInterface {