Revert "[ACM] iSAC audio codec removed"
This reverts commit b46c4bf27ba5c417fcba7f200d80fa4634e7e1a1.
Reason for revert: breaks a downstream project
Original change's description:
> [ACM] iSAC audio codec removed
>
> Note: this CL has to leave behind one part of iSAC, which is its VAD
> currently used by AGC1 in APM. The target visibility has been
> restricted and the VAD will be removed together with AGC1 when the
> time comes.
>
> Tested: see https://chromium-review.googlesource.com/c/chromium/src/+/4013319
>
> Bug: webrtc:14450
> Change-Id: I69cc518b16280eae62a1f1977cdbfa24c08cf5f9
> Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/282421
> Reviewed-by: Henrik Lundin <henrik.lundin@webrtc.org>
> Reviewed-by: Sam Zackrisson <saza@webrtc.org>
> Reviewed-by: Henrik Boström <hbos@webrtc.org>
> Commit-Queue: Alessio Bazzica <alessiob@webrtc.org>
> Cr-Commit-Position: refs/heads/main@{#38652}
Bug: webrtc:14450
Change-Id: Ice138004e84e8c5f896684e8d01133d4b2a77bb7
No-Presubmit: true
No-Tree-Checks: true
No-Try: true
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/283800
Reviewed-by: Alessio Bazzica <alessiob@webrtc.org>
Commit-Queue: Mirko Bonadei <mbonadei@webrtc.org>
Auto-Submit: Alessio Bazzica <alessiob@webrtc.org>
Reviewed-by: Mirko Bonadei <mbonadei@webrtc.org>
Owners-Override: Mirko Bonadei <mbonadei@webrtc.org>
Bot-Commit: rubber-stamper@appspot.gserviceaccount.com <rubber-stamper@appspot.gserviceaccount.com>
Cr-Commit-Position: refs/heads/main@{#38655}
diff --git a/modules/audio_coding/acm2/acm_receiver_unittest.cc b/modules/audio_coding/acm2/acm_receiver_unittest.cc
index 6dd44b6..e73acc2 100644
--- a/modules/audio_coding/acm2/acm_receiver_unittest.cc
+++ b/modules/audio_coding/acm2/acm_receiver_unittest.cc
@@ -13,7 +13,6 @@
#include <algorithm> // std::min
#include <memory>
-#include "absl/types/optional.h"
#include "api/audio_codecs/builtin_audio_decoder_factory.h"
#include "api/audio_codecs/builtin_audio_encoder_factory.h"
#include "modules/audio_coding/codecs/cng/audio_encoder_cng.h"
@@ -65,14 +64,12 @@
const SdpAudioFormat& format,
const std::map<int, int> cng_payload_types = {}) {
// Create the speech encoder.
- absl::optional<AudioCodecInfo> info =
- encoder_factory_->QueryAudioEncoder(format);
- RTC_CHECK(info.has_value());
+ AudioCodecInfo info = encoder_factory_->QueryAudioEncoder(format).value();
std::unique_ptr<AudioEncoder> enc =
encoder_factory_->MakeAudioEncoder(payload_type, format, absl::nullopt);
// If we have a compatible CN specification, stack a CNG on top.
- auto it = cng_payload_types.find(info->sample_rate_hz);
+ auto it = cng_payload_types.find(info.sample_rate_hz);
if (it != cng_payload_types.end()) {
AudioEncoderCngConfig config;
config.speech_encoder = std::move(enc);
@@ -84,7 +81,7 @@
// Actually start using the new encoder.
acm_->SetEncoder(std::move(enc));
- return *info;
+ return info;
}
int InsertOnePacketOfSilence(const AudioCodecInfo& info) {
@@ -151,7 +148,8 @@
#define MAYBE_SampleRate SampleRate
#endif
TEST_F(AcmReceiverTestOldApi, MAYBE_SampleRate) {
- const std::map<int, SdpAudioFormat> codecs = {{0, {"OPUS", 48000, 2}}};
+ const std::map<int, SdpAudioFormat> codecs = {{0, {"ISAC", 16000, 1}},
+ {1, {"ISAC", 32000, 1}}};
receiver_->SetCodecs(codecs);
constexpr int kOutSampleRateHz = 8000; // Different than codec sample rate.
@@ -235,6 +233,15 @@
}
#if defined(WEBRTC_ANDROID)
+#define MAYBE_VerifyAudioFrameISAC DISABLED_VerifyAudioFrameISAC
+#else
+#define MAYBE_VerifyAudioFrameISAC VerifyAudioFrameISAC
+#endif
+TEST_F(AcmReceiverTestFaxModeOldApi, MAYBE_VerifyAudioFrameISAC) {
+ RunVerifyAudioFrame({"ISAC", 16000, 1});
+}
+
+#if defined(WEBRTC_ANDROID)
#define MAYBE_VerifyAudioFrameOpus DISABLED_VerifyAudioFrameOpus
#else
#define MAYBE_VerifyAudioFrameOpus VerifyAudioFrameOpus
@@ -303,10 +310,12 @@
#else
#define MAYBE_LastAudioCodec LastAudioCodec
#endif
-#if defined(WEBRTC_CODEC_OPUS)
+#if defined(WEBRTC_CODEC_ISAC)
TEST_F(AcmReceiverTestOldApi, MAYBE_LastAudioCodec) {
- const std::map<int, SdpAudioFormat> codecs = {
- {0, {"PCMU", 8000, 1}}, {1, {"PCMA", 8000, 1}}, {2, {"L16", 32000, 1}}};
+ const std::map<int, SdpAudioFormat> codecs = {{0, {"ISAC", 16000, 1}},
+ {1, {"PCMA", 8000, 1}},
+ {2, {"ISAC", 32000, 1}},
+ {3, {"L16", 32000, 1}}};
const std::map<int, int> cng_payload_types = {
{8000, 100}, {16000, 101}, {32000, 102}};
{
diff --git a/modules/audio_coding/acm2/audio_coding_module_unittest.cc b/modules/audio_coding/acm2/audio_coding_module_unittest.cc
index f1eb81c..7e4b764 100644
--- a/modules/audio_coding/acm2/audio_coding_module_unittest.cc
+++ b/modules/audio_coding/acm2/audio_coding_module_unittest.cc
@@ -30,6 +30,7 @@
#include "modules/audio_coding/codecs/cng/audio_encoder_cng.h"
#include "modules/audio_coding/codecs/g711/audio_decoder_pcm.h"
#include "modules/audio_coding/codecs/g711/audio_encoder_pcm.h"
+#include "modules/audio_coding/codecs/isac/main/include/audio_encoder_isac.h"
#include "modules/audio_coding/include/audio_coding_module_typedefs.h"
#include "modules/audio_coding/neteq/tools/audio_checksum.h"
#include "modules/audio_coding/neteq/tools/audio_loop.h"
@@ -301,6 +302,44 @@
EXPECT_EQ(AudioFrameType::kAudioFrameSpeech, packet_cb_.last_frame_type());
}
+#if defined(WEBRTC_CODEC_ISAC) || defined(WEBRTC_CODEC_ISACFX)
+// Verifies that the RTP timestamp series is not reset when the codec is
+// changed.
+TEST_F(AudioCodingModuleTestOldApi, TimestampSeriesContinuesWhenCodecChanges) {
+ RegisterCodec(); // This registers the default codec.
+ uint32_t expected_ts = input_frame_.timestamp_;
+ int blocks_per_packet = pac_size_ / (kSampleRateHz / 100);
+ // Encode 5 packets of the first codec type.
+ const int kNumPackets1 = 5;
+ for (int j = 0; j < kNumPackets1; ++j) {
+ for (int i = 0; i < blocks_per_packet; ++i) {
+ EXPECT_EQ(j, packet_cb_.num_calls());
+ InsertAudio();
+ }
+ EXPECT_EQ(j + 1, packet_cb_.num_calls());
+ EXPECT_EQ(expected_ts, packet_cb_.last_timestamp());
+ expected_ts += pac_size_;
+ }
+
+ // Change codec.
+ audio_format_ = SdpAudioFormat("ISAC", kSampleRateHz, 1);
+ pac_size_ = 480;
+ RegisterCodec();
+ blocks_per_packet = pac_size_ / (kSampleRateHz / 100);
+ // Encode another 5 packets.
+ const int kNumPackets2 = 5;
+ for (int j = 0; j < kNumPackets2; ++j) {
+ for (int i = 0; i < blocks_per_packet; ++i) {
+ EXPECT_EQ(kNumPackets1 + j, packet_cb_.num_calls());
+ InsertAudio();
+ }
+ EXPECT_EQ(kNumPackets1 + j + 1, packet_cb_.num_calls());
+ EXPECT_EQ(expected_ts, packet_cb_.last_timestamp());
+ expected_ts += pac_size_;
+ }
+}
+#endif
+
// Introduce this class to set different expectations on the number of encoded
// bytes. This class expects all encoded packets to be 9 bytes (matching one
// CNG SID frame) or 0 bytes. This test depends on `input_frame_` containing
@@ -381,7 +420,8 @@
DoTest(k10MsBlocksPerPacket, kCngPayloadType);
}
-// A multi-threaded test for ACM that uses the PCM16b 16 kHz codec.
+// A multi-threaded test for ACM. This base class is using the PCM16b 16 kHz
+// codec, while the derive class AcmIsacMtTest is using iSAC.
class AudioCodingModuleMtTestOldApi : public AudioCodingModuleTestOldApi {
protected:
static const int kNumPackets = 500;
@@ -520,6 +560,272 @@
EXPECT_TRUE(RunTest());
}
+// This is a multi-threaded ACM test using iSAC. The test encodes audio
+// from a PCM file. The most recent encoded frame is used as input to the
+// receiving part. Depending on timing, it may happen that the same RTP packet
+// is inserted into the receiver multiple times, but this is a valid use-case,
+// and simplifies the test code a lot.
+class AcmIsacMtTestOldApi : public AudioCodingModuleMtTestOldApi {
+ protected:
+ static const int kNumPackets = 500;
+ static const int kNumPullCalls = 500;
+
+ AcmIsacMtTestOldApi()
+ : AudioCodingModuleMtTestOldApi(), last_packet_number_(0) {}
+
+ ~AcmIsacMtTestOldApi() {}
+
+ void SetUp() override {
+ AudioCodingModuleTestOldApi::SetUp();
+ RegisterCodec(); // Must be called before the threads start below.
+
+ // Set up input audio source to read from specified file, loop after 5
+ // seconds, and deliver blocks of 10 ms.
+ const std::string input_file_name =
+ webrtc::test::ResourcePath("audio_coding/speech_mono_16kHz", "pcm");
+ audio_loop_.Init(input_file_name, 5 * kSampleRateHz, kNumSamples10ms);
+
+ // Generate one packet to have something to insert.
+ int loop_counter = 0;
+ while (packet_cb_.last_payload_len_bytes() == 0) {
+ InsertAudio();
+ ASSERT_LT(loop_counter++, 10);
+ }
+ // Set `last_packet_number_` to one less that `num_calls` so that the packet
+ // will be fetched in the next InsertPacket() call.
+ last_packet_number_ = packet_cb_.num_calls() - 1;
+
+ StartThreads();
+ }
+
+ void RegisterCodec() override {
+ static_assert(kSampleRateHz == 16000, "test designed for iSAC 16 kHz");
+ audio_format_ = SdpAudioFormat("isac", kSampleRateHz, 1);
+ pac_size_ = 480;
+
+ // Register iSAC codec in ACM, effectively unregistering the PCM16B codec
+ // registered in AudioCodingModuleTestOldApi::SetUp();
+ acm_->SetReceiveCodecs({{kPayloadType, *audio_format_}});
+ acm_->SetEncoder(CreateBuiltinAudioEncoderFactory()->MakeAudioEncoder(
+ kPayloadType, *audio_format_, absl::nullopt));
+ }
+
+ void InsertPacket() override {
+ int num_calls = packet_cb_.num_calls(); // Store locally for thread safety.
+ if (num_calls > last_packet_number_) {
+ // Get the new payload out from the callback handler.
+ // Note that since we swap buffers here instead of directly inserting
+ // a pointer to the data in `packet_cb_`, we avoid locking the callback
+ // for the duration of the IncomingPacket() call.
+ packet_cb_.SwapBuffers(&last_payload_vec_);
+ ASSERT_GT(last_payload_vec_.size(), 0u);
+ rtp_utility_->Forward(&rtp_header_);
+ last_packet_number_ = num_calls;
+ }
+ ASSERT_GT(last_payload_vec_.size(), 0u);
+ ASSERT_EQ(0, acm_->IncomingPacket(&last_payload_vec_[0],
+ last_payload_vec_.size(), rtp_header_));
+ }
+
+ void InsertAudio() override {
+ // TODO(kwiberg): Use std::copy here. Might be complications because AFAICS
+ // this call confuses the number of samples with the number of bytes, and
+ // ends up copying only half of what it should.
+ memcpy(input_frame_.mutable_data(), audio_loop_.GetNextBlock().data(),
+ kNumSamples10ms);
+ AudioCodingModuleTestOldApi::InsertAudio();
+ }
+
+ // Override the verification function with no-op, since iSAC produces variable
+ // payload sizes.
+ void VerifyEncoding() override {}
+
+ // This method is the same as AudioCodingModuleMtTestOldApi::TestDone(), but
+ // here it is using the constants defined in this class (i.e., shorter test
+ // run).
+ bool TestDone() override {
+ if (packet_cb_.num_calls() > kNumPackets) {
+ MutexLock lock(&mutex_);
+ if (pull_audio_count_ > kNumPullCalls) {
+ // Both conditions for completion are met. End the test.
+ return true;
+ }
+ }
+ return false;
+ }
+
+ int last_packet_number_;
+ std::vector<uint8_t> last_payload_vec_;
+ test::AudioLoop audio_loop_;
+};
+
+#if defined(WEBRTC_IOS)
+#define MAYBE_DoTest DISABLED_DoTest
+#else
+#define MAYBE_DoTest DoTest
+#endif
+#if defined(WEBRTC_CODEC_ISAC) || defined(WEBRTC_CODEC_ISACFX)
+TEST_F(AcmIsacMtTestOldApi, MAYBE_DoTest) {
+ EXPECT_TRUE(RunTest());
+}
+#endif
+
+class AcmReRegisterIsacMtTestOldApi : public AudioCodingModuleTestOldApi {
+ protected:
+ static const int kRegisterAfterNumPackets = 5;
+ static const int kNumPackets = 10;
+ static const int kPacketSizeMs = 30;
+ static const int kPacketSizeSamples = kPacketSizeMs * 16;
+
+ AcmReRegisterIsacMtTestOldApi()
+ : AudioCodingModuleTestOldApi(),
+ codec_registered_(false),
+ receive_packet_count_(0),
+ next_insert_packet_time_ms_(0),
+ fake_clock_(new SimulatedClock(0)) {
+ AudioEncoderIsacFloatImpl::Config config;
+ config.payload_type = kPayloadType;
+ isac_encoder_.reset(new AudioEncoderIsacFloatImpl(config));
+ clock_ = fake_clock_.get();
+ }
+
+ void SetUp() override {
+ AudioCodingModuleTestOldApi::SetUp();
+ // Set up input audio source to read from specified file, loop after 5
+ // seconds, and deliver blocks of 10 ms.
+ const std::string input_file_name =
+ webrtc::test::ResourcePath("audio_coding/speech_mono_16kHz", "pcm");
+ audio_loop_.Init(input_file_name, 5 * kSampleRateHz, kNumSamples10ms);
+ RegisterCodec(); // Must be called before the threads start below.
+ StartThreads();
+ }
+
+ void RegisterCodec() override {
+ // Register iSAC codec in ACM, effectively unregistering the PCM16B codec
+ // registered in AudioCodingModuleTestOldApi::SetUp();
+ // Only register the decoder for now. The encoder is registered later.
+ static_assert(kSampleRateHz == 16000, "test designed for iSAC 16 kHz");
+ acm_->SetReceiveCodecs({{kPayloadType, {"ISAC", kSampleRateHz, 1}}});
+ }
+
+ void StartThreads() {
+ quit_.store(false);
+ const auto attributes =
+ rtc::ThreadAttributes().SetPriority(rtc::ThreadPriority::kRealtime);
+ receive_thread_ = rtc::PlatformThread::SpawnJoinable(
+ [this] {
+ while (!quit_.load() && CbReceiveImpl()) {
+ }
+ },
+ "receive", attributes);
+ codec_registration_thread_ = rtc::PlatformThread::SpawnJoinable(
+ [this] {
+ while (!quit_.load()) {
+ CbCodecRegistrationImpl();
+ }
+ },
+ "codec_registration", attributes);
+ }
+
+ void TearDown() override {
+ AudioCodingModuleTestOldApi::TearDown();
+ quit_.store(true);
+ receive_thread_.Finalize();
+ codec_registration_thread_.Finalize();
+ }
+
+ bool RunTest() { return test_complete_.Wait(TimeDelta::Minutes(10)); }
+
+ bool CbReceiveImpl() {
+ SleepMs(1);
+ rtc::Buffer encoded;
+ AudioEncoder::EncodedInfo info;
+ {
+ MutexLock lock(&mutex_);
+ if (clock_->TimeInMilliseconds() < next_insert_packet_time_ms_) {
+ return true;
+ }
+ next_insert_packet_time_ms_ += kPacketSizeMs;
+ ++receive_packet_count_;
+
+ // Encode new frame.
+ uint32_t input_timestamp = rtp_header_.timestamp;
+ while (info.encoded_bytes == 0) {
+ info = isac_encoder_->Encode(input_timestamp,
+ audio_loop_.GetNextBlock(), &encoded);
+ input_timestamp += 160; // 10 ms at 16 kHz.
+ }
+ EXPECT_EQ(rtp_header_.timestamp + kPacketSizeSamples, input_timestamp);
+ EXPECT_EQ(rtp_header_.timestamp, info.encoded_timestamp);
+ EXPECT_EQ(rtp_header_.payloadType, info.payload_type);
+ }
+ // Now we're not holding the crit sect when calling ACM.
+
+ // Insert into ACM.
+ EXPECT_EQ(0, acm_->IncomingPacket(encoded.data(), info.encoded_bytes,
+ rtp_header_));
+
+ // Pull audio.
+ for (int i = 0; i < rtc::CheckedDivExact(kPacketSizeMs, 10); ++i) {
+ AudioFrame audio_frame;
+ bool muted;
+ EXPECT_EQ(0, acm_->PlayoutData10Ms(-1 /* default output frequency */,
+ &audio_frame, &muted));
+ if (muted) {
+ ADD_FAILURE();
+ return false;
+ }
+ fake_clock_->AdvanceTimeMilliseconds(10);
+ }
+ rtp_utility_->Forward(&rtp_header_);
+ return true;
+ }
+
+ void CbCodecRegistrationImpl() {
+ SleepMs(1);
+ if (HasFatalFailure()) {
+ // End the test early if a fatal failure (ASSERT_*) has occurred.
+ test_complete_.Set();
+ }
+ MutexLock lock(&mutex_);
+ if (!codec_registered_ &&
+ receive_packet_count_ > kRegisterAfterNumPackets) {
+ // Register the iSAC encoder.
+ acm_->SetEncoder(CreateBuiltinAudioEncoderFactory()->MakeAudioEncoder(
+ kPayloadType, *audio_format_, absl::nullopt));
+ codec_registered_ = true;
+ }
+ if (codec_registered_ && receive_packet_count_ > kNumPackets) {
+ test_complete_.Set();
+ }
+ }
+
+ rtc::PlatformThread receive_thread_;
+ rtc::PlatformThread codec_registration_thread_;
+ // Used to force worker threads to stop looping.
+ std::atomic<bool> quit_;
+
+ rtc::Event test_complete_;
+ Mutex mutex_;
+ bool codec_registered_ RTC_GUARDED_BY(mutex_);
+ int receive_packet_count_ RTC_GUARDED_BY(mutex_);
+ int64_t next_insert_packet_time_ms_ RTC_GUARDED_BY(mutex_);
+ std::unique_ptr<AudioEncoderIsacFloatImpl> isac_encoder_;
+ std::unique_ptr<SimulatedClock> fake_clock_;
+ test::AudioLoop audio_loop_;
+};
+
+#if defined(WEBRTC_IOS)
+#define MAYBE_DoTest DISABLED_DoTest
+#else
+#define MAYBE_DoTest DoTest
+#endif
+#if defined(WEBRTC_CODEC_ISAC) || defined(WEBRTC_CODEC_ISACFX)
+TEST_F(AcmReRegisterIsacMtTestOldApi, MAYBE_DoTest) {
+ EXPECT_TRUE(RunTest());
+}
+#endif
+
// Disabling all of these tests on iOS until file support has been added.
// See https://code.google.com/p/webrtc/issues/detail?id=4752 for details.
#if !defined(WEBRTC_IOS)
@@ -719,6 +1025,38 @@
class AcmSenderBitExactnessNewApi : public AcmSenderBitExactnessOldApi {};
+// Run bit exactness tests only for release builds.
+#if (defined(WEBRTC_CODEC_ISAC) || defined(WEBRTC_CODEC_ISACFX)) && \
+ defined(NDEBUG) && defined(WEBRTC_LINUX) && defined(WEBRTC_ARCH_X86_64)
+TEST_F(AcmSenderBitExactnessOldApi, IsacWb30ms) {
+ ASSERT_NO_FATAL_FAILURE(SetUpTest("ISAC", 16000, 1, 103, 480, 480));
+ Run(/*audio_checksum_ref=*/"37ecdabad1698a857cf811e6d1fa91df",
+ /*payload_checksum_ref=*/"3c79f16f34218271f3dca4e2b1dfe1bb",
+ /*expected_packets=*/33,
+ /*expected_channels=*/test::AcmReceiveTestOldApi::kMonoOutput);
+}
+
+TEST_F(AcmSenderBitExactnessOldApi, IsacWb60ms) {
+ ASSERT_NO_FATAL_FAILURE(SetUpTest("ISAC", 16000, 1, 103, 960, 960));
+ Run(/*audio_checksum_ref=*/"0e9078d23454901496a88362ba0740c3",
+ /*payload_checksum_ref=*/"9e0a0ab743ad987b55b8e14802769c56",
+ /*expected_packets=*/16,
+ /*expected_channels=*/test::AcmReceiveTestOldApi::kMonoOutput);
+}
+#endif
+
+// Run bit exactness test only for release build.
+#if defined(WEBRTC_CODEC_ISAC) && defined(NDEBUG) && defined(WEBRTC_LINUX) && \
+ defined(WEBRTC_ARCH_X86_64)
+TEST_F(AcmSenderBitExactnessOldApi, IsacSwb30ms) {
+ ASSERT_NO_FATAL_FAILURE(SetUpTest("ISAC", 32000, 1, 104, 960, 960));
+ Run(/*audio_checksum_ref=*/"f4cf577f28a0dcbac33358b757518e0c",
+ /*payload_checksum_ref=*/"ce86106a93419aefb063097108ec94ab",
+ /*expected_packets=*/33,
+ /*expected_channels=*/test::AcmReceiveTestOldApi::kMonoOutput);
+}
+#endif
+
TEST_F(AcmSenderBitExactnessOldApi, Pcm16_8000khz_10ms) {
ASSERT_NO_FATAL_FAILURE(SetUpTest("L16", 8000, 1, 107, 80, 80));
Run(/*audio_checksum_ref=*/"69118ed438ac76252d023e0463819471",