Adds detection of audio glitches for playout on iOS (reland)
Second attempt to land https://chromium-review.googlesource.com/c/522563/
TBR: minyue
Bug: b/38018041
Change-Id: I938f4a490b6357cd1ac7b34fe445215a746fab43
Reviewed-on: https://chromium-review.googlesource.com/533214
Commit-Queue: Henrik Andreasson <henrika@webrtc.org>
Reviewed-by: Minyue Li <minyue@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#18572}diff --git a/webrtc/modules/audio_device/BUILD.gn b/webrtc/modules/audio_device/BUILD.gn
index b3ba852..511f159 100644
--- a/webrtc/modules/audio_device/BUILD.gn
+++ b/webrtc/modules/audio_device/BUILD.gn
@@ -92,6 +92,9 @@
}
defines = []
cflags = []
+ if (rtc_audio_device_plays_sinus_tone) {
+ defines += [ "AUDIO_DEVICE_PLAYS_SINUS_TONE" ]
+ }
if (rtc_include_internal_audio_device) {
sources += [
"audio_device_data_observer.cc",
diff --git a/webrtc/modules/audio_device/audio_device_buffer.cc b/webrtc/modules/audio_device/audio_device_buffer.cc
index 8b49a3f..b074c67 100644
--- a/webrtc/modules/audio_device/audio_device_buffer.cc
+++ b/webrtc/modules/audio_device/audio_device_buffer.cc
@@ -9,6 +9,7 @@
*/
#include <algorithm>
+#include <cmath>
#include "webrtc/modules/audio_device/audio_device_buffer.h"
@@ -36,6 +37,9 @@
static const size_t kMinValidCallTimeTimeInSeconds = 10;
static const size_t kMinValidCallTimeTimeInMilliseconds =
kMinValidCallTimeTimeInSeconds * rtc::kNumMillisecsPerSec;
+#ifdef AUDIO_DEVICE_PLAYS_SINUS_TONE
+static const double k2Pi = 6.28318530717959;
+#endif
AudioDeviceBuffer::AudioDeviceBuffer()
: task_queue_(kTimerQueueName),
@@ -60,6 +64,10 @@
only_silence_recorded_(true),
log_stats_(false) {
LOG(INFO) << "AudioDeviceBuffer::ctor";
+#ifdef AUDIO_DEVICE_PLAYS_SINUS_TONE
+ phase_ = 0.0;
+ LOG(WARNING) << "AUDIO_DEVICE_PLAYS_SINUS_TONE is defined!";
+#endif
playout_thread_checker_.DetachFromThread();
recording_thread_checker_.DetachFromThread();
}
@@ -391,9 +399,18 @@
int32_t AudioDeviceBuffer::GetPlayoutData(void* audio_buffer) {
RTC_DCHECK_RUN_ON(&playout_thread_checker_);
RTC_DCHECK_GT(play_buffer_.size(), 0);
- const size_t bytes_per_sample = sizeof(int16_t);
+#ifdef AUDIO_DEVICE_PLAYS_SINUS_TONE
+ const double phase_increment =
+ k2Pi * 440.0 / static_cast<double>(play_sample_rate_);
+ int16_t* destination_r = reinterpret_cast<int16_t*>(audio_buffer);
+ for (size_t i = 0; i < play_buffer_.size(); ++i) {
+ destination_r[i] = static_cast<int16_t>((sin(phase_) * (1 << 14)));
+ phase_ += phase_increment;
+ }
+#else
memcpy(audio_buffer, play_buffer_.data(),
- play_buffer_.size() * bytes_per_sample);
+ play_buffer_.size() * sizeof(int16_t));
+#endif
// Return samples per channel or number of frames.
return static_cast<int32_t>(play_buffer_.size() / play_channels_);
}
diff --git a/webrtc/modules/audio_device/audio_device_buffer.h b/webrtc/modules/audio_device/audio_device_buffer.h
index 1466e69..c2ea638 100644
--- a/webrtc/modules/audio_device/audio_device_buffer.h
+++ b/webrtc/modules/audio_device/audio_device_buffer.h
@@ -21,6 +21,7 @@
#include "webrtc/typedefs.h"
namespace webrtc {
+
// Delta times between two successive playout callbacks are limited to this
// value before added to an internal array.
const size_t kMaxDeltaTimeInMs = 500;
@@ -251,6 +252,12 @@
// Setting this member to false prevents (possiby invalid) log messages from
// being printed in the LogStats() task.
bool log_stats_ ACCESS_ON(task_queue_);
+
+// Should *never* be defined in production builds. Only used for testing.
+// When defined, the output signal will be replaced by a sinus tone at 440Hz.
+#ifdef AUDIO_DEVICE_PLAYS_SINUS_TONE
+ double phase_ ACCESS_ON(playout_thread_checker_);
+#endif
};
} // namespace webrtc
diff --git a/webrtc/modules/audio_device/ios/audio_device_ios.h b/webrtc/modules/audio_device/ios/audio_device_ios.h
index 0faa1c6..c69640a 100644
--- a/webrtc/modules/audio_device/ios/audio_device_ios.h
+++ b/webrtc/modules/audio_device/ios/audio_device_ios.h
@@ -189,6 +189,7 @@
void HandleValidRouteChange();
void HandleCanPlayOrRecordChange(bool can_play_or_record);
void HandleSampleRateChange(float sample_rate);
+ void HandlePlayoutGlitchDetected();
// Uses current |playout_parameters_| and |record_parameters_| to inform the
// audio device buffer (ADB) about our internal audio parameters.
@@ -290,6 +291,13 @@
// Set to true if we've activated the audio session.
bool has_configured_session_;
+ // Counts number of detected audio glitches on the playout side.
+ int64_t num_detected_playout_glitches_;
+ int64_t last_playout_time_;
+
+ // Counts number of playout callbacks per call.
+ int64_t num_playout_callbacks_;
+
// Exposes private members for testing purposes only.
FRIEND_TEST_ALL_PREFIXES(AudioDeviceTest, testInterruptedAudioSession);
};
diff --git a/webrtc/modules/audio_device/ios/audio_device_ios.mm b/webrtc/modules/audio_device/ios/audio_device_ios.mm
index 128ea53..3df9c71 100644
--- a/webrtc/modules/audio_device/ios/audio_device_ios.mm
+++ b/webrtc/modules/audio_device/ios/audio_device_ios.mm
@@ -23,8 +23,10 @@
#include "webrtc/base/logging.h"
#include "webrtc/base/thread.h"
#include "webrtc/base/thread_annotations.h"
+#include "webrtc/base/timeutils.h"
#include "webrtc/modules/audio_device/fine_audio_buffer.h"
#include "webrtc/sdk/objc/Framework/Classes/Common/helpers.h"
+#include "webrtc/system_wrappers/include/metrics.h"
#import "WebRTC/RTCLogging.h"
#import "webrtc/modules/audio_device/ios/objc/RTCAudioSessionDelegateAdapter.h"
@@ -66,6 +68,7 @@
kMessageTypeInterruptionEnd,
kMessageTypeValidRouteChange,
kMessageTypeCanPlayOrRecordChange,
+ kMessageTypePlayoutGlitchDetected,
};
using ios::CheckAndLogError;
@@ -109,7 +112,10 @@
initialized_(false),
audio_is_initialized_(false),
is_interrupted_(false),
- has_configured_session_(false) {
+ has_configured_session_(false),
+ num_detected_playout_glitches_(0),
+ last_playout_time_(0),
+ num_playout_callbacks_(0) {
LOGI() << "ctor" << ios::GetCurrentThreadDescription();
thread_ = rtc::Thread::Current();
audio_session_observer_ =
@@ -220,6 +226,7 @@
LOG(LS_INFO) << "Voice-Processing I/O audio unit is now started";
}
rtc::AtomicOps::ReleaseStore(&playing_, 1);
+ num_playout_callbacks_ = 0;
return 0;
}
@@ -234,6 +241,19 @@
audio_is_initialized_ = false;
}
rtc::AtomicOps::ReleaseStore(&playing_, 0);
+
+ // Derive average number of calls to OnGetPlayoutData() between detected
+ // audio glitches and add the result to a histogram.
+ int average_number_of_playout_callbacks_between_glitches = 100000;
+ if (num_detected_playout_glitches_ > 0) {
+ average_number_of_playout_callbacks_between_glitches =
+ num_playout_callbacks_ / num_detected_playout_glitches_;
+ }
+ RTC_HISTOGRAM_COUNTS_100000(
+ "WebRTC.Audio.AveragePlayoutCallbacksBetweenGlitches",
+ average_number_of_playout_callbacks_between_glitches);
+ RTCLog(@"Average number of playout callbacks between glitches: %d",
+ average_number_of_playout_callbacks_between_glitches);
return 0;
}
@@ -419,6 +439,7 @@
RTC_DCHECK_EQ(1, io_data->mNumberBuffers);
AudioBuffer* audio_buffer = &io_data->mBuffers[0];
RTC_DCHECK_EQ(1, audio_buffer->mNumberChannels);
+
// Get pointer to internal audio buffer to which new audio data shall be
// written.
const size_t size_in_bytes = audio_buffer->mDataByteSize;
@@ -433,10 +454,30 @@
return noErr;
}
+ // Measure time since last call to OnGetPlayoutData() and see if it is larger
+ // than a well defined threshold. If so, we have a clear indication of a
+ // glitch in the output audio since the core audio layer will most likely run
+ // dry in this state.
+ ++num_playout_callbacks_;
+ const int64_t now_time = rtc::TimeMillis();
+ if (time_stamp->mSampleTime != num_frames) {
+ const int64_t delta_time = now_time - last_playout_time_;
+ const int glitch_threshold =
+ 1.5 * playout_parameters_.GetBufferSizeInMilliseconds() - 1;
+ if (delta_time > glitch_threshold) {
+ RTCLogWarning(@"Playout audio glitch detected.\n"
+ " Time since last OnGetPlayoutData was %lld ms.",
+ delta_time);
+ thread_->Post(RTC_FROM_HERE, this, kMessageTypePlayoutGlitchDetected);
+ }
+ }
+ last_playout_time_ = now_time;
+
// Read decoded 16-bit PCM samples from WebRTC (using a size that matches
// the native I/O audio unit) and copy the result to the audio buffer in the
// |io_data| destination.
- fine_audio_buffer_->GetPlayoutData(rtc::ArrayView<int8_t>(destination, size_in_bytes));
+ fine_audio_buffer_->GetPlayoutData(
+ rtc::ArrayView<int8_t>(destination, size_in_bytes));
return noErr;
}
@@ -458,6 +499,9 @@
delete data;
break;
}
+ case kMessageTypePlayoutGlitchDetected:
+ HandlePlayoutGlitchDetected();
+ break;
}
}
@@ -530,8 +574,10 @@
" Session sample rate: %f frames_per_buffer: %lu\n"
" ADM sample rate: %f frames_per_buffer: %lu",
sample_rate,
- session_sample_rate, (unsigned long)session_frames_per_buffer,
- current_sample_rate, (unsigned long)current_frames_per_buffer);;
+ session_sample_rate,
+ (unsigned long)session_frames_per_buffer,
+ current_sample_rate,
+ (unsigned long)current_frames_per_buffer);
// Sample rate and buffer size are the same, no work to do.
if (std::abs(current_sample_rate - session_sample_rate) <= DBL_EPSILON &&
@@ -572,6 +618,13 @@
RTCLog(@"Successfully handled sample rate change.");
}
+void AudioDeviceIOS::HandlePlayoutGlitchDetected() {
+ RTC_DCHECK(thread_checker_.CalledOnValidThread());
+ num_detected_playout_glitches_++;
+ RTCLog(@"Number of detected playout glitches: %lld",
+ num_detected_playout_glitches_);
+}
+
void AudioDeviceIOS::UpdateAudioDeviceBuffer() {
LOGI() << "UpdateAudioDevicebuffer";
// AttachAudioBuffer() is called at construction by the main class but check
diff --git a/webrtc/sdk/objc/Framework/Classes/Audio/RTCAudioSessionConfiguration.m b/webrtc/sdk/objc/Framework/Classes/Audio/RTCAudioSessionConfiguration.m
index fe7b544..b95d30a 100644
--- a/webrtc/sdk/objc/Framework/Classes/Audio/RTCAudioSessionConfiguration.m
+++ b/webrtc/sdk/objc/Framework/Classes/Audio/RTCAudioSessionConfiguration.m
@@ -41,8 +41,8 @@
// ~10.6667ms or 512 audio frames per buffer. The FineAudioBuffer instance will
// take care of any buffering required to convert between native buffers and
// buffers used by WebRTC. It is beneficial for the performance if the native
-// size is as close to 10ms as possible since it results in "clean" callback
-// sequence without bursts of callbacks back to back.
+// size is as an even multiple of 10ms as possible since it results in "clean"
+// callback sequence without bursts of callbacks back to back.
const double kRTCAudioSessionHighPerformanceIOBufferDuration = 0.01;
// Use a larger buffer size on devices with only one core (e.g. iPhone 4).
diff --git a/webrtc/webrtc.gni b/webrtc/webrtc.gni
index d4acaf5..676ab2f 100644
--- a/webrtc/webrtc.gni
+++ b/webrtc/webrtc.gni
@@ -147,6 +147,12 @@
# use file-based audio playout and record.
rtc_use_dummy_audio_file_devices = false
+ # When set to true, replace the audio output with a sinus tone at 440Hz.
+ # The ADM will ask for audio data from WebRTC but instead of reading real
+ # audio samples from NetEQ, a sinus tone will be generated and replace the
+ # real audio samples.
+ rtc_audio_device_plays_sinus_tone = false
+
# When set to true, test targets will declare the files needed to run memcheck
# as data dependencies. This is to enable memcheck execution on swarming bots.
rtc_use_memcheck = false