rtc::Event::Wait(kForever): Print stack trace when we deadlock
After being stuck "forever" (3 seconds) waiting for an event to
trigger, log the stack trace of the current thread to aid debugging of
the deadlock.
Bug: webrtc:10308
Change-Id: I04852f191027294d7e7a7f5e63de4c6c7fdd6326
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/128342
Commit-Queue: Karl Wiberg <kwiberg@webrtc.org>
Reviewed-by: Magnus Jedvert <magjed@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#27263}diff --git a/rtc_base/BUILD.gn b/rtc_base/BUILD.gn
index 5882ade..35f1f61 100644
--- a/rtc_base/BUILD.gn
+++ b/rtc_base/BUILD.gn
@@ -247,23 +247,25 @@
}
rtc_source_set("rtc_event") {
- deps = [
- ":checks",
- ]
-
if (build_with_chromium) {
- # Dependency on chromium's waitable_event (in //base).
- deps += [ "//base" ]
sources = [
"../../webrtc_overrides/rtc_base/event.cc",
"../../webrtc_overrides/rtc_base/event.h",
]
+ deps = [
+ ":checks",
+ "//base", # Dependency on chromium's waitable_event.
+ ]
} else {
- deps += [ "synchronization:yield_policy" ]
sources = [
"event.cc",
"event.h",
]
+ deps = [
+ ":checks",
+ "synchronization:yield_policy",
+ "system:warn_current_thread_is_deadlocked",
+ ]
}
}
diff --git a/rtc_base/event.cc b/rtc_base/event.cc
index 71dca49..4b8e9ff 100644
--- a/rtc_base/event.cc
+++ b/rtc_base/event.cc
@@ -13,6 +13,7 @@
#if defined(WEBRTC_WIN)
#include <windows.h>
#elif defined(WEBRTC_POSIX)
+#include <errno.h>
#include <pthread.h>
#include <sys/time.h>
#include <time.h>
@@ -22,6 +23,7 @@
#include "rtc_base/checks.h"
#include "rtc_base/synchronization/yield_policy.h"
+#include "rtc_base/system/warn_current_thread_is_deadlocked.h"
namespace rtc {
@@ -103,45 +105,65 @@
pthread_mutex_unlock(&event_mutex_);
}
-bool Event::Wait(int milliseconds) {
- ScopedYieldPolicy::YieldExecution();
+namespace {
- int error = 0;
+timespec GetTimespec(const int milliseconds_from_now) {
+ timespec ts;
- struct timespec ts;
- if (milliseconds != kForever) {
+ // Get the current time.
#if USE_CLOCK_GETTIME
- clock_gettime(CLOCK_MONOTONIC, &ts);
+ clock_gettime(CLOCK_MONOTONIC, &ts);
#else
- struct timeval tv;
- gettimeofday(&tv, nullptr);
- ts.tv_sec = tv.tv_sec;
- ts.tv_nsec = tv.tv_usec * 1000;
+ timeval tv;
+ gettimeofday(&tv, nullptr);
+ ts.tv_sec = tv.tv_sec;
+ ts.tv_nsec = tv.tv_usec * 1000;
#endif
- ts.tv_sec += (milliseconds / 1000);
- ts.tv_nsec += (milliseconds % 1000) * 1000000;
+ // Add the specified number of milliseconds to it.
+ ts.tv_sec += (milliseconds_from_now / 1000);
+ ts.tv_nsec += (milliseconds_from_now % 1000) * 1000000;
- // Handle overflow.
- if (ts.tv_nsec >= 1000000000) {
- ts.tv_sec++;
- ts.tv_nsec -= 1000000000;
- }
+ // Normalize.
+ if (ts.tv_nsec >= 1000000000) {
+ ts.tv_sec++;
+ ts.tv_nsec -= 1000000000;
}
+ return ts;
+}
+
+} // namespace
+
+bool Event::Wait(const int milliseconds) {
+ // Set a timeout for the given number of milliseconds, or 3000 ms if the
+ // caller asked for kForever.
+ const timespec ts =
+ GetTimespec(milliseconds == kForever ? 3000 : milliseconds);
+
+ ScopedYieldPolicy::YieldExecution();
pthread_mutex_lock(&event_mutex_);
- if (milliseconds != kForever) {
- while (!event_status_ && error == 0) {
+
+ // Wait for the event to trigger or the timeout to expire, whichever comes
+ // first.
+ int error = 0;
+ while (!event_status_ && error == 0) {
#if USE_PTHREAD_COND_TIMEDWAIT_MONOTONIC_NP
- error =
- pthread_cond_timedwait_monotonic_np(&event_cond_, &event_mutex_, &ts);
+ error =
+ pthread_cond_timedwait_monotonic_np(&event_cond_, &event_mutex_, &ts);
#else
- error = pthread_cond_timedwait(&event_cond_, &event_mutex_, &ts);
+ error = pthread_cond_timedwait(&event_cond_, &event_mutex_, &ts);
#endif
- }
- } else {
- while (!event_status_ && error == 0)
+ }
+
+ if (milliseconds == kForever && error == ETIMEDOUT) {
+ // Our 3000 ms timeout expired, but the caller asked us to wait forever, so
+ // do that.
+ webrtc::WarnThatTheCurrentThreadIsProbablyDeadlocked();
+ error = 0;
+ while (!event_status_ && error == 0) {
error = pthread_cond_wait(&event_cond_, &event_mutex_);
+ }
}
// NOTE(liulk): Exactly one thread will auto-reset this event. All
diff --git a/rtc_base/system/BUILD.gn b/rtc_base/system/BUILD.gn
index 6448348..fb6ce50 100644
--- a/rtc_base/system/BUILD.gn
+++ b/rtc_base/system/BUILD.gn
@@ -94,3 +94,17 @@
]
}
}
+
+rtc_source_set("warn_current_thread_is_deadlocked") {
+ sources = [
+ "warn_current_thread_is_deadlocked.h",
+ ]
+ deps = []
+ if (is_android && !build_with_chromium) {
+ sources += [ "warn_current_thread_is_deadlocked.cc" ]
+ deps += [
+ "..:logging",
+ "../../sdk/android:native_api_stacktrace",
+ ]
+ }
+}
diff --git a/rtc_base/system/DEPS b/rtc_base/system/DEPS
index 09d6c93..39293d0 100644
--- a/rtc_base/system/DEPS
+++ b/rtc_base/system/DEPS
@@ -2,4 +2,7 @@
"thread_registry\.cc": [
"+sdk/android/native_api/stacktrace/stacktrace.h",
],
+ "warn_current_thread_is_deadlocked\.cc": [
+ "+sdk/android/native_api/stacktrace/stacktrace.h",
+ ],
}
diff --git a/rtc_base/system/warn_current_thread_is_deadlocked.cc b/rtc_base/system/warn_current_thread_is_deadlocked.cc
new file mode 100644
index 0000000..d39b040
--- /dev/null
+++ b/rtc_base/system/warn_current_thread_is_deadlocked.cc
@@ -0,0 +1,23 @@
+/*
+ * Copyright 2019 The WebRTC Project Authors. All rights reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "rtc_base/system/warn_current_thread_is_deadlocked.h"
+
+#include "rtc_base/logging.h"
+#include "sdk/android/native_api/stacktrace/stacktrace.h"
+
+namespace webrtc {
+
+void WarnThatTheCurrentThreadIsProbablyDeadlocked() {
+ RTC_LOG(LS_WARNING) << "Probable deadlock:";
+ RTC_LOG(LS_WARNING) << StackTraceToString(GetStackTrace());
+}
+
+} // namespace webrtc
diff --git a/rtc_base/system/warn_current_thread_is_deadlocked.h b/rtc_base/system/warn_current_thread_is_deadlocked.h
new file mode 100644
index 0000000..4a0ba9d
--- /dev/null
+++ b/rtc_base/system/warn_current_thread_is_deadlocked.h
@@ -0,0 +1,24 @@
+/*
+ * Copyright 2019 The WebRTC Project Authors. All rights reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef RTC_BASE_SYSTEM_WARN_CURRENT_THREAD_IS_DEADLOCKED_H_
+#define RTC_BASE_SYSTEM_WARN_CURRENT_THREAD_IS_DEADLOCKED_H_
+
+namespace webrtc {
+
+#if defined(WEBRTC_ANDROID) && !defined(WEBRTC_CHROMIUM_BUILD)
+void WarnThatTheCurrentThreadIsProbablyDeadlocked();
+#else
+inline void WarnThatTheCurrentThreadIsProbablyDeadlocked() {}
+#endif
+
+} // namespace webrtc
+
+#endif // RTC_BASE_SYSTEM_WARN_CURRENT_THREAD_IS_DEADLOCKED_H_