blob: af000956286c18addf92daeedc563aa26f99541d [file] [log] [blame]
Don Zickus58687ac2010-05-07 17:11:44 -04001/*
2 * Detect hard and soft lockups on a system
3 *
4 * started by Don Zickus, Copyright (C) 2010 Red Hat, Inc.
5 *
Fernando Luis Vázquez Cao86f5e6a2012-02-09 17:42:22 -05006 * Note: Most of this code is borrowed heavily from the original softlockup
7 * detector, so thanks to Ingo for the initial implementation.
8 * Some chunks also taken from the old x86-specific nmi watchdog code, thanks
Don Zickus58687ac2010-05-07 17:11:44 -04009 * to those contributors as well.
10 */
11
Kefeng Wang5f92a7b2017-07-14 14:49:46 -070012#define pr_fmt(fmt) "watchdog: " fmt
Andrew Morton45019802012-03-23 15:01:55 -070013
Don Zickus58687ac2010-05-07 17:11:44 -040014#include <linux/mm.h>
15#include <linux/cpu.h>
16#include <linux/nmi.h>
17#include <linux/init.h>
Don Zickus58687ac2010-05-07 17:11:44 -040018#include <linux/module.h>
19#include <linux/sysctl.h>
Thomas Gleixnerbcd951c2012-07-16 10:42:38 +000020#include <linux/smpboot.h>
Clark Williams8bd75c72013-02-07 09:47:07 -060021#include <linux/sched/rt.h>
Ingo Molnarae7e81c2017-02-01 18:07:51 +010022#include <uapi/linux/sched/types.h>
Chris Metcalffe4ba3c2015-06-24 16:55:45 -070023#include <linux/tick.h>
Tejun Heo82607adc2015-12-08 11:28:04 -050024#include <linux/workqueue.h>
Ingo Molnare6017572017-02-01 16:36:40 +010025#include <linux/sched/clock.h>
Ingo Molnarb17b0152017-02-08 18:51:35 +010026#include <linux/sched/debug.h>
Don Zickus58687ac2010-05-07 17:11:44 -040027
28#include <asm/irq_regs.h>
Eric B Munson5d1c0f42012-03-10 14:37:28 -050029#include <linux/kvm_para.h>
Ulrich Obergfell81a4bee2015-09-04 15:45:15 -070030#include <linux/kthread.h>
Don Zickus58687ac2010-05-07 17:11:44 -040031
Thomas Gleixner946d1972017-09-12 21:37:01 +020032static DEFINE_MUTEX(watchdog_mutex);
Peter Zijlstraab992dc2015-05-18 11:31:50 +020033
Nicholas Piggin05a4a952017-07-12 14:35:46 -070034int __read_mostly nmi_watchdog_enabled;
35
36#if defined(CONFIG_HARDLOCKUP_DETECTOR) || defined(CONFIG_HAVE_NMI_WATCHDOG)
37unsigned long __read_mostly watchdog_enabled = SOFT_WATCHDOG_ENABLED |
38 NMI_WATCHDOG_ENABLED;
Ulrich Obergfell84d56e62015-04-14 15:43:55 -070039#else
Babu Moger249e52e2016-12-14 15:06:21 -080040unsigned long __read_mostly watchdog_enabled = SOFT_WATCHDOG_ENABLED;
Ulrich Obergfell84d56e62015-04-14 15:43:55 -070041#endif
Nicholas Piggin05a4a952017-07-12 14:35:46 -070042
43#ifdef CONFIG_HARDLOCKUP_DETECTOR
44/* boot commands */
45/*
46 * Should we panic when a soft-lockup or hard-lockup occurs:
47 */
48unsigned int __read_mostly hardlockup_panic =
49 CONFIG_BOOTPARAM_HARDLOCKUP_PANIC_VALUE;
50/*
51 * We may not want to enable hard lockup detection by default in all cases,
52 * for example when running the kernel as a guest on a hypervisor. In these
53 * cases this function can be called to disable hard lockup detection. This
54 * function should only be executed once by the boot processor before the
55 * kernel command line parameters are parsed, because otherwise it is not
56 * possible to override this in hardlockup_panic_setup().
57 */
Thomas Gleixner7a355822017-09-12 21:37:02 +020058void __init hardlockup_detector_disable(void)
Nicholas Piggin05a4a952017-07-12 14:35:46 -070059{
60 watchdog_enabled &= ~NMI_WATCHDOG_ENABLED;
61}
62
63static int __init hardlockup_panic_setup(char *str)
64{
65 if (!strncmp(str, "panic", 5))
66 hardlockup_panic = 1;
67 else if (!strncmp(str, "nopanic", 7))
68 hardlockup_panic = 0;
69 else if (!strncmp(str, "0", 1))
70 watchdog_enabled &= ~NMI_WATCHDOG_ENABLED;
71 else if (!strncmp(str, "1", 1))
72 watchdog_enabled |= NMI_WATCHDOG_ENABLED;
73 return 1;
74}
75__setup("nmi_watchdog=", hardlockup_panic_setup);
76
77#endif
78
79#ifdef CONFIG_SOFTLOCKUP_DETECTOR
Ulrich Obergfell84d56e62015-04-14 15:43:55 -070080int __read_mostly soft_watchdog_enabled;
Nicholas Piggin05a4a952017-07-12 14:35:46 -070081#endif
82
Ulrich Obergfell84d56e62015-04-14 15:43:55 -070083int __read_mostly watchdog_user_enabled;
Mandeep Singh Baines4eec42f2011-05-22 22:10:23 -070084int __read_mostly watchdog_thresh = 10;
Ulrich Obergfell84d56e62015-04-14 15:43:55 -070085
Aaron Tomlined235872014-06-23 13:22:05 -070086#ifdef CONFIG_SMP
87int __read_mostly sysctl_softlockup_all_cpu_backtrace;
Jiri Kosina55537872015-11-05 18:44:41 -080088int __read_mostly sysctl_hardlockup_all_cpu_backtrace;
Aaron Tomlined235872014-06-23 13:22:05 -070089#endif
Nicholas Piggin05a4a952017-07-12 14:35:46 -070090struct cpumask watchdog_cpumask __read_mostly;
Chris Metcalffe4ba3c2015-06-24 16:55:45 -070091unsigned long *watchdog_cpumask_bits = cpumask_bits(&watchdog_cpumask);
92
Ulrich Obergfellec6a9062015-09-04 15:45:28 -070093/*
94 * The 'watchdog_running' variable is set to 1 when the watchdog threads
95 * are registered/started and is set to 0 when the watchdog threads are
96 * unregistered/stopped, so it is an indicator whether the threads exist.
97 */
Frederic Weisbecker3c00ea82013-05-19 20:45:15 +020098static int __read_mostly watchdog_running;
Nicholas Piggin05a4a952017-07-12 14:35:46 -070099
100/*
101 * These functions can be overridden if an architecture implements its
102 * own hardlockup detector.
Nicholas Piggina10a8422017-07-12 14:35:49 -0700103 *
104 * watchdog_nmi_enable/disable can be implemented to start and stop when
105 * softlockup watchdog threads start and stop. The arch must select the
106 * SOFTLOCKUP_DETECTOR Kconfig.
Nicholas Piggin05a4a952017-07-12 14:35:46 -0700107 */
108int __weak watchdog_nmi_enable(unsigned int cpu)
109{
110 return 0;
111}
112void __weak watchdog_nmi_disable(unsigned int cpu)
113{
114}
115
Nicholas Piggina10a8422017-07-12 14:35:49 -0700116/*
117 * watchdog_nmi_reconfigure can be implemented to be notified after any
118 * watchdog configuration change. The arch hardlockup watchdog should
119 * respond to the following variables:
120 * - nmi_watchdog_enabled
121 * - watchdog_thresh
122 * - watchdog_cpumask
123 * - sysctl_hardlockup_all_cpu_backtrace
124 * - hardlockup_panic
Nicholas Piggina10a8422017-07-12 14:35:49 -0700125 */
126void __weak watchdog_nmi_reconfigure(void)
127{
128}
129
130
Nicholas Piggin05a4a952017-07-12 14:35:46 -0700131#ifdef CONFIG_SOFTLOCKUP_DETECTOR
132
133/* Helper for online, unparked cpus. */
134#define for_each_watchdog_cpu(cpu) \
135 for_each_cpu_and((cpu), cpu_online_mask, &watchdog_cpumask)
136
137atomic_t watchdog_park_in_progress = ATOMIC_INIT(0);
Ulrich Obergfellec6a9062015-09-04 15:45:28 -0700138
Chuansheng Liu0f34c402012-12-17 15:59:50 -0800139static u64 __read_mostly sample_period;
Don Zickus58687ac2010-05-07 17:11:44 -0400140
141static DEFINE_PER_CPU(unsigned long, watchdog_touch_ts);
142static DEFINE_PER_CPU(struct task_struct *, softlockup_watchdog);
143static DEFINE_PER_CPU(struct hrtimer, watchdog_hrtimer);
144static DEFINE_PER_CPU(bool, softlockup_touch_sync);
Don Zickus58687ac2010-05-07 17:11:44 -0400145static DEFINE_PER_CPU(bool, soft_watchdog_warn);
Thomas Gleixnerbcd951c2012-07-16 10:42:38 +0000146static DEFINE_PER_CPU(unsigned long, hrtimer_interrupts);
147static DEFINE_PER_CPU(unsigned long, soft_lockup_hrtimer_cnt);
chai wenb1a8de12014-10-09 15:25:17 -0700148static DEFINE_PER_CPU(struct task_struct *, softlockup_task_ptr_saved);
Don Zickus58687ac2010-05-07 17:11:44 -0400149static DEFINE_PER_CPU(unsigned long, hrtimer_interrupts_saved);
Aaron Tomlined235872014-06-23 13:22:05 -0700150static unsigned long soft_lockup_nmi_warn;
Don Zickus58687ac2010-05-07 17:11:44 -0400151
Don Zickus58687ac2010-05-07 17:11:44 -0400152unsigned int __read_mostly softlockup_panic =
153 CONFIG_BOOTPARAM_SOFTLOCKUP_PANIC_VALUE;
154
155static int __init softlockup_panic_setup(char *str)
156{
157 softlockup_panic = simple_strtoul(str, NULL, 0);
158
159 return 1;
160}
161__setup("softlockup_panic=", softlockup_panic_setup);
162
163static int __init nowatchdog_setup(char *str)
164{
Ulrich Obergfell195daf62015-04-14 15:44:13 -0700165 watchdog_enabled = 0;
Don Zickus58687ac2010-05-07 17:11:44 -0400166 return 1;
167}
168__setup("nowatchdog", nowatchdog_setup);
169
Don Zickus58687ac2010-05-07 17:11:44 -0400170static int __init nosoftlockup_setup(char *str)
171{
Ulrich Obergfell195daf62015-04-14 15:44:13 -0700172 watchdog_enabled &= ~SOFT_WATCHDOG_ENABLED;
Don Zickus58687ac2010-05-07 17:11:44 -0400173 return 1;
174}
175__setup("nosoftlockup", nosoftlockup_setup);
Ulrich Obergfell195daf62015-04-14 15:44:13 -0700176
Aaron Tomlined235872014-06-23 13:22:05 -0700177#ifdef CONFIG_SMP
178static int __init softlockup_all_cpu_backtrace_setup(char *str)
179{
180 sysctl_softlockup_all_cpu_backtrace =
181 !!simple_strtol(str, NULL, 0);
182 return 1;
183}
184__setup("softlockup_all_cpu_backtrace=", softlockup_all_cpu_backtrace_setup);
Nicholas Piggin05a4a952017-07-12 14:35:46 -0700185#ifdef CONFIG_HARDLOCKUP_DETECTOR
Jiri Kosina55537872015-11-05 18:44:41 -0800186static int __init hardlockup_all_cpu_backtrace_setup(char *str)
187{
188 sysctl_hardlockup_all_cpu_backtrace =
189 !!simple_strtol(str, NULL, 0);
190 return 1;
191}
192__setup("hardlockup_all_cpu_backtrace=", hardlockup_all_cpu_backtrace_setup);
Aaron Tomlined235872014-06-23 13:22:05 -0700193#endif
Nicholas Piggin05a4a952017-07-12 14:35:46 -0700194#endif
Don Zickus58687ac2010-05-07 17:11:44 -0400195
Mandeep Singh Baines4eec42f2011-05-22 22:10:23 -0700196/*
197 * Hard-lockup warnings should be triggered after just a few seconds. Soft-
198 * lockups can have false positives under extreme conditions. So we generally
199 * want a higher threshold for soft lockups than for hard lockups. So we couple
200 * the thresholds with a factor: we make the soft threshold twice the amount of
201 * time the hard threshold is.
202 */
Ingo Molnar6e9101a2011-05-24 05:43:18 +0200203static int get_softlockup_thresh(void)
Mandeep Singh Baines4eec42f2011-05-22 22:10:23 -0700204{
205 return watchdog_thresh * 2;
206}
Don Zickus58687ac2010-05-07 17:11:44 -0400207
208/*
209 * Returns seconds, approximately. We don't need nanosecond
210 * resolution, and we don't need to waste time with a big divide when
211 * 2^30ns == 1.074s.
212 */
Namhyung Kimc06b4f12012-12-27 11:49:44 +0900213static unsigned long get_timestamp(void)
Don Zickus58687ac2010-05-07 17:11:44 -0400214{
Cyril Bur545a2bf2015-02-12 15:01:24 -0800215 return running_clock() >> 30LL; /* 2^30 ~= 10^9 */
Don Zickus58687ac2010-05-07 17:11:44 -0400216}
217
Chuansheng Liu0f34c402012-12-17 15:59:50 -0800218static void set_sample_period(void)
Don Zickus58687ac2010-05-07 17:11:44 -0400219{
220 /*
Mandeep Singh Baines586692a2011-05-22 22:10:22 -0700221 * convert watchdog_thresh from seconds to ns
Fernando Luis Vázquez Cao86f5e6a2012-02-09 17:42:22 -0500222 * the divide by 5 is to give hrtimer several chances (two
223 * or three with the current relation between the soft
224 * and hard thresholds) to increment before the
225 * hardlockup detector generates a warning
Don Zickus58687ac2010-05-07 17:11:44 -0400226 */
Chuansheng Liu0f34c402012-12-17 15:59:50 -0800227 sample_period = get_softlockup_thresh() * ((u64)NSEC_PER_SEC / 5);
Thomas Gleixner7edaeb62017-08-15 09:50:13 +0200228 watchdog_update_hrtimer_threshold(sample_period);
Don Zickus58687ac2010-05-07 17:11:44 -0400229}
230
231/* Commands for resetting the watchdog */
232static void __touch_watchdog(void)
233{
Namhyung Kimc06b4f12012-12-27 11:49:44 +0900234 __this_cpu_write(watchdog_touch_ts, get_timestamp());
Don Zickus58687ac2010-05-07 17:11:44 -0400235}
236
Tejun Heo03e0d462015-12-08 11:28:04 -0500237/**
238 * touch_softlockup_watchdog_sched - touch watchdog on scheduler stalls
239 *
240 * Call when the scheduler may have stalled for legitimate reasons
241 * preventing the watchdog task from executing - e.g. the scheduler
242 * entering idle state. This should only be used for scheduler events.
243 * Use touch_softlockup_watchdog() for everything else.
244 */
245void touch_softlockup_watchdog_sched(void)
Don Zickus58687ac2010-05-07 17:11:44 -0400246{
Andrew Morton78611442014-04-18 15:07:12 -0700247 /*
248 * Preemption can be enabled. It doesn't matter which CPU's timestamp
249 * gets zeroed here, so use the raw_ operation.
250 */
251 raw_cpu_write(watchdog_touch_ts, 0);
Don Zickus58687ac2010-05-07 17:11:44 -0400252}
Tejun Heo03e0d462015-12-08 11:28:04 -0500253
254void touch_softlockup_watchdog(void)
255{
256 touch_softlockup_watchdog_sched();
Tejun Heo82607adc2015-12-08 11:28:04 -0500257 wq_watchdog_touch(raw_smp_processor_id());
Tejun Heo03e0d462015-12-08 11:28:04 -0500258}
Ingo Molnar0167c782010-05-13 08:53:33 +0200259EXPORT_SYMBOL(touch_softlockup_watchdog);
Don Zickus58687ac2010-05-07 17:11:44 -0400260
Don Zickus332fbdb2010-05-07 17:11:45 -0400261void touch_all_softlockup_watchdogs(void)
Don Zickus58687ac2010-05-07 17:11:44 -0400262{
263 int cpu;
264
265 /*
266 * this is done lockless
267 * do we care if a 0 races with a timestamp?
268 * all it means is the softlock check starts one cycle later
269 */
Chris Metcalffe4ba3c2015-06-24 16:55:45 -0700270 for_each_watchdog_cpu(cpu)
Don Zickus58687ac2010-05-07 17:11:44 -0400271 per_cpu(watchdog_touch_ts, cpu) = 0;
Tejun Heo82607adc2015-12-08 11:28:04 -0500272 wq_watchdog_touch(-1);
Don Zickus58687ac2010-05-07 17:11:44 -0400273}
274
Don Zickus58687ac2010-05-07 17:11:44 -0400275void touch_softlockup_watchdog_sync(void)
276{
Christoph Lameterf7f66b02014-08-17 12:30:34 -0500277 __this_cpu_write(softlockup_touch_sync, true);
278 __this_cpu_write(watchdog_touch_ts, 0);
Don Zickus58687ac2010-05-07 17:11:44 -0400279}
280
Don Zickus26e09c62010-05-17 18:06:04 -0400281static int is_softlockup(unsigned long touch_ts)
Don Zickus58687ac2010-05-07 17:11:44 -0400282{
Namhyung Kimc06b4f12012-12-27 11:49:44 +0900283 unsigned long now = get_timestamp();
Don Zickus58687ac2010-05-07 17:11:44 -0400284
Ulrich Obergfell39d2da22015-11-05 18:44:56 -0800285 if ((watchdog_enabled & SOFT_WATCHDOG_ENABLED) && watchdog_thresh){
Ulrich Obergfell195daf62015-04-14 15:44:13 -0700286 /* Warn about unreasonable delays. */
287 if (time_after(now, touch_ts + get_softlockup_thresh()))
288 return now - touch_ts;
289 }
Don Zickus58687ac2010-05-07 17:11:44 -0400290 return 0;
291}
292
Nicholas Piggin05a4a952017-07-12 14:35:46 -0700293/* watchdog detector functions */
294bool is_hardlockup(void)
295{
296 unsigned long hrint = __this_cpu_read(hrtimer_interrupts);
297
298 if (__this_cpu_read(hrtimer_interrupts_saved) == hrint)
299 return true;
300
301 __this_cpu_write(hrtimer_interrupts_saved, hrint);
302 return false;
303}
304
Don Zickus58687ac2010-05-07 17:11:44 -0400305static void watchdog_interrupt_count(void)
306{
Christoph Lameter909ea962010-12-08 16:22:55 +0100307 __this_cpu_inc(hrtimer_interrupts);
Don Zickus58687ac2010-05-07 17:11:44 -0400308}
Thomas Gleixnerbcd951c2012-07-16 10:42:38 +0000309
Ulrich Obergfell58cf6902015-11-05 18:44:30 -0800310static int watchdog_enable_all_cpus(void);
311static void watchdog_disable_all_cpus(void);
312
Don Zickus58687ac2010-05-07 17:11:44 -0400313/* watchdog kicker functions */
314static enum hrtimer_restart watchdog_timer_fn(struct hrtimer *hrtimer)
315{
Christoph Lameter909ea962010-12-08 16:22:55 +0100316 unsigned long touch_ts = __this_cpu_read(watchdog_touch_ts);
Don Zickus58687ac2010-05-07 17:11:44 -0400317 struct pt_regs *regs = get_irq_regs();
318 int duration;
Aaron Tomlined235872014-06-23 13:22:05 -0700319 int softlockup_all_cpu_backtrace = sysctl_softlockup_all_cpu_backtrace;
Don Zickus58687ac2010-05-07 17:11:44 -0400320
Thomas Gleixner6554fd82017-09-12 21:36:57 +0200321 if (!watchdog_enabled ||
322 atomic_read(&watchdog_park_in_progress) != 0)
Don Zickusb94f5112017-01-24 15:17:53 -0800323 return HRTIMER_NORESTART;
324
Don Zickus58687ac2010-05-07 17:11:44 -0400325 /* kick the hardlockup detector */
326 watchdog_interrupt_count();
327
328 /* kick the softlockup detector */
Christoph Lameter909ea962010-12-08 16:22:55 +0100329 wake_up_process(__this_cpu_read(softlockup_watchdog));
Don Zickus58687ac2010-05-07 17:11:44 -0400330
331 /* .. and repeat */
Chuansheng Liu0f34c402012-12-17 15:59:50 -0800332 hrtimer_forward_now(hrtimer, ns_to_ktime(sample_period));
Don Zickus58687ac2010-05-07 17:11:44 -0400333
334 if (touch_ts == 0) {
Christoph Lameter909ea962010-12-08 16:22:55 +0100335 if (unlikely(__this_cpu_read(softlockup_touch_sync))) {
Don Zickus58687ac2010-05-07 17:11:44 -0400336 /*
337 * If the time stamp was touched atomically
338 * make sure the scheduler tick is up to date.
339 */
Christoph Lameter909ea962010-12-08 16:22:55 +0100340 __this_cpu_write(softlockup_touch_sync, false);
Don Zickus58687ac2010-05-07 17:11:44 -0400341 sched_clock_tick();
342 }
Eric B Munson5d1c0f42012-03-10 14:37:28 -0500343
344 /* Clear the guest paused flag on watchdog reset */
345 kvm_check_and_clear_guest_paused();
Don Zickus58687ac2010-05-07 17:11:44 -0400346 __touch_watchdog();
347 return HRTIMER_RESTART;
348 }
349
350 /* check for a softlockup
351 * This is done by making sure a high priority task is
352 * being scheduled. The task touches the watchdog to
353 * indicate it is getting cpu time. If it hasn't then
354 * this is a good indication some task is hogging the cpu
355 */
Don Zickus26e09c62010-05-17 18:06:04 -0400356 duration = is_softlockup(touch_ts);
Don Zickus58687ac2010-05-07 17:11:44 -0400357 if (unlikely(duration)) {
Eric B Munson5d1c0f42012-03-10 14:37:28 -0500358 /*
359 * If a virtual machine is stopped by the host it can look to
360 * the watchdog like a soft lockup, check to see if the host
361 * stopped the vm before we issue the warning
362 */
363 if (kvm_check_and_clear_guest_paused())
364 return HRTIMER_RESTART;
365
Don Zickus58687ac2010-05-07 17:11:44 -0400366 /* only warn once */
chai wenb1a8de12014-10-09 15:25:17 -0700367 if (__this_cpu_read(soft_watchdog_warn) == true) {
368 /*
369 * When multiple processes are causing softlockups the
370 * softlockup detector only warns on the first one
371 * because the code relies on a full quiet cycle to
372 * re-arm. The second process prevents the quiet cycle
373 * and never gets reported. Use task pointers to detect
374 * this.
375 */
376 if (__this_cpu_read(softlockup_task_ptr_saved) !=
377 current) {
378 __this_cpu_write(soft_watchdog_warn, false);
379 __touch_watchdog();
380 }
Don Zickus58687ac2010-05-07 17:11:44 -0400381 return HRTIMER_RESTART;
chai wenb1a8de12014-10-09 15:25:17 -0700382 }
Don Zickus58687ac2010-05-07 17:11:44 -0400383
Aaron Tomlined235872014-06-23 13:22:05 -0700384 if (softlockup_all_cpu_backtrace) {
385 /* Prevent multiple soft-lockup reports if one cpu is already
386 * engaged in dumping cpu back traces
387 */
388 if (test_and_set_bit(0, &soft_lockup_nmi_warn)) {
389 /* Someone else will report us. Let's give up */
390 __this_cpu_write(soft_watchdog_warn, true);
391 return HRTIMER_RESTART;
392 }
393 }
394
Fabian Frederick656c3b72014-08-06 16:04:03 -0700395 pr_emerg("BUG: soft lockup - CPU#%d stuck for %us! [%s:%d]\n",
Don Zickus26e09c62010-05-17 18:06:04 -0400396 smp_processor_id(), duration,
Don Zickus58687ac2010-05-07 17:11:44 -0400397 current->comm, task_pid_nr(current));
chai wenb1a8de12014-10-09 15:25:17 -0700398 __this_cpu_write(softlockup_task_ptr_saved, current);
Don Zickus58687ac2010-05-07 17:11:44 -0400399 print_modules();
400 print_irqtrace_events(current);
401 if (regs)
402 show_regs(regs);
403 else
404 dump_stack();
405
Aaron Tomlined235872014-06-23 13:22:05 -0700406 if (softlockup_all_cpu_backtrace) {
407 /* Avoid generating two back traces for current
408 * given that one is already made above
409 */
410 trigger_allbutself_cpu_backtrace();
411
412 clear_bit(0, &soft_lockup_nmi_warn);
413 /* Barrier to sync with other cpus */
414 smp_mb__after_atomic();
415 }
416
Josh Hunt69361ee2014-08-08 14:22:31 -0700417 add_taint(TAINT_SOFTLOCKUP, LOCKDEP_STILL_OK);
Don Zickus58687ac2010-05-07 17:11:44 -0400418 if (softlockup_panic)
419 panic("softlockup: hung tasks");
Christoph Lameter909ea962010-12-08 16:22:55 +0100420 __this_cpu_write(soft_watchdog_warn, true);
Don Zickus58687ac2010-05-07 17:11:44 -0400421 } else
Christoph Lameter909ea962010-12-08 16:22:55 +0100422 __this_cpu_write(soft_watchdog_warn, false);
Don Zickus58687ac2010-05-07 17:11:44 -0400423
424 return HRTIMER_RESTART;
425}
426
Thomas Gleixnerbcd951c2012-07-16 10:42:38 +0000427static void watchdog_set_prio(unsigned int policy, unsigned int prio)
Don Zickus58687ac2010-05-07 17:11:44 -0400428{
Thomas Gleixnerbcd951c2012-07-16 10:42:38 +0000429 struct sched_param param = { .sched_priority = prio };
430
431 sched_setscheduler(current, policy, &param);
432}
433
434static void watchdog_enable(unsigned int cpu)
435{
Christoph Lameterf7f66b02014-08-17 12:30:34 -0500436 struct hrtimer *hrtimer = raw_cpu_ptr(&watchdog_hrtimer);
Don Zickus58687ac2010-05-07 17:11:44 -0400437
Bjørn Mork3935e8952012-12-19 20:51:31 +0100438 /* kick off the timer for the hardlockup detector */
439 hrtimer_init(hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
440 hrtimer->function = watchdog_timer_fn;
441
Thomas Gleixnerbcd951c2012-07-16 10:42:38 +0000442 /* Enable the perf event */
443 watchdog_nmi_enable(cpu);
Don Zickus58687ac2010-05-07 17:11:44 -0400444
Don Zickus58687ac2010-05-07 17:11:44 -0400445 /* done here because hrtimer_start can only pin to smp_processor_id() */
Chuansheng Liu0f34c402012-12-17 15:59:50 -0800446 hrtimer_start(hrtimer, ns_to_ktime(sample_period),
Don Zickus58687ac2010-05-07 17:11:44 -0400447 HRTIMER_MODE_REL_PINNED);
448
Thomas Gleixnerbcd951c2012-07-16 10:42:38 +0000449 /* initialize timestamp */
450 watchdog_set_prio(SCHED_FIFO, MAX_RT_PRIO - 1);
451 __touch_watchdog();
Don Zickus58687ac2010-05-07 17:11:44 -0400452}
453
Thomas Gleixnerbcd951c2012-07-16 10:42:38 +0000454static void watchdog_disable(unsigned int cpu)
455{
Christoph Lameterf7f66b02014-08-17 12:30:34 -0500456 struct hrtimer *hrtimer = raw_cpu_ptr(&watchdog_hrtimer);
Thomas Gleixnerbcd951c2012-07-16 10:42:38 +0000457
458 watchdog_set_prio(SCHED_NORMAL, 0);
459 hrtimer_cancel(hrtimer);
460 /* disable the perf event */
461 watchdog_nmi_disable(cpu);
462}
463
Frederic Weisbeckerb8900bc2013-06-06 15:42:53 +0200464static void watchdog_cleanup(unsigned int cpu, bool online)
465{
466 watchdog_disable(cpu);
467}
468
Thomas Gleixnerbcd951c2012-07-16 10:42:38 +0000469static int watchdog_should_run(unsigned int cpu)
470{
471 return __this_cpu_read(hrtimer_interrupts) !=
472 __this_cpu_read(soft_lockup_hrtimer_cnt);
473}
474
475/*
476 * The watchdog thread function - touches the timestamp.
477 *
Chuansheng Liu0f34c402012-12-17 15:59:50 -0800478 * It only runs once every sample_period seconds (4 seconds by
Thomas Gleixnerbcd951c2012-07-16 10:42:38 +0000479 * default) to reset the softlockup timestamp. If this gets delayed
480 * for more than 2*watchdog_thresh seconds then the debug-printout
481 * triggers in watchdog_timer_fn().
482 */
483static void watchdog(unsigned int cpu)
484{
485 __this_cpu_write(soft_lockup_hrtimer_cnt,
486 __this_cpu_read(hrtimer_interrupts));
487 __touch_watchdog();
488}
Don Zickus58687ac2010-05-07 17:11:44 -0400489
Frederic Weisbeckerb8900bc2013-06-06 15:42:53 +0200490static struct smp_hotplug_thread watchdog_threads = {
491 .store = &softlockup_watchdog,
492 .thread_should_run = watchdog_should_run,
493 .thread_fn = watchdog,
494 .thread_comm = "watchdog/%u",
495 .setup = watchdog_enable,
496 .cleanup = watchdog_cleanup,
497 .park = watchdog_disable,
498 .unpark = watchdog_enable,
499};
500
Ulrich Obergfell81a4bee2015-09-04 15:45:15 -0700501/*
502 * park all watchdog threads that are specified in 'watchdog_cpumask'
Ulrich Obergfellee7fed52015-11-05 18:44:39 -0800503 *
504 * This function returns an error if kthread_park() of a watchdog thread
505 * fails. In this situation, the watchdog threads of some CPUs can already
506 * be parked and the watchdog threads of other CPUs can still be runnable.
507 * Callers are expected to handle this special condition as appropriate in
508 * their context.
Ulrich Obergfella2a45b82015-11-05 18:44:53 -0800509 *
510 * This function may only be called in a context that is protected against
511 * races with CPU hotplug - for example, via get_online_cpus().
Ulrich Obergfell81a4bee2015-09-04 15:45:15 -0700512 */
513static int watchdog_park_threads(void)
514{
515 int cpu, ret = 0;
516
Don Zickusb94f5112017-01-24 15:17:53 -0800517 atomic_set(&watchdog_park_in_progress, 1);
518
Ulrich Obergfell81a4bee2015-09-04 15:45:15 -0700519 for_each_watchdog_cpu(cpu) {
520 ret = kthread_park(per_cpu(softlockup_watchdog, cpu));
521 if (ret)
522 break;
523 }
Ulrich Obergfell81a4bee2015-09-04 15:45:15 -0700524
Don Zickusb94f5112017-01-24 15:17:53 -0800525 atomic_set(&watchdog_park_in_progress, 0);
526
Ulrich Obergfell81a4bee2015-09-04 15:45:15 -0700527 return ret;
528}
529
530/*
531 * unpark all watchdog threads that are specified in 'watchdog_cpumask'
Ulrich Obergfella2a45b82015-11-05 18:44:53 -0800532 *
533 * This function may only be called in a context that is protected against
534 * races with CPU hotplug - for example, via get_online_cpus().
Ulrich Obergfell81a4bee2015-09-04 15:45:15 -0700535 */
536static void watchdog_unpark_threads(void)
537{
538 int cpu;
539
Ulrich Obergfell81a4bee2015-09-04 15:45:15 -0700540 for_each_watchdog_cpu(cpu)
541 kthread_unpark(per_cpu(softlockup_watchdog, cpu));
Ulrich Obergfell81a4bee2015-09-04 15:45:15 -0700542}
543
Ulrich Obergfellb43cb432015-11-05 18:44:33 -0800544static int update_watchdog_all_cpus(void)
Michal Hocko9809b182013-09-24 15:27:30 -0700545{
Ulrich Obergfellb43cb432015-11-05 18:44:33 -0800546 int ret;
547
548 ret = watchdog_park_threads();
549 if (ret)
550 return ret;
551
Ulrich Obergfelld4bdd0b212015-09-04 15:45:21 -0700552 watchdog_unpark_threads();
Ulrich Obergfellb43cb432015-11-05 18:44:33 -0800553
554 return 0;
Michal Hocko9809b182013-09-24 15:27:30 -0700555}
556
Ulrich Obergfellb2f57c32015-04-14 15:44:16 -0700557static int watchdog_enable_all_cpus(void)
Frederic Weisbeckerb8900bc2013-06-06 15:42:53 +0200558{
559 int err = 0;
560
Frederic Weisbecker3c00ea82013-05-19 20:45:15 +0200561 if (!watchdog_running) {
Frederic Weisbecker230ec932015-09-04 15:45:06 -0700562 err = smpboot_register_percpu_thread_cpumask(&watchdog_threads,
563 &watchdog_cpumask);
Frederic Weisbeckerb8900bc2013-06-06 15:42:53 +0200564 if (err)
565 pr_err("Failed to create watchdog threads, disabled\n");
Frederic Weisbecker230ec932015-09-04 15:45:06 -0700566 else
Frederic Weisbecker3c00ea82013-05-19 20:45:15 +0200567 watchdog_running = 1;
Ulrich Obergfellb2f57c32015-04-14 15:44:16 -0700568 } else {
569 /*
570 * Enable/disable the lockup detectors or
571 * change the sample period 'on the fly'.
572 */
Ulrich Obergfellb43cb432015-11-05 18:44:33 -0800573 err = update_watchdog_all_cpus();
574
575 if (err) {
576 watchdog_disable_all_cpus();
577 pr_err("Failed to update lockup detectors, disabled\n");
578 }
Frederic Weisbeckerb8900bc2013-06-06 15:42:53 +0200579 }
580
Ulrich Obergfellb43cb432015-11-05 18:44:33 -0800581 if (err)
582 watchdog_enabled = 0;
583
Frederic Weisbeckerb8900bc2013-06-06 15:42:53 +0200584 return err;
585}
586
Don Zickus58687ac2010-05-07 17:11:44 -0400587static void watchdog_disable_all_cpus(void)
588{
Frederic Weisbecker3c00ea82013-05-19 20:45:15 +0200589 if (watchdog_running) {
590 watchdog_running = 0;
Frederic Weisbeckerb8900bc2013-06-06 15:42:53 +0200591 smpboot_unregister_percpu_thread(&watchdog_threads);
Thomas Gleixnerbcd951c2012-07-16 10:42:38 +0000592 }
Don Zickus58687ac2010-05-07 17:11:44 -0400593}
594
Nicholas Piggina10a8422017-07-12 14:35:49 -0700595#ifdef CONFIG_SYSCTL
596static int watchdog_update_cpus(void)
597{
598 return smpboot_update_cpumask_percpu_thread(
599 &watchdog_threads, &watchdog_cpumask);
600}
601#endif
602
Nicholas Piggin05a4a952017-07-12 14:35:46 -0700603#else /* SOFTLOCKUP */
604static int watchdog_park_threads(void)
605{
606 return 0;
607}
608
609static void watchdog_unpark_threads(void)
610{
611}
612
613static int watchdog_enable_all_cpus(void)
614{
615 return 0;
616}
617
618static void watchdog_disable_all_cpus(void)
619{
620}
621
Nicholas Piggina10a8422017-07-12 14:35:49 -0700622#ifdef CONFIG_SYSCTL
623static int watchdog_update_cpus(void)
624{
625 return 0;
626}
627#endif
628
Nicholas Piggin05a4a952017-07-12 14:35:46 -0700629static void set_sample_period(void)
630{
631}
632#endif /* SOFTLOCKUP */
633
Thomas Gleixner6554fd82017-09-12 21:36:57 +0200634/**
635 * lockup_detector_soft_poweroff - Interface to stop lockup detector(s)
636 *
637 * Special interface for parisc. It prevents lockup detector warnings from
638 * the default pm_poweroff() function which busy loops forever.
639 */
640void lockup_detector_soft_poweroff(void)
641{
642 watchdog_enabled = 0;
643}
644
Ulrich Obergfell58cf6902015-11-05 18:44:30 -0800645#ifdef CONFIG_SYSCTL
646
Don Zickus58687ac2010-05-07 17:11:44 -0400647/*
Ulrich Obergfella0c9cbb2015-04-14 15:43:58 -0700648 * Update the run state of the lockup detectors.
Don Zickus58687ac2010-05-07 17:11:44 -0400649 */
Ulrich Obergfella0c9cbb2015-04-14 15:43:58 -0700650static int proc_watchdog_update(void)
Don Zickus58687ac2010-05-07 17:11:44 -0400651{
Ulrich Obergfella0c9cbb2015-04-14 15:43:58 -0700652 int err = 0;
653
654 /*
655 * Watchdog threads won't be started if they are already active.
656 * The 'watchdog_running' variable in watchdog_*_all_cpus() takes
657 * care of this. If those threads are already active, the sample
658 * period will be updated and the lockup detectors will be enabled
659 * or disabled 'on the fly'.
660 */
661 if (watchdog_enabled && watchdog_thresh)
Ulrich Obergfellb2f57c32015-04-14 15:44:16 -0700662 err = watchdog_enable_all_cpus();
Ulrich Obergfella0c9cbb2015-04-14 15:43:58 -0700663 else
664 watchdog_disable_all_cpus();
665
Nicholas Piggina10a8422017-07-12 14:35:49 -0700666 watchdog_nmi_reconfigure();
667
Ulrich Obergfella0c9cbb2015-04-14 15:43:58 -0700668 return err;
669
670}
671
672/*
Ulrich Obergfellef246a22015-04-14 15:44:05 -0700673 * common function for watchdog, nmi_watchdog and soft_watchdog parameter
674 *
675 * caller | table->data points to | 'which' contains the flag(s)
676 * -------------------|-----------------------|-----------------------------
677 * proc_watchdog | watchdog_user_enabled | NMI_WATCHDOG_ENABLED or'ed
678 * | | with SOFT_WATCHDOG_ENABLED
679 * -------------------|-----------------------|-----------------------------
680 * proc_nmi_watchdog | nmi_watchdog_enabled | NMI_WATCHDOG_ENABLED
681 * -------------------|-----------------------|-----------------------------
682 * proc_soft_watchdog | soft_watchdog_enabled | SOFT_WATCHDOG_ENABLED
683 */
684static int proc_watchdog_common(int which, struct ctl_table *table, int write,
685 void __user *buffer, size_t *lenp, loff_t *ppos)
686{
687 int err, old, new;
688 int *watchdog_param = (int *)table->data;
Don Zickus58687ac2010-05-07 17:11:44 -0400689
Thomas Gleixnerb7a34982017-09-12 21:37:00 +0200690 cpu_hotplug_disable();
Thomas Gleixner946d1972017-09-12 21:37:01 +0200691 mutex_lock(&watchdog_mutex);
Thomas Gleixnerbcd951c2012-07-16 10:42:38 +0000692
Ulrich Obergfellef246a22015-04-14 15:44:05 -0700693 /*
694 * If the parameter is being read return the state of the corresponding
695 * bit(s) in 'watchdog_enabled', else update 'watchdog_enabled' and the
696 * run state of the lockup detectors.
697 */
698 if (!write) {
699 *watchdog_param = (watchdog_enabled & which) != 0;
700 err = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
701 } else {
702 err = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
703 if (err)
704 goto out;
705
706 /*
707 * There is a race window between fetching the current value
708 * from 'watchdog_enabled' and storing the new value. During
709 * this race window, watchdog_nmi_enable() can sneak in and
710 * clear the NMI_WATCHDOG_ENABLED bit in 'watchdog_enabled'.
711 * The 'cmpxchg' detects this race and the loop retries.
712 */
713 do {
714 old = watchdog_enabled;
715 /*
716 * If the parameter value is not zero set the
717 * corresponding bit(s), else clear it(them).
718 */
719 if (*watchdog_param)
720 new = old | which;
721 else
722 new = old & ~which;
723 } while (cmpxchg(&watchdog_enabled, old, new) != old);
724
725 /*
Ulrich Obergfellb43cb432015-11-05 18:44:33 -0800726 * Update the run state of the lockup detectors. There is _no_
727 * need to check the value returned by proc_watchdog_update()
728 * and to restore the previous value of 'watchdog_enabled' as
729 * both lockup detectors are disabled if proc_watchdog_update()
730 * returns an error.
Ulrich Obergfellef246a22015-04-14 15:44:05 -0700731 */
Joshua Hunta1ee1932016-03-17 14:17:23 -0700732 if (old == new)
733 goto out;
734
Ulrich Obergfellef246a22015-04-14 15:44:05 -0700735 err = proc_watchdog_update();
Ulrich Obergfellef246a22015-04-14 15:44:05 -0700736 }
737out:
Thomas Gleixner946d1972017-09-12 21:37:01 +0200738 mutex_unlock(&watchdog_mutex);
Thomas Gleixnerb7a34982017-09-12 21:37:00 +0200739 cpu_hotplug_enable();
Ulrich Obergfellef246a22015-04-14 15:44:05 -0700740 return err;
741}
742
743/*
Ulrich Obergfell83a80a32015-04-14 15:44:08 -0700744 * /proc/sys/kernel/watchdog
745 */
746int proc_watchdog(struct ctl_table *table, int write,
747 void __user *buffer, size_t *lenp, loff_t *ppos)
748{
749 return proc_watchdog_common(NMI_WATCHDOG_ENABLED|SOFT_WATCHDOG_ENABLED,
750 table, write, buffer, lenp, ppos);
751}
752
753/*
754 * /proc/sys/kernel/nmi_watchdog
755 */
756int proc_nmi_watchdog(struct ctl_table *table, int write,
757 void __user *buffer, size_t *lenp, loff_t *ppos)
758{
759 return proc_watchdog_common(NMI_WATCHDOG_ENABLED,
760 table, write, buffer, lenp, ppos);
761}
762
763/*
764 * /proc/sys/kernel/soft_watchdog
765 */
766int proc_soft_watchdog(struct ctl_table *table, int write,
767 void __user *buffer, size_t *lenp, loff_t *ppos)
768{
769 return proc_watchdog_common(SOFT_WATCHDOG_ENABLED,
770 table, write, buffer, lenp, ppos);
771}
772
773/*
774 * /proc/sys/kernel/watchdog_thresh
775 */
776int proc_watchdog_thresh(struct ctl_table *table, int write,
777 void __user *buffer, size_t *lenp, loff_t *ppos)
778{
Joshua Hunta1ee1932016-03-17 14:17:23 -0700779 int err, old, new;
Ulrich Obergfell83a80a32015-04-14 15:44:08 -0700780
Thomas Gleixnerb7a34982017-09-12 21:37:00 +0200781 cpu_hotplug_disable();
Thomas Gleixner946d1972017-09-12 21:37:01 +0200782 mutex_lock(&watchdog_mutex);
Ulrich Obergfell83a80a32015-04-14 15:44:08 -0700783
784 old = ACCESS_ONCE(watchdog_thresh);
Frederic Weisbeckerb8900bc2013-06-06 15:42:53 +0200785 err = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
Ulrich Obergfell83a80a32015-04-14 15:44:08 -0700786
Frederic Weisbeckerb8900bc2013-06-06 15:42:53 +0200787 if (err || !write)
Michal Hocko359e6fa2013-09-24 15:27:29 -0700788 goto out;
Mandeep Singh Bainese04ab2b2011-05-22 22:10:21 -0700789
anish kumarb66a23562013-03-12 14:44:08 -0400790 /*
Ulrich Obergfelld283c642015-11-05 18:44:27 -0800791 * Update the sample period. Restore on failure.
anish kumarb66a23562013-03-12 14:44:08 -0400792 */
Joshua Hunta1ee1932016-03-17 14:17:23 -0700793 new = ACCESS_ONCE(watchdog_thresh);
794 if (old == new)
795 goto out;
796
Ulrich Obergfell83a80a32015-04-14 15:44:08 -0700797 set_sample_period();
798 err = proc_watchdog_update();
Ulrich Obergfelld283c642015-11-05 18:44:27 -0800799 if (err) {
Ulrich Obergfell83a80a32015-04-14 15:44:08 -0700800 watchdog_thresh = old;
Ulrich Obergfelld283c642015-11-05 18:44:27 -0800801 set_sample_period();
802 }
Michal Hocko359e6fa2013-09-24 15:27:29 -0700803out:
Thomas Gleixner946d1972017-09-12 21:37:01 +0200804 mutex_unlock(&watchdog_mutex);
Thomas Gleixnerb7a34982017-09-12 21:37:00 +0200805 cpu_hotplug_enable();
Frederic Weisbeckerb8900bc2013-06-06 15:42:53 +0200806 return err;
Don Zickus58687ac2010-05-07 17:11:44 -0400807}
Chris Metcalffe4ba3c2015-06-24 16:55:45 -0700808
809/*
810 * The cpumask is the mask of possible cpus that the watchdog can run
811 * on, not the mask of cpus it is actually running on. This allows the
812 * user to specify a mask that will include cpus that have not yet
813 * been brought online, if desired.
814 */
815int proc_watchdog_cpumask(struct ctl_table *table, int write,
816 void __user *buffer, size_t *lenp, loff_t *ppos)
817{
818 int err;
819
Thomas Gleixnerb7a34982017-09-12 21:37:00 +0200820 cpu_hotplug_disable();
Thomas Gleixner946d1972017-09-12 21:37:01 +0200821 mutex_lock(&watchdog_mutex);
Ulrich Obergfell8c073d22015-09-04 15:45:18 -0700822
Chris Metcalffe4ba3c2015-06-24 16:55:45 -0700823 err = proc_do_large_bitmap(table, write, buffer, lenp, ppos);
824 if (!err && write) {
825 /* Remove impossible cpus to keep sysctl output cleaner. */
826 cpumask_and(&watchdog_cpumask, &watchdog_cpumask,
827 cpu_possible_mask);
828
829 if (watchdog_running) {
830 /*
831 * Failure would be due to being unable to allocate
832 * a temporary cpumask, so we are likely not in a
833 * position to do much else to make things better.
834 */
Nicholas Piggina10a8422017-07-12 14:35:49 -0700835 if (watchdog_update_cpus() != 0)
Chris Metcalffe4ba3c2015-06-24 16:55:45 -0700836 pr_err("cpumask update failed\n");
837 }
Nicholas Piggina10a8422017-07-12 14:35:49 -0700838
839 watchdog_nmi_reconfigure();
Chris Metcalffe4ba3c2015-06-24 16:55:45 -0700840 }
Thomas Gleixner54901252017-09-12 21:36:59 +0200841
Thomas Gleixner946d1972017-09-12 21:37:01 +0200842 mutex_unlock(&watchdog_mutex);
Thomas Gleixnerb7a34982017-09-12 21:37:00 +0200843 cpu_hotplug_enable();
Chris Metcalffe4ba3c2015-06-24 16:55:45 -0700844 return err;
845}
846
Don Zickus58687ac2010-05-07 17:11:44 -0400847#endif /* CONFIG_SYSCTL */
848
Peter Zijlstra004417a2010-11-25 18:38:29 +0100849void __init lockup_detector_init(void)
Don Zickus58687ac2010-05-07 17:11:44 -0400850{
Chuansheng Liu0f34c402012-12-17 15:59:50 -0800851 set_sample_period();
Frederic Weisbeckerb8900bc2013-06-06 15:42:53 +0200852
Chris Metcalffe4ba3c2015-06-24 16:55:45 -0700853#ifdef CONFIG_NO_HZ_FULL
854 if (tick_nohz_full_enabled()) {
Frederic Weisbecker314b08ff2015-09-04 15:45:09 -0700855 pr_info("Disabling watchdog on nohz_full cores by default\n");
856 cpumask_copy(&watchdog_cpumask, housekeeping_mask);
Chris Metcalffe4ba3c2015-06-24 16:55:45 -0700857 } else
858 cpumask_copy(&watchdog_cpumask, cpu_possible_mask);
859#else
860 cpumask_copy(&watchdog_cpumask, cpu_possible_mask);
861#endif
862
Ulrich Obergfell195daf62015-04-14 15:44:13 -0700863 if (watchdog_enabled)
Ulrich Obergfellb2f57c32015-04-14 15:44:16 -0700864 watchdog_enable_all_cpus();
Don Zickus58687ac2010-05-07 17:11:44 -0400865}