blob: d36d8e0abfb8ddbb4c65a4762d10edfaabdbe451 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/* CPU control.
2 * (C) 2001, 2002, 2003, 2004 Rusty Russell
3 *
4 * This code is licenced under the GPL.
5 */
6#include <linux/proc_fs.h>
7#include <linux/smp.h>
8#include <linux/init.h>
9#include <linux/notifier.h>
10#include <linux/sched.h>
11#include <linux/unistd.h>
12#include <linux/cpu.h>
Anton Vorontsovcb792952012-05-31 16:26:22 -070013#include <linux/oom.h>
14#include <linux/rcupdate.h>
Paul Gortmaker9984de12011-05-23 14:51:41 -040015#include <linux/export.h>
Anton Vorontsove4cc2f82012-05-31 16:26:26 -070016#include <linux/bug.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070017#include <linux/kthread.h>
18#include <linux/stop_machine.h>
Ingo Molnar81615b622006-06-26 00:24:32 -070019#include <linux/mutex.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090020#include <linux/gfp.h>
Srivatsa S. Bhat79cfbdf2011-11-03 00:59:25 +010021#include <linux/suspend.h>
Gautham R. Shenoya19423b2014-03-11 02:04:03 +053022#include <linux/lockdep.h>
Preeti U Murthy345527b2015-03-30 14:59:19 +053023#include <linux/tick.h>
Thomas Gleixnera8994182015-07-05 17:12:30 +000024#include <linux/irq.h>
Thomas Gleixner4cb28ce2016-02-26 18:43:38 +000025#include <linux/smpboot.h>
Thomas Gleixnercff7d372016-02-26 18:43:28 +000026
Todd E Brandtbb3632c2014-06-06 05:40:17 -070027#include <trace/events/power.h>
Thomas Gleixnercff7d372016-02-26 18:43:28 +000028#define CREATE_TRACE_POINTS
29#include <trace/events/cpuhp.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070030
Thomas Gleixner38498a62012-04-20 13:05:44 +000031#include "smpboot.h"
32
Thomas Gleixnercff7d372016-02-26 18:43:28 +000033/**
34 * cpuhp_cpu_state - Per cpu hotplug state storage
35 * @state: The current cpu state
36 * @target: The target state
Thomas Gleixner4cb28ce2016-02-26 18:43:38 +000037 * @thread: Pointer to the hotplug thread
38 * @should_run: Thread should execute
Sebastian Andrzej Siewior3b9d6da2016-04-08 14:40:15 +020039 * @rollback: Perform a rollback
Thomas Gleixnera7246322016-08-12 19:49:38 +020040 * @single: Single callback invocation
41 * @bringup: Single callback bringup or teardown selector
42 * @cb_state: The state for a single callback (install/uninstall)
Thomas Gleixner4cb28ce2016-02-26 18:43:38 +000043 * @result: Result of the operation
44 * @done: Signal completion to the issuer of the task
Thomas Gleixnercff7d372016-02-26 18:43:28 +000045 */
46struct cpuhp_cpu_state {
47 enum cpuhp_state state;
48 enum cpuhp_state target;
Thomas Gleixner4cb28ce2016-02-26 18:43:38 +000049#ifdef CONFIG_SMP
50 struct task_struct *thread;
51 bool should_run;
Sebastian Andrzej Siewior3b9d6da2016-04-08 14:40:15 +020052 bool rollback;
Thomas Gleixnera7246322016-08-12 19:49:38 +020053 bool single;
54 bool bringup;
Thomas Gleixner4cb28ce2016-02-26 18:43:38 +000055 enum cpuhp_state cb_state;
Thomas Gleixner4cb28ce2016-02-26 18:43:38 +000056 int result;
57 struct completion done;
58#endif
Thomas Gleixnercff7d372016-02-26 18:43:28 +000059};
60
61static DEFINE_PER_CPU(struct cpuhp_cpu_state, cpuhp_state);
62
63/**
64 * cpuhp_step - Hotplug state machine step
65 * @name: Name of the step
66 * @startup: Startup function of the step
67 * @teardown: Teardown function of the step
68 * @skip_onerr: Do not invoke the functions on error rollback
69 * Will go away once the notifiers are gone
Thomas Gleixner757c9892016-02-26 18:43:32 +000070 * @cant_stop: Bringup/teardown can't be stopped at this step
Thomas Gleixnercff7d372016-02-26 18:43:28 +000071 */
72struct cpuhp_step {
73 const char *name;
74 int (*startup)(unsigned int cpu);
75 int (*teardown)(unsigned int cpu);
76 bool skip_onerr;
Thomas Gleixner757c9892016-02-26 18:43:32 +000077 bool cant_stop;
Thomas Gleixnercff7d372016-02-26 18:43:28 +000078};
79
Thomas Gleixner98f8cdc2016-02-26 18:43:31 +000080static DEFINE_MUTEX(cpuhp_state_mutex);
Thomas Gleixnercff7d372016-02-26 18:43:28 +000081static struct cpuhp_step cpuhp_bp_states[];
Thomas Gleixner4baa0af2016-02-26 18:43:29 +000082static struct cpuhp_step cpuhp_ap_states[];
Thomas Gleixnercff7d372016-02-26 18:43:28 +000083
Thomas Gleixnera7246322016-08-12 19:49:38 +020084static bool cpuhp_is_ap_state(enum cpuhp_state state)
85{
86 /*
87 * The extra check for CPUHP_TEARDOWN_CPU is only for documentation
88 * purposes as that state is handled explicitly in cpu_down.
89 */
90 return state > CPUHP_BRINGUP_CPU && state != CPUHP_TEARDOWN_CPU;
91}
92
93static struct cpuhp_step *cpuhp_get_step(enum cpuhp_state state)
94{
95 struct cpuhp_step *sp;
96
97 sp = cpuhp_is_ap_state(state) ? cpuhp_ap_states : cpuhp_bp_states;
98 return sp + state;
99}
100
Thomas Gleixnercff7d372016-02-26 18:43:28 +0000101/**
102 * cpuhp_invoke_callback _ Invoke the callbacks for a given state
103 * @cpu: The cpu for which the callback should be invoked
104 * @step: The step in the state machine
Thomas Gleixnera7246322016-08-12 19:49:38 +0200105 * @bringup: True if the bringup callback should be invoked
Thomas Gleixnercff7d372016-02-26 18:43:28 +0000106 *
107 * Called from cpu hotplug and from the state register machinery
108 */
Thomas Gleixnera7246322016-08-12 19:49:38 +0200109static int cpuhp_invoke_callback(unsigned int cpu, enum cpuhp_state state,
110 bool bringup)
Thomas Gleixnercff7d372016-02-26 18:43:28 +0000111{
112 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
Thomas Gleixnera7246322016-08-12 19:49:38 +0200113 struct cpuhp_step *step = cpuhp_get_step(state);
114 int (*cb)(unsigned int cpu) = bringup ? step->startup : step->teardown;
Thomas Gleixnercff7d372016-02-26 18:43:28 +0000115 int ret = 0;
116
117 if (cb) {
Thomas Gleixnera7246322016-08-12 19:49:38 +0200118 trace_cpuhp_enter(cpu, st->target, state, cb);
Thomas Gleixnercff7d372016-02-26 18:43:28 +0000119 ret = cb(cpu);
Thomas Gleixnera7246322016-08-12 19:49:38 +0200120 trace_cpuhp_exit(cpu, st->state, state, ret);
Thomas Gleixnercff7d372016-02-26 18:43:28 +0000121 }
122 return ret;
123}
124
Rusty Russell98a79d62008-12-13 21:19:41 +1030125#ifdef CONFIG_SMP
Rusty Russellb3199c02008-12-30 09:05:14 +1030126/* Serializes the updates to cpu_online_mask, cpu_present_mask */
Linus Torvaldsaa953872006-07-23 12:12:16 -0700127static DEFINE_MUTEX(cpu_add_remove_lock);
Thomas Gleixner090e77c2016-02-26 18:43:23 +0000128bool cpuhp_tasks_frozen;
129EXPORT_SYMBOL_GPL(cpuhp_tasks_frozen);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700130
Lai Jiangshan79a6cde2010-05-26 14:43:36 -0700131/*
Srivatsa S. Bhat93ae4f92014-03-11 02:04:14 +0530132 * The following two APIs (cpu_maps_update_begin/done) must be used when
133 * attempting to serialize the updates to cpu_online_mask & cpu_present_mask.
134 * The APIs cpu_notifier_register_begin/done() must be used to protect CPU
135 * hotplug callback (un)registration performed using __register_cpu_notifier()
136 * or __unregister_cpu_notifier().
Lai Jiangshan79a6cde2010-05-26 14:43:36 -0700137 */
138void cpu_maps_update_begin(void)
139{
140 mutex_lock(&cpu_add_remove_lock);
141}
Srivatsa S. Bhat93ae4f92014-03-11 02:04:14 +0530142EXPORT_SYMBOL(cpu_notifier_register_begin);
Lai Jiangshan79a6cde2010-05-26 14:43:36 -0700143
144void cpu_maps_update_done(void)
145{
146 mutex_unlock(&cpu_add_remove_lock);
147}
Srivatsa S. Bhat93ae4f92014-03-11 02:04:14 +0530148EXPORT_SYMBOL(cpu_notifier_register_done);
Lai Jiangshan79a6cde2010-05-26 14:43:36 -0700149
Daniel J Blueman5c113fb2010-06-01 12:15:11 +0100150static RAW_NOTIFIER_HEAD(cpu_chain);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700151
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -0700152/* If set, cpu_up and cpu_down will return -EBUSY and do nothing.
153 * Should always be manipulated under cpu_add_remove_lock
154 */
155static int cpu_hotplug_disabled;
156
Lai Jiangshan79a6cde2010-05-26 14:43:36 -0700157#ifdef CONFIG_HOTPLUG_CPU
158
Gautham R Shenoyd2219382008-01-25 21:08:01 +0100159static struct {
160 struct task_struct *active_writer;
David Hildenbrand87af9e72014-12-12 10:11:44 +0100161 /* wait queue to wake up the active_writer */
162 wait_queue_head_t wq;
163 /* verifies that no writer will get active while readers are active */
164 struct mutex lock;
Gautham R Shenoyd2219382008-01-25 21:08:01 +0100165 /*
166 * Also blocks the new readers during
167 * an ongoing cpu hotplug operation.
168 */
David Hildenbrand87af9e72014-12-12 10:11:44 +0100169 atomic_t refcount;
Gautham R. Shenoya19423b2014-03-11 02:04:03 +0530170
171#ifdef CONFIG_DEBUG_LOCK_ALLOC
172 struct lockdep_map dep_map;
173#endif
Linus Torvalds31950eb2009-06-22 21:18:12 -0700174} cpu_hotplug = {
175 .active_writer = NULL,
David Hildenbrand87af9e72014-12-12 10:11:44 +0100176 .wq = __WAIT_QUEUE_HEAD_INITIALIZER(cpu_hotplug.wq),
Linus Torvalds31950eb2009-06-22 21:18:12 -0700177 .lock = __MUTEX_INITIALIZER(cpu_hotplug.lock),
Gautham R. Shenoya19423b2014-03-11 02:04:03 +0530178#ifdef CONFIG_DEBUG_LOCK_ALLOC
179 .dep_map = {.name = "cpu_hotplug.lock" },
180#endif
Linus Torvalds31950eb2009-06-22 21:18:12 -0700181};
Gautham R Shenoyd2219382008-01-25 21:08:01 +0100182
Gautham R. Shenoya19423b2014-03-11 02:04:03 +0530183/* Lockdep annotations for get/put_online_cpus() and cpu_hotplug_begin/end() */
184#define cpuhp_lock_acquire_read() lock_map_acquire_read(&cpu_hotplug.dep_map)
Paul E. McKenneydd56af42014-08-25 20:25:06 -0700185#define cpuhp_lock_acquire_tryread() \
186 lock_map_acquire_tryread(&cpu_hotplug.dep_map)
Gautham R. Shenoya19423b2014-03-11 02:04:03 +0530187#define cpuhp_lock_acquire() lock_map_acquire(&cpu_hotplug.dep_map)
188#define cpuhp_lock_release() lock_map_release(&cpu_hotplug.dep_map)
189
Paul E. McKenney62db99f2014-10-22 14:51:49 -0700190
Gautham R Shenoy86ef5c92008-01-25 21:08:02 +0100191void get_online_cpus(void)
Ashok Raja9d9baa2005-11-28 13:43:46 -0800192{
Gautham R Shenoyd2219382008-01-25 21:08:01 +0100193 might_sleep();
194 if (cpu_hotplug.active_writer == current)
Linus Torvaldsaa953872006-07-23 12:12:16 -0700195 return;
Gautham R. Shenoya19423b2014-03-11 02:04:03 +0530196 cpuhp_lock_acquire_read();
Gautham R Shenoyd2219382008-01-25 21:08:01 +0100197 mutex_lock(&cpu_hotplug.lock);
David Hildenbrand87af9e72014-12-12 10:11:44 +0100198 atomic_inc(&cpu_hotplug.refcount);
Gautham R Shenoyd2219382008-01-25 21:08:01 +0100199 mutex_unlock(&cpu_hotplug.lock);
Ashok Raja9d9baa2005-11-28 13:43:46 -0800200}
Gautham R Shenoy86ef5c92008-01-25 21:08:02 +0100201EXPORT_SYMBOL_GPL(get_online_cpus);
Ashok Raj90d45d12005-11-08 21:34:24 -0800202
Gautham R Shenoy86ef5c92008-01-25 21:08:02 +0100203void put_online_cpus(void)
Ashok Raja9d9baa2005-11-28 13:43:46 -0800204{
David Hildenbrand87af9e72014-12-12 10:11:44 +0100205 int refcount;
206
Gautham R Shenoyd2219382008-01-25 21:08:01 +0100207 if (cpu_hotplug.active_writer == current)
Linus Torvaldsaa953872006-07-23 12:12:16 -0700208 return;
Srivatsa S. Bhat075663d2012-10-08 16:28:20 -0700209
David Hildenbrand87af9e72014-12-12 10:11:44 +0100210 refcount = atomic_dec_return(&cpu_hotplug.refcount);
211 if (WARN_ON(refcount < 0)) /* try to fix things up */
212 atomic_inc(&cpu_hotplug.refcount);
Srivatsa S. Bhat075663d2012-10-08 16:28:20 -0700213
David Hildenbrand87af9e72014-12-12 10:11:44 +0100214 if (refcount <= 0 && waitqueue_active(&cpu_hotplug.wq))
215 wake_up(&cpu_hotplug.wq);
216
Gautham R. Shenoya19423b2014-03-11 02:04:03 +0530217 cpuhp_lock_release();
Gautham R Shenoyd2219382008-01-25 21:08:01 +0100218
Ashok Raja9d9baa2005-11-28 13:43:46 -0800219}
Gautham R Shenoy86ef5c92008-01-25 21:08:02 +0100220EXPORT_SYMBOL_GPL(put_online_cpus);
Ashok Raja9d9baa2005-11-28 13:43:46 -0800221
Gautham R Shenoyd2219382008-01-25 21:08:01 +0100222/*
223 * This ensures that the hotplug operation can begin only when the
224 * refcount goes to zero.
225 *
226 * Note that during a cpu-hotplug operation, the new readers, if any,
227 * will be blocked by the cpu_hotplug.lock
228 *
Oleg Nesterovd2ba7e22008-04-29 01:00:29 -0700229 * Since cpu_hotplug_begin() is always called after invoking
230 * cpu_maps_update_begin(), we can be sure that only one writer is active.
Gautham R Shenoyd2219382008-01-25 21:08:01 +0100231 *
232 * Note that theoretically, there is a possibility of a livelock:
233 * - Refcount goes to zero, last reader wakes up the sleeping
234 * writer.
235 * - Last reader unlocks the cpu_hotplug.lock.
236 * - A new reader arrives at this moment, bumps up the refcount.
237 * - The writer acquires the cpu_hotplug.lock finds the refcount
238 * non zero and goes to sleep again.
239 *
240 * However, this is very difficult to achieve in practice since
Gautham R Shenoy86ef5c92008-01-25 21:08:02 +0100241 * get_online_cpus() not an api which is called all that often.
Gautham R Shenoyd2219382008-01-25 21:08:01 +0100242 *
243 */
Toshi Kanib9d10be2013-08-12 09:45:53 -0600244void cpu_hotplug_begin(void)
Gautham R Shenoyd2219382008-01-25 21:08:01 +0100245{
David Hildenbrand87af9e72014-12-12 10:11:44 +0100246 DEFINE_WAIT(wait);
Oleg Nesterovd2ba7e22008-04-29 01:00:29 -0700247
David Hildenbrand87af9e72014-12-12 10:11:44 +0100248 cpu_hotplug.active_writer = current;
Gautham R. Shenoya19423b2014-03-11 02:04:03 +0530249 cpuhp_lock_acquire();
David Hildenbrand87af9e72014-12-12 10:11:44 +0100250
Oleg Nesterovd2ba7e22008-04-29 01:00:29 -0700251 for (;;) {
252 mutex_lock(&cpu_hotplug.lock);
David Hildenbrand87af9e72014-12-12 10:11:44 +0100253 prepare_to_wait(&cpu_hotplug.wq, &wait, TASK_UNINTERRUPTIBLE);
254 if (likely(!atomic_read(&cpu_hotplug.refcount)))
255 break;
Gautham R Shenoyd2219382008-01-25 21:08:01 +0100256 mutex_unlock(&cpu_hotplug.lock);
257 schedule();
Gautham R Shenoyd2219382008-01-25 21:08:01 +0100258 }
David Hildenbrand87af9e72014-12-12 10:11:44 +0100259 finish_wait(&cpu_hotplug.wq, &wait);
Gautham R Shenoyd2219382008-01-25 21:08:01 +0100260}
261
Toshi Kanib9d10be2013-08-12 09:45:53 -0600262void cpu_hotplug_done(void)
Gautham R Shenoyd2219382008-01-25 21:08:01 +0100263{
264 cpu_hotplug.active_writer = NULL;
265 mutex_unlock(&cpu_hotplug.lock);
Gautham R. Shenoya19423b2014-03-11 02:04:03 +0530266 cpuhp_lock_release();
Gautham R Shenoyd2219382008-01-25 21:08:01 +0100267}
Lai Jiangshan79a6cde2010-05-26 14:43:36 -0700268
Srivatsa S. Bhat16e53db2013-06-12 14:04:36 -0700269/*
270 * Wait for currently running CPU hotplug operations to complete (if any) and
271 * disable future CPU hotplug (from sysfs). The 'cpu_add_remove_lock' protects
272 * the 'cpu_hotplug_disabled' flag. The same lock is also acquired by the
273 * hotplug path before performing hotplug operations. So acquiring that lock
274 * guarantees mutual exclusion from any currently running hotplug operations.
275 */
276void cpu_hotplug_disable(void)
277{
278 cpu_maps_update_begin();
Vitaly Kuznetsov89af7ba2015-08-05 00:52:46 -0700279 cpu_hotplug_disabled++;
Srivatsa S. Bhat16e53db2013-06-12 14:04:36 -0700280 cpu_maps_update_done();
281}
Vitaly Kuznetsov32145c42015-08-05 00:52:47 -0700282EXPORT_SYMBOL_GPL(cpu_hotplug_disable);
Srivatsa S. Bhat16e53db2013-06-12 14:04:36 -0700283
284void cpu_hotplug_enable(void)
285{
286 cpu_maps_update_begin();
Vitaly Kuznetsov89af7ba2015-08-05 00:52:46 -0700287 WARN_ON(--cpu_hotplug_disabled < 0);
Srivatsa S. Bhat16e53db2013-06-12 14:04:36 -0700288 cpu_maps_update_done();
289}
Vitaly Kuznetsov32145c42015-08-05 00:52:47 -0700290EXPORT_SYMBOL_GPL(cpu_hotplug_enable);
Toshi Kanib9d10be2013-08-12 09:45:53 -0600291#endif /* CONFIG_HOTPLUG_CPU */
Lai Jiangshan79a6cde2010-05-26 14:43:36 -0700292
Linus Torvalds1da177e2005-04-16 15:20:36 -0700293/* Need to know about CPUs going up/down? */
Mathias Krause71cf5ae2015-07-19 20:06:22 +0200294int register_cpu_notifier(struct notifier_block *nb)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700295{
Neil Brownbd5349c2006-10-17 00:10:35 -0700296 int ret;
Gautham R Shenoyd2219382008-01-25 21:08:01 +0100297 cpu_maps_update_begin();
Neil Brownbd5349c2006-10-17 00:10:35 -0700298 ret = raw_notifier_chain_register(&cpu_chain, nb);
Gautham R Shenoyd2219382008-01-25 21:08:01 +0100299 cpu_maps_update_done();
Neil Brownbd5349c2006-10-17 00:10:35 -0700300 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700301}
Chandra Seetharaman65edc682006-06-27 02:54:08 -0700302
Mathias Krause71cf5ae2015-07-19 20:06:22 +0200303int __register_cpu_notifier(struct notifier_block *nb)
Srivatsa S. Bhat93ae4f92014-03-11 02:04:14 +0530304{
305 return raw_notifier_chain_register(&cpu_chain, nb);
306}
307
Thomas Gleixner090e77c2016-02-26 18:43:23 +0000308static int __cpu_notify(unsigned long val, unsigned int cpu, int nr_to_call,
Akinobu Mitae9fb7632010-05-26 14:43:28 -0700309 int *nr_calls)
310{
Thomas Gleixner090e77c2016-02-26 18:43:23 +0000311 unsigned long mod = cpuhp_tasks_frozen ? CPU_TASKS_FROZEN : 0;
312 void *hcpu = (void *)(long)cpu;
313
Akinobu Mitae6bde732010-05-26 14:43:29 -0700314 int ret;
315
Thomas Gleixner090e77c2016-02-26 18:43:23 +0000316 ret = __raw_notifier_call_chain(&cpu_chain, val | mod, hcpu, nr_to_call,
Akinobu Mitae9fb7632010-05-26 14:43:28 -0700317 nr_calls);
Akinobu Mitae6bde732010-05-26 14:43:29 -0700318
319 return notifier_to_errno(ret);
Akinobu Mitae9fb7632010-05-26 14:43:28 -0700320}
321
Thomas Gleixner090e77c2016-02-26 18:43:23 +0000322static int cpu_notify(unsigned long val, unsigned int cpu)
Akinobu Mitae9fb7632010-05-26 14:43:28 -0700323{
Thomas Gleixner090e77c2016-02-26 18:43:23 +0000324 return __cpu_notify(val, cpu, -1, NULL);
Akinobu Mitae9fb7632010-05-26 14:43:28 -0700325}
326
Sebastian Andrzej Siewior3b9d6da2016-04-08 14:40:15 +0200327static void cpu_notify_nofail(unsigned long val, unsigned int cpu)
328{
329 BUG_ON(cpu_notify(val, cpu));
330}
331
Thomas Gleixnerba997462016-02-26 18:43:24 +0000332/* Notifier wrappers for transitioning to state machine */
333static int notify_prepare(unsigned int cpu)
334{
335 int nr_calls = 0;
336 int ret;
337
338 ret = __cpu_notify(CPU_UP_PREPARE, cpu, -1, &nr_calls);
339 if (ret) {
340 nr_calls--;
341 printk(KERN_WARNING "%s: attempt to bring up CPU %u failed\n",
342 __func__, cpu);
343 __cpu_notify(CPU_UP_CANCELED, cpu, nr_calls, NULL);
344 }
345 return ret;
346}
347
348static int notify_online(unsigned int cpu)
349{
350 cpu_notify(CPU_ONLINE, cpu);
351 return 0;
352}
353
Thomas Gleixner4baa0af2016-02-26 18:43:29 +0000354static int notify_starting(unsigned int cpu)
355{
356 cpu_notify(CPU_STARTING, cpu);
357 return 0;
358}
359
Thomas Gleixner8df3e072016-02-26 18:43:41 +0000360static int bringup_wait_for_ap(unsigned int cpu)
361{
362 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
363
364 wait_for_completion(&st->done);
365 return st->result;
366}
367
Thomas Gleixnerba997462016-02-26 18:43:24 +0000368static int bringup_cpu(unsigned int cpu)
369{
370 struct task_struct *idle = idle_thread_get(cpu);
371 int ret;
372
Boris Ostrovskyaa877172016-08-03 13:22:28 -0400373 /*
374 * Some architectures have to walk the irq descriptors to
375 * setup the vector space for the cpu which comes online.
376 * Prevent irq alloc/free across the bringup.
377 */
378 irq_lock_sparse();
379
Thomas Gleixnerba997462016-02-26 18:43:24 +0000380 /* Arch-specific enabling code. */
381 ret = __cpu_up(cpu, idle);
Boris Ostrovskyaa877172016-08-03 13:22:28 -0400382 irq_unlock_sparse();
Thomas Gleixnerba997462016-02-26 18:43:24 +0000383 if (ret) {
384 cpu_notify(CPU_UP_CANCELED, cpu);
385 return ret;
386 }
Thomas Gleixner8df3e072016-02-26 18:43:41 +0000387 ret = bringup_wait_for_ap(cpu);
Thomas Gleixnerba997462016-02-26 18:43:24 +0000388 BUG_ON(!cpu_online(cpu));
Thomas Gleixner8df3e072016-02-26 18:43:41 +0000389 return ret;
Thomas Gleixnerba997462016-02-26 18:43:24 +0000390}
391
Thomas Gleixner2e1a3482016-02-26 18:43:37 +0000392/*
393 * Hotplug state machine related functions
394 */
Thomas Gleixnera7246322016-08-12 19:49:38 +0200395static void undo_cpu_down(unsigned int cpu, struct cpuhp_cpu_state *st)
Thomas Gleixner2e1a3482016-02-26 18:43:37 +0000396{
397 for (st->state++; st->state < st->target; st->state++) {
Thomas Gleixnera7246322016-08-12 19:49:38 +0200398 struct cpuhp_step *step = cpuhp_get_step(st->state);
Thomas Gleixner2e1a3482016-02-26 18:43:37 +0000399
400 if (!step->skip_onerr)
Thomas Gleixnera7246322016-08-12 19:49:38 +0200401 cpuhp_invoke_callback(cpu, st->state, true);
Thomas Gleixner2e1a3482016-02-26 18:43:37 +0000402 }
403}
404
405static int cpuhp_down_callbacks(unsigned int cpu, struct cpuhp_cpu_state *st,
Thomas Gleixnera7246322016-08-12 19:49:38 +0200406 enum cpuhp_state target)
Thomas Gleixner2e1a3482016-02-26 18:43:37 +0000407{
408 enum cpuhp_state prev_state = st->state;
409 int ret = 0;
410
411 for (; st->state > target; st->state--) {
Thomas Gleixnera7246322016-08-12 19:49:38 +0200412 ret = cpuhp_invoke_callback(cpu, st->state, false);
Thomas Gleixner2e1a3482016-02-26 18:43:37 +0000413 if (ret) {
414 st->target = prev_state;
Thomas Gleixnera7246322016-08-12 19:49:38 +0200415 undo_cpu_down(cpu, st);
Thomas Gleixner2e1a3482016-02-26 18:43:37 +0000416 break;
417 }
418 }
419 return ret;
420}
421
Thomas Gleixnera7246322016-08-12 19:49:38 +0200422static void undo_cpu_up(unsigned int cpu, struct cpuhp_cpu_state *st)
Thomas Gleixner2e1a3482016-02-26 18:43:37 +0000423{
424 for (st->state--; st->state > st->target; st->state--) {
Thomas Gleixnera7246322016-08-12 19:49:38 +0200425 struct cpuhp_step *step = cpuhp_get_step(st->state);
Thomas Gleixner2e1a3482016-02-26 18:43:37 +0000426
427 if (!step->skip_onerr)
Thomas Gleixnera7246322016-08-12 19:49:38 +0200428 cpuhp_invoke_callback(cpu, st->state, false);
Thomas Gleixner2e1a3482016-02-26 18:43:37 +0000429 }
430}
431
432static int cpuhp_up_callbacks(unsigned int cpu, struct cpuhp_cpu_state *st,
Thomas Gleixnera7246322016-08-12 19:49:38 +0200433 enum cpuhp_state target)
Thomas Gleixner2e1a3482016-02-26 18:43:37 +0000434{
435 enum cpuhp_state prev_state = st->state;
436 int ret = 0;
437
438 while (st->state < target) {
Thomas Gleixner2e1a3482016-02-26 18:43:37 +0000439 st->state++;
Thomas Gleixnera7246322016-08-12 19:49:38 +0200440 ret = cpuhp_invoke_callback(cpu, st->state, true);
Thomas Gleixner2e1a3482016-02-26 18:43:37 +0000441 if (ret) {
442 st->target = prev_state;
Thomas Gleixnera7246322016-08-12 19:49:38 +0200443 undo_cpu_up(cpu, st);
Thomas Gleixner2e1a3482016-02-26 18:43:37 +0000444 break;
445 }
446 }
447 return ret;
448}
449
Thomas Gleixner4cb28ce2016-02-26 18:43:38 +0000450/*
451 * The cpu hotplug threads manage the bringup and teardown of the cpus
452 */
453static void cpuhp_create(unsigned int cpu)
454{
455 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
456
457 init_completion(&st->done);
458}
459
460static int cpuhp_should_run(unsigned int cpu)
461{
462 struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);
463
464 return st->should_run;
465}
466
467/* Execute the teardown callbacks. Used to be CPU_DOWN_PREPARE */
468static int cpuhp_ap_offline(unsigned int cpu, struct cpuhp_cpu_state *st)
469{
Thomas Gleixner1cf4f622016-02-26 18:43:39 +0000470 enum cpuhp_state target = max((int)st->target, CPUHP_TEARDOWN_CPU);
Thomas Gleixner4cb28ce2016-02-26 18:43:38 +0000471
Thomas Gleixnera7246322016-08-12 19:49:38 +0200472 return cpuhp_down_callbacks(cpu, st, target);
Thomas Gleixner4cb28ce2016-02-26 18:43:38 +0000473}
474
475/* Execute the online startup callbacks. Used to be CPU_ONLINE */
476static int cpuhp_ap_online(unsigned int cpu, struct cpuhp_cpu_state *st)
477{
Thomas Gleixnera7246322016-08-12 19:49:38 +0200478 return cpuhp_up_callbacks(cpu, st, st->target);
Thomas Gleixner4cb28ce2016-02-26 18:43:38 +0000479}
480
481/*
482 * Execute teardown/startup callbacks on the plugged cpu. Also used to invoke
483 * callbacks when a state gets [un]installed at runtime.
484 */
485static void cpuhp_thread_fun(unsigned int cpu)
486{
487 struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);
488 int ret = 0;
489
490 /*
491 * Paired with the mb() in cpuhp_kick_ap_work and
492 * cpuhp_invoke_ap_callback, so the work set is consistent visible.
493 */
494 smp_mb();
495 if (!st->should_run)
496 return;
497
498 st->should_run = false;
499
500 /* Single callback invocation for [un]install ? */
Thomas Gleixnera7246322016-08-12 19:49:38 +0200501 if (st->single) {
Thomas Gleixner4cb28ce2016-02-26 18:43:38 +0000502 if (st->cb_state < CPUHP_AP_ONLINE) {
503 local_irq_disable();
Thomas Gleixnera7246322016-08-12 19:49:38 +0200504 ret = cpuhp_invoke_callback(cpu, st->cb_state,
505 st->bringup);
Thomas Gleixner4cb28ce2016-02-26 18:43:38 +0000506 local_irq_enable();
507 } else {
Thomas Gleixnera7246322016-08-12 19:49:38 +0200508 ret = cpuhp_invoke_callback(cpu, st->cb_state,
509 st->bringup);
Thomas Gleixner4cb28ce2016-02-26 18:43:38 +0000510 }
Sebastian Andrzej Siewior3b9d6da2016-04-08 14:40:15 +0200511 } else if (st->rollback) {
512 BUG_ON(st->state < CPUHP_AP_ONLINE_IDLE);
513
Thomas Gleixnera7246322016-08-12 19:49:38 +0200514 undo_cpu_down(cpu, st);
Sebastian Andrzej Siewior3b9d6da2016-04-08 14:40:15 +0200515 /*
516 * This is a momentary workaround to keep the notifier users
517 * happy. Will go away once we got rid of the notifiers.
518 */
519 cpu_notify_nofail(CPU_DOWN_FAILED, cpu);
520 st->rollback = false;
Thomas Gleixner4cb28ce2016-02-26 18:43:38 +0000521 } else {
Thomas Gleixner1cf4f622016-02-26 18:43:39 +0000522 /* Cannot happen .... */
Thomas Gleixner8df3e072016-02-26 18:43:41 +0000523 BUG_ON(st->state < CPUHP_AP_ONLINE_IDLE);
Thomas Gleixner1cf4f622016-02-26 18:43:39 +0000524
Thomas Gleixner4cb28ce2016-02-26 18:43:38 +0000525 /* Regular hotplug work */
526 if (st->state < st->target)
527 ret = cpuhp_ap_online(cpu, st);
528 else if (st->state > st->target)
529 ret = cpuhp_ap_offline(cpu, st);
530 }
531 st->result = ret;
532 complete(&st->done);
533}
534
535/* Invoke a single callback on a remote cpu */
Thomas Gleixnera7246322016-08-12 19:49:38 +0200536static int
537cpuhp_invoke_ap_callback(int cpu, enum cpuhp_state state, bool bringup)
Thomas Gleixner4cb28ce2016-02-26 18:43:38 +0000538{
539 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
540
541 if (!cpu_online(cpu))
542 return 0;
543
Thomas Gleixner6a4e2452016-07-13 17:16:03 +0000544 /*
545 * If we are up and running, use the hotplug thread. For early calls
546 * we invoke the thread function directly.
547 */
548 if (!st->thread)
Thomas Gleixnera7246322016-08-12 19:49:38 +0200549 return cpuhp_invoke_callback(cpu, state, bringup);
Thomas Gleixner6a4e2452016-07-13 17:16:03 +0000550
Thomas Gleixner4cb28ce2016-02-26 18:43:38 +0000551 st->cb_state = state;
Thomas Gleixnera7246322016-08-12 19:49:38 +0200552 st->single = true;
553 st->bringup = bringup;
554
Thomas Gleixner4cb28ce2016-02-26 18:43:38 +0000555 /*
556 * Make sure the above stores are visible before should_run becomes
557 * true. Paired with the mb() above in cpuhp_thread_fun()
558 */
559 smp_mb();
560 st->should_run = true;
561 wake_up_process(st->thread);
562 wait_for_completion(&st->done);
563 return st->result;
564}
565
566/* Regular hotplug invocation of the AP hotplug thread */
Thomas Gleixner1cf4f622016-02-26 18:43:39 +0000567static void __cpuhp_kick_ap_work(struct cpuhp_cpu_state *st)
Thomas Gleixner4cb28ce2016-02-26 18:43:38 +0000568{
Thomas Gleixner4cb28ce2016-02-26 18:43:38 +0000569 st->result = 0;
Thomas Gleixnera7246322016-08-12 19:49:38 +0200570 st->single = false;
Thomas Gleixner4cb28ce2016-02-26 18:43:38 +0000571 /*
572 * Make sure the above stores are visible before should_run becomes
573 * true. Paired with the mb() above in cpuhp_thread_fun()
574 */
575 smp_mb();
576 st->should_run = true;
577 wake_up_process(st->thread);
Thomas Gleixner1cf4f622016-02-26 18:43:39 +0000578}
579
580static int cpuhp_kick_ap_work(unsigned int cpu)
581{
582 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
583 enum cpuhp_state state = st->state;
584
585 trace_cpuhp_enter(cpu, st->target, state, cpuhp_kick_ap_work);
586 __cpuhp_kick_ap_work(st);
Thomas Gleixner4cb28ce2016-02-26 18:43:38 +0000587 wait_for_completion(&st->done);
588 trace_cpuhp_exit(cpu, st->state, state, st->result);
589 return st->result;
590}
591
592static struct smp_hotplug_thread cpuhp_threads = {
593 .store = &cpuhp_state.thread,
594 .create = &cpuhp_create,
595 .thread_should_run = cpuhp_should_run,
596 .thread_fn = cpuhp_thread_fun,
597 .thread_comm = "cpuhp/%u",
598 .selfparking = true,
599};
600
601void __init cpuhp_threads_init(void)
602{
603 BUG_ON(smpboot_register_percpu_thread(&cpuhp_threads));
604 kthread_unpark(this_cpu_read(cpuhp_state.thread));
605}
606
Linus Torvalds00b9b0a2010-05-27 10:32:08 -0700607#ifdef CONFIG_HOTPLUG_CPU
Linus Torvalds1da177e2005-04-16 15:20:36 -0700608EXPORT_SYMBOL(register_cpu_notifier);
Srivatsa S. Bhat93ae4f92014-03-11 02:04:14 +0530609EXPORT_SYMBOL(__register_cpu_notifier);
Mathias Krause71cf5ae2015-07-19 20:06:22 +0200610void unregister_cpu_notifier(struct notifier_block *nb)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700611{
Gautham R Shenoyd2219382008-01-25 21:08:01 +0100612 cpu_maps_update_begin();
Neil Brownbd5349c2006-10-17 00:10:35 -0700613 raw_notifier_chain_unregister(&cpu_chain, nb);
Gautham R Shenoyd2219382008-01-25 21:08:01 +0100614 cpu_maps_update_done();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700615}
616EXPORT_SYMBOL(unregister_cpu_notifier);
617
Mathias Krause71cf5ae2015-07-19 20:06:22 +0200618void __unregister_cpu_notifier(struct notifier_block *nb)
Srivatsa S. Bhat93ae4f92014-03-11 02:04:14 +0530619{
620 raw_notifier_chain_unregister(&cpu_chain, nb);
621}
622EXPORT_SYMBOL(__unregister_cpu_notifier);
623
Anton Vorontsove4cc2f82012-05-31 16:26:26 -0700624/**
625 * clear_tasks_mm_cpumask - Safely clear tasks' mm_cpumask for a CPU
626 * @cpu: a CPU id
627 *
628 * This function walks all processes, finds a valid mm struct for each one and
629 * then clears a corresponding bit in mm's cpumask. While this all sounds
630 * trivial, there are various non-obvious corner cases, which this function
631 * tries to solve in a safe manner.
632 *
633 * Also note that the function uses a somewhat relaxed locking scheme, so it may
634 * be called only for an already offlined CPU.
635 */
Anton Vorontsovcb792952012-05-31 16:26:22 -0700636void clear_tasks_mm_cpumask(int cpu)
637{
638 struct task_struct *p;
639
640 /*
641 * This function is called after the cpu is taken down and marked
642 * offline, so its not like new tasks will ever get this cpu set in
643 * their mm mask. -- Peter Zijlstra
644 * Thus, we may use rcu_read_lock() here, instead of grabbing
645 * full-fledged tasklist_lock.
646 */
Anton Vorontsove4cc2f82012-05-31 16:26:26 -0700647 WARN_ON(cpu_online(cpu));
Anton Vorontsovcb792952012-05-31 16:26:22 -0700648 rcu_read_lock();
649 for_each_process(p) {
650 struct task_struct *t;
651
Anton Vorontsove4cc2f82012-05-31 16:26:26 -0700652 /*
653 * Main thread might exit, but other threads may still have
654 * a valid mm. Find one.
655 */
Anton Vorontsovcb792952012-05-31 16:26:22 -0700656 t = find_lock_task_mm(p);
657 if (!t)
658 continue;
659 cpumask_clear_cpu(cpu, mm_cpumask(t->mm));
660 task_unlock(t);
661 }
662 rcu_read_unlock();
663}
664
Kirill Tkhaib728ca02014-06-25 12:19:55 +0400665static inline void check_for_tasks(int dead_cpu)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700666{
Kirill Tkhaib728ca02014-06-25 12:19:55 +0400667 struct task_struct *g, *p;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700668
Oleg Nesterova75a6062015-09-10 15:07:50 +0200669 read_lock(&tasklist_lock);
670 for_each_process_thread(g, p) {
Kirill Tkhaib728ca02014-06-25 12:19:55 +0400671 if (!p->on_rq)
672 continue;
673 /*
674 * We do the check with unlocked task_rq(p)->lock.
675 * Order the reading to do not warn about a task,
676 * which was running on this cpu in the past, and
677 * it's just been woken on another cpu.
678 */
679 rmb();
680 if (task_cpu(p) != dead_cpu)
681 continue;
682
683 pr_warn("Task %s (pid=%d) is on cpu %d (state=%ld, flags=%x)\n",
684 p->comm, task_pid_nr(p), dead_cpu, p->state, p->flags);
Oleg Nesterova75a6062015-09-10 15:07:50 +0200685 }
686 read_unlock(&tasklist_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700687}
688
Thomas Gleixner98458172016-02-26 18:43:25 +0000689static int notify_down_prepare(unsigned int cpu)
690{
691 int err, nr_calls = 0;
692
693 err = __cpu_notify(CPU_DOWN_PREPARE, cpu, -1, &nr_calls);
694 if (err) {
695 nr_calls--;
696 __cpu_notify(CPU_DOWN_FAILED, cpu, nr_calls, NULL);
697 pr_warn("%s: attempt to take down CPU %u failed\n",
698 __func__, cpu);
699 }
700 return err;
701}
702
Thomas Gleixner4baa0af2016-02-26 18:43:29 +0000703static int notify_dying(unsigned int cpu)
704{
705 cpu_notify(CPU_DYING, cpu);
706 return 0;
707}
708
Linus Torvalds1da177e2005-04-16 15:20:36 -0700709/* Take this CPU down. */
Mathias Krause71cf5ae2015-07-19 20:06:22 +0200710static int take_cpu_down(void *_param)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700711{
Thomas Gleixner4baa0af2016-02-26 18:43:29 +0000712 struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);
713 enum cpuhp_state target = max((int)st->target, CPUHP_AP_OFFLINE);
Thomas Gleixner090e77c2016-02-26 18:43:23 +0000714 int err, cpu = smp_processor_id();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700715
Linus Torvalds1da177e2005-04-16 15:20:36 -0700716 /* Ensure this CPU doesn't handle any more interrupts. */
717 err = __cpu_disable();
718 if (err < 0)
Zwane Mwaikambof3705132005-06-25 14:54:50 -0700719 return err;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700720
Thomas Gleixnera7246322016-08-12 19:49:38 +0200721 /*
722 * We get here while we are in CPUHP_TEARDOWN_CPU state and we must not
723 * do this step again.
724 */
725 WARN_ON(st->state != CPUHP_TEARDOWN_CPU);
726 st->state--;
Thomas Gleixner4baa0af2016-02-26 18:43:29 +0000727 /* Invoke the former CPU_DYING callbacks */
Thomas Gleixnera7246322016-08-12 19:49:38 +0200728 for (; st->state > target; st->state--)
729 cpuhp_invoke_callback(cpu, st->state, false);
Thomas Gleixner4baa0af2016-02-26 18:43:29 +0000730
Thomas Gleixner52c063d2015-04-03 02:37:24 +0200731 /* Give up timekeeping duties */
732 tick_handover_do_timer();
Thomas Gleixner14e568e2013-01-31 12:11:14 +0000733 /* Park the stopper thread */
Thomas Gleixner090e77c2016-02-26 18:43:23 +0000734 stop_machine_park(cpu);
Zwane Mwaikambof3705132005-06-25 14:54:50 -0700735 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700736}
737
Thomas Gleixner98458172016-02-26 18:43:25 +0000738static int takedown_cpu(unsigned int cpu)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700739{
Thomas Gleixnere69aab12016-02-26 18:43:43 +0000740 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
Thomas Gleixner98458172016-02-26 18:43:25 +0000741 int err;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700742
Thomas Gleixner2a58c522016-03-10 20:42:08 +0100743 /* Park the smpboot threads */
Thomas Gleixner1cf4f622016-02-26 18:43:39 +0000744 kthread_park(per_cpu_ptr(&cpuhp_state, cpu)->thread);
Thomas Gleixner2a58c522016-03-10 20:42:08 +0100745 smpboot_park_threads(cpu);
Thomas Gleixner1cf4f622016-02-26 18:43:39 +0000746
Peter Zijlstra6acce3e2013-10-11 14:38:20 +0200747 /*
Thomas Gleixnera8994182015-07-05 17:12:30 +0000748 * Prevent irq alloc/free while the dying cpu reorganizes the
749 * interrupt affinities.
750 */
751 irq_lock_sparse();
752
753 /*
Peter Zijlstra6acce3e2013-10-11 14:38:20 +0200754 * So now all preempt/rcu users must observe !cpu_active().
755 */
Thomas Gleixner090e77c2016-02-26 18:43:23 +0000756 err = stop_machine(take_cpu_down, NULL, cpumask_of(cpu));
Rusty Russell04321582008-07-28 12:16:29 -0500757 if (err) {
Sebastian Andrzej Siewior3b9d6da2016-04-08 14:40:15 +0200758 /* CPU refused to die */
Thomas Gleixnera8994182015-07-05 17:12:30 +0000759 irq_unlock_sparse();
Sebastian Andrzej Siewior3b9d6da2016-04-08 14:40:15 +0200760 /* Unpark the hotplug thread so we can rollback there */
761 kthread_unpark(per_cpu_ptr(&cpuhp_state, cpu)->thread);
Thomas Gleixner98458172016-02-26 18:43:25 +0000762 return err;
Satoru Takeuchi8fa1d7d2006-10-28 10:38:57 -0700763 }
Rusty Russell04321582008-07-28 12:16:29 -0500764 BUG_ON(cpu_online(cpu));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700765
Peter Zijlstra48c5ccae2010-11-13 19:32:29 +0100766 /*
767 * The migration_call() CPU_DYING callback will have removed all
768 * runnable tasks from the cpu, there's only the idle task left now
769 * that the migration thread is done doing the stop_machine thing.
Peter Zijlstra51a96c72010-11-19 20:37:53 +0100770 *
771 * Wait for the stop thread to go away.
Peter Zijlstra48c5ccae2010-11-13 19:32:29 +0100772 */
Thomas Gleixnere69aab12016-02-26 18:43:43 +0000773 wait_for_completion(&st->done);
774 BUG_ON(st->state != CPUHP_AP_IDLE_DEAD);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700775
Thomas Gleixnera8994182015-07-05 17:12:30 +0000776 /* Interrupts are moved away from the dying cpu, reenable alloc/free */
777 irq_unlock_sparse();
778
Preeti U Murthy345527b2015-03-30 14:59:19 +0530779 hotplug_cpu__broadcast_tick_pull(cpu);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700780 /* This actually kills the CPU. */
781 __cpu_die(cpu);
782
Thomas Gleixnera49b1162015-04-03 02:38:05 +0200783 tick_cleanup_dead_cpu(cpu);
Thomas Gleixner98458172016-02-26 18:43:25 +0000784 return 0;
785}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700786
Thomas Gleixner98458172016-02-26 18:43:25 +0000787static int notify_dead(unsigned int cpu)
788{
789 cpu_notify_nofail(CPU_DEAD, cpu);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700790 check_for_tasks(cpu);
Thomas Gleixner98458172016-02-26 18:43:25 +0000791 return 0;
792}
793
Thomas Gleixner71f87b22016-03-03 10:52:10 +0100794static void cpuhp_complete_idle_dead(void *arg)
795{
796 struct cpuhp_cpu_state *st = arg;
797
798 complete(&st->done);
799}
800
Thomas Gleixnere69aab12016-02-26 18:43:43 +0000801void cpuhp_report_idle_dead(void)
802{
803 struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);
804
805 BUG_ON(st->state != CPUHP_AP_OFFLINE);
Thomas Gleixner27d50c72016-02-26 18:43:44 +0000806 rcu_report_dead(smp_processor_id());
Thomas Gleixner71f87b22016-03-03 10:52:10 +0100807 st->state = CPUHP_AP_IDLE_DEAD;
808 /*
809 * We cannot call complete after rcu_report_dead() so we delegate it
810 * to an online cpu.
811 */
812 smp_call_function_single(cpumask_first(cpu_online_mask),
813 cpuhp_complete_idle_dead, st, 0);
Thomas Gleixnere69aab12016-02-26 18:43:43 +0000814}
815
Thomas Gleixnercff7d372016-02-26 18:43:28 +0000816#else
817#define notify_down_prepare NULL
818#define takedown_cpu NULL
819#define notify_dead NULL
Thomas Gleixner4baa0af2016-02-26 18:43:29 +0000820#define notify_dying NULL
Thomas Gleixnercff7d372016-02-26 18:43:28 +0000821#endif
822
823#ifdef CONFIG_HOTPLUG_CPU
Thomas Gleixnercff7d372016-02-26 18:43:28 +0000824
Thomas Gleixner98458172016-02-26 18:43:25 +0000825/* Requires cpu_add_remove_lock to be held */
Thomas Gleixneraf1f4042016-02-26 18:43:30 +0000826static int __ref _cpu_down(unsigned int cpu, int tasks_frozen,
827 enum cpuhp_state target)
Thomas Gleixner98458172016-02-26 18:43:25 +0000828{
Thomas Gleixnercff7d372016-02-26 18:43:28 +0000829 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
830 int prev_state, ret = 0;
831 bool hasdied = false;
Thomas Gleixner98458172016-02-26 18:43:25 +0000832
833 if (num_online_cpus() == 1)
834 return -EBUSY;
835
Thomas Gleixner757c9892016-02-26 18:43:32 +0000836 if (!cpu_present(cpu))
Thomas Gleixner98458172016-02-26 18:43:25 +0000837 return -EINVAL;
838
839 cpu_hotplug_begin();
840
841 cpuhp_tasks_frozen = tasks_frozen;
842
Thomas Gleixnercff7d372016-02-26 18:43:28 +0000843 prev_state = st->state;
Thomas Gleixneraf1f4042016-02-26 18:43:30 +0000844 st->target = target;
Thomas Gleixner1cf4f622016-02-26 18:43:39 +0000845 /*
846 * If the current CPU state is in the range of the AP hotplug thread,
847 * then we need to kick the thread.
848 */
Thomas Gleixner8df3e072016-02-26 18:43:41 +0000849 if (st->state > CPUHP_TEARDOWN_CPU) {
Thomas Gleixner1cf4f622016-02-26 18:43:39 +0000850 ret = cpuhp_kick_ap_work(cpu);
851 /*
852 * The AP side has done the error rollback already. Just
853 * return the error code..
854 */
855 if (ret)
856 goto out;
857
858 /*
859 * We might have stopped still in the range of the AP hotplug
860 * thread. Nothing to do anymore.
861 */
Thomas Gleixner8df3e072016-02-26 18:43:41 +0000862 if (st->state > CPUHP_TEARDOWN_CPU)
Thomas Gleixner1cf4f622016-02-26 18:43:39 +0000863 goto out;
864 }
865 /*
Thomas Gleixner8df3e072016-02-26 18:43:41 +0000866 * The AP brought itself down to CPUHP_TEARDOWN_CPU. So we need
Thomas Gleixner1cf4f622016-02-26 18:43:39 +0000867 * to do the further cleanups.
868 */
Thomas Gleixnera7246322016-08-12 19:49:38 +0200869 ret = cpuhp_down_callbacks(cpu, st, target);
Sebastian Andrzej Siewior3b9d6da2016-04-08 14:40:15 +0200870 if (ret && st->state > CPUHP_TEARDOWN_CPU && st->state < prev_state) {
871 st->target = prev_state;
872 st->rollback = true;
873 cpuhp_kick_ap_work(cpu);
874 }
Thomas Gleixner98458172016-02-26 18:43:25 +0000875
Thomas Gleixnercff7d372016-02-26 18:43:28 +0000876 hasdied = prev_state != st->state && st->state == CPUHP_OFFLINE;
Thomas Gleixner1cf4f622016-02-26 18:43:39 +0000877out:
Gautham R Shenoyd2219382008-01-25 21:08:01 +0100878 cpu_hotplug_done();
Thomas Gleixnercff7d372016-02-26 18:43:28 +0000879 /* This post dead nonsense must die */
880 if (!ret && hasdied)
Thomas Gleixner090e77c2016-02-26 18:43:23 +0000881 cpu_notify_nofail(CPU_POST_DEAD, cpu);
Thomas Gleixnercff7d372016-02-26 18:43:28 +0000882 return ret;
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -0700883}
884
Thomas Gleixneraf1f4042016-02-26 18:43:30 +0000885static int do_cpu_down(unsigned int cpu, enum cpuhp_state target)
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -0700886{
Heiko Carstens9ea09af2008-12-22 12:36:30 +0100887 int err;
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -0700888
Gautham R Shenoyd2219382008-01-25 21:08:01 +0100889 cpu_maps_update_begin();
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -0700890
Max Krasnyanskye761b772008-07-15 04:43:49 -0700891 if (cpu_hotplug_disabled) {
892 err = -EBUSY;
893 goto out;
894 }
895
Thomas Gleixneraf1f4042016-02-26 18:43:30 +0000896 err = _cpu_down(cpu, 0, target);
Max Krasnyanskye761b772008-07-15 04:43:49 -0700897
Max Krasnyanskye761b772008-07-15 04:43:49 -0700898out:
Gautham R Shenoyd2219382008-01-25 21:08:01 +0100899 cpu_maps_update_done();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700900 return err;
901}
Thomas Gleixneraf1f4042016-02-26 18:43:30 +0000902int cpu_down(unsigned int cpu)
903{
904 return do_cpu_down(cpu, CPUHP_OFFLINE);
905}
Zhang Ruib62b8ef2008-04-29 02:35:56 -0400906EXPORT_SYMBOL(cpu_down);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700907#endif /*CONFIG_HOTPLUG_CPU*/
908
Thomas Gleixner4baa0af2016-02-26 18:43:29 +0000909/**
910 * notify_cpu_starting(cpu) - call the CPU_STARTING notifiers
911 * @cpu: cpu that just started
912 *
913 * This function calls the cpu_chain notifiers with CPU_STARTING.
914 * It must be called by the arch code on the new cpu, before the new cpu
915 * enables interrupts and before the "boot" cpu returns from __cpu_up().
916 */
917void notify_cpu_starting(unsigned int cpu)
918{
919 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
920 enum cpuhp_state target = min((int)st->target, CPUHP_AP_ONLINE);
921
922 while (st->state < target) {
Thomas Gleixner4baa0af2016-02-26 18:43:29 +0000923 st->state++;
Thomas Gleixnera7246322016-08-12 19:49:38 +0200924 cpuhp_invoke_callback(cpu, st->state, true);
Thomas Gleixner4baa0af2016-02-26 18:43:29 +0000925 }
926}
927
Thomas Gleixner949338e2016-02-26 18:43:35 +0000928/*
929 * Called from the idle task. We need to set active here, so we can kick off
Thomas Gleixner8df3e072016-02-26 18:43:41 +0000930 * the stopper thread and unpark the smpboot threads. If the target state is
931 * beyond CPUHP_AP_ONLINE_IDLE we kick cpuhp thread and let it bring up the
932 * cpu further.
Thomas Gleixner949338e2016-02-26 18:43:35 +0000933 */
Thomas Gleixner8df3e072016-02-26 18:43:41 +0000934void cpuhp_online_idle(enum cpuhp_state state)
Thomas Gleixner949338e2016-02-26 18:43:35 +0000935{
Thomas Gleixner8df3e072016-02-26 18:43:41 +0000936 struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);
937 unsigned int cpu = smp_processor_id();
938
939 /* Happens for the boot cpu */
940 if (state != CPUHP_AP_ONLINE_IDLE)
941 return;
942
943 st->state = CPUHP_AP_ONLINE_IDLE;
Thomas Gleixner1cf4f622016-02-26 18:43:39 +0000944
Thomas Gleixner8df3e072016-02-26 18:43:41 +0000945 /* Unpark the stopper thread and the hotplug thread of this cpu */
Thomas Gleixner949338e2016-02-26 18:43:35 +0000946 stop_machine_unpark(cpu);
Thomas Gleixner1cf4f622016-02-26 18:43:39 +0000947 kthread_unpark(st->thread);
Thomas Gleixner8df3e072016-02-26 18:43:41 +0000948
949 /* Should we go further up ? */
950 if (st->target > CPUHP_AP_ONLINE_IDLE)
951 __cpuhp_kick_ap_work(st);
952 else
953 complete(&st->done);
Thomas Gleixner949338e2016-02-26 18:43:35 +0000954}
955
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -0700956/* Requires cpu_add_remove_lock to be held */
Thomas Gleixneraf1f4042016-02-26 18:43:30 +0000957static int _cpu_up(unsigned int cpu, int tasks_frozen, enum cpuhp_state target)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700958{
Thomas Gleixnercff7d372016-02-26 18:43:28 +0000959 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
Suresh Siddha3bb5d2e2012-04-20 17:08:50 -0700960 struct task_struct *idle;
Thomas Gleixner2e1a3482016-02-26 18:43:37 +0000961 int ret = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700962
Gautham R Shenoyd2219382008-01-25 21:08:01 +0100963 cpu_hotplug_begin();
Thomas Gleixner38498a62012-04-20 13:05:44 +0000964
Thomas Gleixner757c9892016-02-26 18:43:32 +0000965 if (!cpu_present(cpu)) {
Yasuaki Ishimatsu5e5041f2012-10-23 01:30:54 +0200966 ret = -EINVAL;
967 goto out;
968 }
969
Thomas Gleixner757c9892016-02-26 18:43:32 +0000970 /*
971 * The caller of do_cpu_up might have raced with another
972 * caller. Ignore it for now.
973 */
974 if (st->state >= target)
Thomas Gleixner38498a62012-04-20 13:05:44 +0000975 goto out;
Thomas Gleixner757c9892016-02-26 18:43:32 +0000976
977 if (st->state == CPUHP_OFFLINE) {
978 /* Let it fail before we try to bring the cpu up */
979 idle = idle_thread_get(cpu);
980 if (IS_ERR(idle)) {
981 ret = PTR_ERR(idle);
982 goto out;
983 }
Suresh Siddha3bb5d2e2012-04-20 17:08:50 -0700984 }
Thomas Gleixner38498a62012-04-20 13:05:44 +0000985
Thomas Gleixnerba997462016-02-26 18:43:24 +0000986 cpuhp_tasks_frozen = tasks_frozen;
987
Thomas Gleixneraf1f4042016-02-26 18:43:30 +0000988 st->target = target;
Thomas Gleixner1cf4f622016-02-26 18:43:39 +0000989 /*
990 * If the current CPU state is in the range of the AP hotplug thread,
991 * then we need to kick the thread once more.
992 */
Thomas Gleixner8df3e072016-02-26 18:43:41 +0000993 if (st->state > CPUHP_BRINGUP_CPU) {
Thomas Gleixner1cf4f622016-02-26 18:43:39 +0000994 ret = cpuhp_kick_ap_work(cpu);
995 /*
996 * The AP side has done the error rollback already. Just
997 * return the error code..
998 */
999 if (ret)
1000 goto out;
1001 }
1002
1003 /*
1004 * Try to reach the target state. We max out on the BP at
Thomas Gleixner8df3e072016-02-26 18:43:41 +00001005 * CPUHP_BRINGUP_CPU. After that the AP hotplug thread is
Thomas Gleixner1cf4f622016-02-26 18:43:39 +00001006 * responsible for bringing it up to the target state.
1007 */
Thomas Gleixner8df3e072016-02-26 18:43:41 +00001008 target = min((int)target, CPUHP_BRINGUP_CPU);
Thomas Gleixnera7246322016-08-12 19:49:38 +02001009 ret = cpuhp_up_callbacks(cpu, st, target);
Thomas Gleixner38498a62012-04-20 13:05:44 +00001010out:
Gautham R Shenoyd2219382008-01-25 21:08:01 +01001011 cpu_hotplug_done();
Linus Torvalds1da177e2005-04-16 15:20:36 -07001012 return ret;
1013}
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -07001014
Thomas Gleixneraf1f4042016-02-26 18:43:30 +00001015static int do_cpu_up(unsigned int cpu, enum cpuhp_state target)
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -07001016{
1017 int err = 0;
minskey guocf234222010-05-24 14:32:41 -07001018
Rusty Russelle0b582e2009-01-01 10:12:28 +10301019 if (!cpu_possible(cpu)) {
Fabian Frederick84117da2014-06-04 16:11:17 -07001020 pr_err("can't online cpu %d because it is not configured as may-hotadd at boot time\n",
1021 cpu);
Chen Gong87d5e0232010-03-05 13:42:38 -08001022#if defined(CONFIG_IA64)
Fabian Frederick84117da2014-06-04 16:11:17 -07001023 pr_err("please check additional_cpus= boot parameter\n");
KAMEZAWA Hiroyuki73e753a2007-10-18 23:40:47 -07001024#endif
1025 return -EINVAL;
1026 }
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -07001027
Toshi Kani01b0f192013-11-12 15:07:25 -08001028 err = try_online_node(cpu_to_node(cpu));
1029 if (err)
1030 return err;
minskey guocf234222010-05-24 14:32:41 -07001031
Gautham R Shenoyd2219382008-01-25 21:08:01 +01001032 cpu_maps_update_begin();
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -07001033
Max Krasnyanskye761b772008-07-15 04:43:49 -07001034 if (cpu_hotplug_disabled) {
1035 err = -EBUSY;
1036 goto out;
1037 }
1038
Thomas Gleixneraf1f4042016-02-26 18:43:30 +00001039 err = _cpu_up(cpu, 0, target);
Max Krasnyanskye761b772008-07-15 04:43:49 -07001040out:
Gautham R Shenoyd2219382008-01-25 21:08:01 +01001041 cpu_maps_update_done();
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -07001042 return err;
1043}
Thomas Gleixneraf1f4042016-02-26 18:43:30 +00001044
1045int cpu_up(unsigned int cpu)
1046{
1047 return do_cpu_up(cpu, CPUHP_ONLINE);
1048}
Paul E. McKenneya513f6b2011-12-11 21:54:45 -08001049EXPORT_SYMBOL_GPL(cpu_up);
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -07001050
Rafael J. Wysockif3de4be2007-08-30 23:56:29 -07001051#ifdef CONFIG_PM_SLEEP_SMP
Rusty Russelle0b582e2009-01-01 10:12:28 +10301052static cpumask_var_t frozen_cpus;
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -07001053
1054int disable_nonboot_cpus(void)
1055{
Rafael J. Wysockie9a5f422010-05-27 22:16:22 +02001056 int cpu, first_cpu, error = 0;
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -07001057
Gautham R Shenoyd2219382008-01-25 21:08:01 +01001058 cpu_maps_update_begin();
Rusty Russelle0b582e2009-01-01 10:12:28 +10301059 first_cpu = cpumask_first(cpu_online_mask);
Xiaotian Feng9ee349a2009-12-16 18:04:32 +01001060 /*
1061 * We take down all of the non-boot CPUs in one shot to avoid races
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -07001062 * with the userspace trying to use the CPU hotplug at the same time
1063 */
Rusty Russelle0b582e2009-01-01 10:12:28 +10301064 cpumask_clear(frozen_cpus);
Peter Zijlstra6ad4c182009-11-25 13:31:39 +01001065
Fabian Frederick84117da2014-06-04 16:11:17 -07001066 pr_info("Disabling non-boot CPUs ...\n");
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -07001067 for_each_online_cpu(cpu) {
1068 if (cpu == first_cpu)
1069 continue;
Todd E Brandtbb3632c2014-06-06 05:40:17 -07001070 trace_suspend_resume(TPS("CPU_OFF"), cpu, true);
Thomas Gleixneraf1f4042016-02-26 18:43:30 +00001071 error = _cpu_down(cpu, 1, CPUHP_OFFLINE);
Todd E Brandtbb3632c2014-06-06 05:40:17 -07001072 trace_suspend_resume(TPS("CPU_OFF"), cpu, false);
Mike Travisfeae3202009-11-17 18:22:13 -06001073 if (!error)
Rusty Russelle0b582e2009-01-01 10:12:28 +10301074 cpumask_set_cpu(cpu, frozen_cpus);
Mike Travisfeae3202009-11-17 18:22:13 -06001075 else {
Fabian Frederick84117da2014-06-04 16:11:17 -07001076 pr_err("Error taking CPU%d down: %d\n", cpu, error);
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -07001077 break;
1078 }
1079 }
Joseph Cihula86886e52009-06-30 19:31:07 -07001080
Vitaly Kuznetsov89af7ba2015-08-05 00:52:46 -07001081 if (!error)
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -07001082 BUG_ON(num_online_cpus() > 1);
Vitaly Kuznetsov89af7ba2015-08-05 00:52:46 -07001083 else
Fabian Frederick84117da2014-06-04 16:11:17 -07001084 pr_err("Non-boot CPUs are not disabled\n");
Vitaly Kuznetsov89af7ba2015-08-05 00:52:46 -07001085
1086 /*
1087 * Make sure the CPUs won't be enabled by someone else. We need to do
1088 * this even in case of failure as all disable_nonboot_cpus() users are
1089 * supposed to do enable_nonboot_cpus() on the failure path.
1090 */
1091 cpu_hotplug_disabled++;
1092
Gautham R Shenoyd2219382008-01-25 21:08:01 +01001093 cpu_maps_update_done();
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -07001094 return error;
1095}
1096
Suresh Siddhad0af9ee2009-08-19 18:05:36 -07001097void __weak arch_enable_nonboot_cpus_begin(void)
1098{
1099}
1100
1101void __weak arch_enable_nonboot_cpus_end(void)
1102{
1103}
1104
Mathias Krause71cf5ae2015-07-19 20:06:22 +02001105void enable_nonboot_cpus(void)
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -07001106{
1107 int cpu, error;
1108
1109 /* Allow everyone to use the CPU hotplug again */
Gautham R Shenoyd2219382008-01-25 21:08:01 +01001110 cpu_maps_update_begin();
Vitaly Kuznetsov89af7ba2015-08-05 00:52:46 -07001111 WARN_ON(--cpu_hotplug_disabled < 0);
Rusty Russelle0b582e2009-01-01 10:12:28 +10301112 if (cpumask_empty(frozen_cpus))
Rafael J. Wysocki1d64b9c2007-04-01 23:49:49 -07001113 goto out;
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -07001114
Fabian Frederick84117da2014-06-04 16:11:17 -07001115 pr_info("Enabling non-boot CPUs ...\n");
Suresh Siddhad0af9ee2009-08-19 18:05:36 -07001116
1117 arch_enable_nonboot_cpus_begin();
1118
Rusty Russelle0b582e2009-01-01 10:12:28 +10301119 for_each_cpu(cpu, frozen_cpus) {
Todd E Brandtbb3632c2014-06-06 05:40:17 -07001120 trace_suspend_resume(TPS("CPU_ON"), cpu, true);
Thomas Gleixneraf1f4042016-02-26 18:43:30 +00001121 error = _cpu_up(cpu, 1, CPUHP_ONLINE);
Todd E Brandtbb3632c2014-06-06 05:40:17 -07001122 trace_suspend_resume(TPS("CPU_ON"), cpu, false);
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -07001123 if (!error) {
Fabian Frederick84117da2014-06-04 16:11:17 -07001124 pr_info("CPU%d is up\n", cpu);
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -07001125 continue;
1126 }
Fabian Frederick84117da2014-06-04 16:11:17 -07001127 pr_warn("Error taking CPU%d up: %d\n", cpu, error);
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -07001128 }
Suresh Siddhad0af9ee2009-08-19 18:05:36 -07001129
1130 arch_enable_nonboot_cpus_end();
1131
Rusty Russelle0b582e2009-01-01 10:12:28 +10301132 cpumask_clear(frozen_cpus);
Rafael J. Wysocki1d64b9c2007-04-01 23:49:49 -07001133out:
Gautham R Shenoyd2219382008-01-25 21:08:01 +01001134 cpu_maps_update_done();
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -07001135}
Rusty Russelle0b582e2009-01-01 10:12:28 +10301136
Fenghua Yud7268a32011-11-15 21:59:31 +01001137static int __init alloc_frozen_cpus(void)
Rusty Russelle0b582e2009-01-01 10:12:28 +10301138{
1139 if (!alloc_cpumask_var(&frozen_cpus, GFP_KERNEL|__GFP_ZERO))
1140 return -ENOMEM;
1141 return 0;
1142}
1143core_initcall(alloc_frozen_cpus);
Srivatsa S. Bhat79cfbdf2011-11-03 00:59:25 +01001144
1145/*
Srivatsa S. Bhat79cfbdf2011-11-03 00:59:25 +01001146 * When callbacks for CPU hotplug notifications are being executed, we must
1147 * ensure that the state of the system with respect to the tasks being frozen
1148 * or not, as reported by the notification, remains unchanged *throughout the
1149 * duration* of the execution of the callbacks.
1150 * Hence we need to prevent the freezer from racing with regular CPU hotplug.
1151 *
1152 * This synchronization is implemented by mutually excluding regular CPU
1153 * hotplug and Suspend/Hibernate call paths by hooking onto the Suspend/
1154 * Hibernate notifications.
1155 */
1156static int
1157cpu_hotplug_pm_callback(struct notifier_block *nb,
1158 unsigned long action, void *ptr)
1159{
1160 switch (action) {
1161
1162 case PM_SUSPEND_PREPARE:
1163 case PM_HIBERNATION_PREPARE:
Srivatsa S. Bhat16e53db2013-06-12 14:04:36 -07001164 cpu_hotplug_disable();
Srivatsa S. Bhat79cfbdf2011-11-03 00:59:25 +01001165 break;
1166
1167 case PM_POST_SUSPEND:
1168 case PM_POST_HIBERNATION:
Srivatsa S. Bhat16e53db2013-06-12 14:04:36 -07001169 cpu_hotplug_enable();
Srivatsa S. Bhat79cfbdf2011-11-03 00:59:25 +01001170 break;
1171
1172 default:
1173 return NOTIFY_DONE;
1174 }
1175
1176 return NOTIFY_OK;
1177}
1178
1179
Fenghua Yud7268a32011-11-15 21:59:31 +01001180static int __init cpu_hotplug_pm_sync_init(void)
Srivatsa S. Bhat79cfbdf2011-11-03 00:59:25 +01001181{
Fenghua Yu6e32d472012-11-13 11:32:43 -08001182 /*
1183 * cpu_hotplug_pm_callback has higher priority than x86
1184 * bsp_pm_callback which depends on cpu_hotplug_pm_callback
1185 * to disable cpu hotplug to avoid cpu hotplug race.
1186 */
Srivatsa S. Bhat79cfbdf2011-11-03 00:59:25 +01001187 pm_notifier(cpu_hotplug_pm_callback, 0);
1188 return 0;
1189}
1190core_initcall(cpu_hotplug_pm_sync_init);
1191
Rafael J. Wysockif3de4be2007-08-30 23:56:29 -07001192#endif /* CONFIG_PM_SLEEP_SMP */
Max Krasnyansky68f4f1e2008-05-29 11:17:02 -07001193
1194#endif /* CONFIG_SMP */
Mike Travisb8d317d2008-07-24 18:21:29 -07001195
Thomas Gleixnercff7d372016-02-26 18:43:28 +00001196/* Boot processor state steps */
1197static struct cpuhp_step cpuhp_bp_states[] = {
1198 [CPUHP_OFFLINE] = {
1199 .name = "offline",
1200 .startup = NULL,
1201 .teardown = NULL,
1202 },
1203#ifdef CONFIG_SMP
1204 [CPUHP_CREATE_THREADS]= {
1205 .name = "threads:create",
1206 .startup = smpboot_create_threads,
1207 .teardown = NULL,
Thomas Gleixner757c9892016-02-26 18:43:32 +00001208 .cant_stop = true,
Thomas Gleixnercff7d372016-02-26 18:43:28 +00001209 },
Thomas Gleixner00e16c32016-07-13 17:16:09 +00001210 [CPUHP_PERF_PREPARE] = {
1211 .name = "perf prepare",
1212 .startup = perf_event_init_cpu,
1213 .teardown = perf_event_exit_cpu,
1214 },
Thomas Gleixner7ee681b2016-07-13 17:16:29 +00001215 [CPUHP_WORKQUEUE_PREP] = {
1216 .name = "workqueue prepare",
1217 .startup = workqueue_prepare_cpu,
1218 .teardown = NULL,
1219 },
Thomas Gleixner27590dc2016-07-15 10:41:04 +02001220 [CPUHP_HRTIMERS_PREPARE] = {
1221 .name = "hrtimers prepare",
1222 .startup = hrtimers_prepare_cpu,
1223 .teardown = hrtimers_dead_cpu,
1224 },
Richard Weinberger31487f82016-07-13 17:17:01 +00001225 [CPUHP_SMPCFD_PREPARE] = {
1226 .name = "SMPCFD prepare",
1227 .startup = smpcfd_prepare_cpu,
1228 .teardown = smpcfd_dead_cpu,
1229 },
Thomas Gleixner4df83742016-07-13 17:17:03 +00001230 [CPUHP_RCUTREE_PREP] = {
1231 .name = "RCU-tree prepare",
1232 .startup = rcutree_prepare_cpu,
1233 .teardown = rcutree_dead_cpu,
1234 },
Thomas Gleixnerd10ef6f2016-03-08 10:36:13 +01001235 /*
1236 * Preparatory and dead notifiers. Will be replaced once the notifiers
1237 * are converted to states.
1238 */
Thomas Gleixnercff7d372016-02-26 18:43:28 +00001239 [CPUHP_NOTIFY_PREPARE] = {
1240 .name = "notify:prepare",
1241 .startup = notify_prepare,
1242 .teardown = notify_dead,
1243 .skip_onerr = true,
Thomas Gleixner757c9892016-02-26 18:43:32 +00001244 .cant_stop = true,
Thomas Gleixnercff7d372016-02-26 18:43:28 +00001245 },
Richard Cochran4fae16d2016-07-27 11:08:18 +02001246 /*
1247 * On the tear-down path, timers_dead_cpu() must be invoked
1248 * before blk_mq_queue_reinit_notify() from notify_dead(),
1249 * otherwise a RCU stall occurs.
1250 */
1251 [CPUHP_TIMERS_DEAD] = {
1252 .name = "timers dead",
1253 .startup = NULL,
1254 .teardown = timers_dead_cpu,
1255 },
Thomas Gleixnerd10ef6f2016-03-08 10:36:13 +01001256 /* Kicks the plugged cpu into life */
Thomas Gleixnercff7d372016-02-26 18:43:28 +00001257 [CPUHP_BRINGUP_CPU] = {
1258 .name = "cpu:bringup",
1259 .startup = bringup_cpu,
Thomas Gleixner4baa0af2016-02-26 18:43:29 +00001260 .teardown = NULL,
Thomas Gleixner757c9892016-02-26 18:43:32 +00001261 .cant_stop = true,
Thomas Gleixner4baa0af2016-02-26 18:43:29 +00001262 },
Richard Weinberger31487f82016-07-13 17:17:01 +00001263 [CPUHP_AP_SMPCFD_DYING] = {
1264 .startup = NULL,
1265 .teardown = smpcfd_dying_cpu,
1266 },
Thomas Gleixnerd10ef6f2016-03-08 10:36:13 +01001267 /*
1268 * Handled on controll processor until the plugged processor manages
1269 * this itself.
1270 */
Thomas Gleixner4baa0af2016-02-26 18:43:29 +00001271 [CPUHP_TEARDOWN_CPU] = {
1272 .name = "cpu:teardown",
1273 .startup = NULL,
Thomas Gleixnercff7d372016-02-26 18:43:28 +00001274 .teardown = takedown_cpu,
Thomas Gleixner757c9892016-02-26 18:43:32 +00001275 .cant_stop = true,
Thomas Gleixnercff7d372016-02-26 18:43:28 +00001276 },
Thomas Gleixnera7c734142016-07-12 21:59:23 +02001277#else
1278 [CPUHP_BRINGUP_CPU] = { },
Thomas Gleixnercff7d372016-02-26 18:43:28 +00001279#endif
Thomas Gleixnercff7d372016-02-26 18:43:28 +00001280};
1281
Thomas Gleixner4baa0af2016-02-26 18:43:29 +00001282/* Application processor state steps */
1283static struct cpuhp_step cpuhp_ap_states[] = {
1284#ifdef CONFIG_SMP
Thomas Gleixnerd10ef6f2016-03-08 10:36:13 +01001285 /* Final state before CPU kills itself */
1286 [CPUHP_AP_IDLE_DEAD] = {
1287 .name = "idle:dead",
1288 },
1289 /*
1290 * Last state before CPU enters the idle loop to die. Transient state
1291 * for synchronization.
1292 */
1293 [CPUHP_AP_OFFLINE] = {
1294 .name = "ap:offline",
1295 .cant_stop = true,
1296 },
Thomas Gleixner9cf72432016-03-10 12:54:09 +01001297 /* First state is scheduler control. Interrupts are disabled */
1298 [CPUHP_AP_SCHED_STARTING] = {
1299 .name = "sched:starting",
1300 .startup = sched_cpu_starting,
Thomas Gleixnerf2785dd2016-03-10 12:54:18 +01001301 .teardown = sched_cpu_dying,
Thomas Gleixner9cf72432016-03-10 12:54:09 +01001302 },
Thomas Gleixner4df83742016-07-13 17:17:03 +00001303 [CPUHP_AP_RCUTREE_DYING] = {
1304 .startup = NULL,
1305 .teardown = rcutree_dying_cpu,
1306 },
Thomas Gleixnerd10ef6f2016-03-08 10:36:13 +01001307 /*
1308 * Low level startup/teardown notifiers. Run with interrupts
1309 * disabled. Will be removed once the notifiers are converted to
1310 * states.
1311 */
Thomas Gleixner4baa0af2016-02-26 18:43:29 +00001312 [CPUHP_AP_NOTIFY_STARTING] = {
1313 .name = "notify:starting",
1314 .startup = notify_starting,
1315 .teardown = notify_dying,
1316 .skip_onerr = true,
Thomas Gleixner757c9892016-02-26 18:43:32 +00001317 .cant_stop = true,
Thomas Gleixner4baa0af2016-02-26 18:43:29 +00001318 },
Thomas Gleixnerd10ef6f2016-03-08 10:36:13 +01001319 /* Entry state on starting. Interrupts enabled from here on. Transient
1320 * state for synchronsization */
1321 [CPUHP_AP_ONLINE] = {
1322 .name = "ap:online",
1323 },
1324 /* Handle smpboot threads park/unpark */
Thomas Gleixner1cf4f622016-02-26 18:43:39 +00001325 [CPUHP_AP_SMPBOOT_THREADS] = {
1326 .name = "smpboot:threads",
1327 .startup = smpboot_unpark_threads,
Thomas Gleixner2a58c522016-03-10 20:42:08 +01001328 .teardown = NULL,
Thomas Gleixner1cf4f622016-02-26 18:43:39 +00001329 },
Thomas Gleixner00e16c32016-07-13 17:16:09 +00001330 [CPUHP_AP_PERF_ONLINE] = {
1331 .name = "perf online",
1332 .startup = perf_event_init_cpu,
1333 .teardown = perf_event_exit_cpu,
1334 },
Thomas Gleixner7ee681b2016-07-13 17:16:29 +00001335 [CPUHP_AP_WORKQUEUE_ONLINE] = {
1336 .name = "workqueue online",
1337 .startup = workqueue_online_cpu,
1338 .teardown = workqueue_offline_cpu,
1339 },
Thomas Gleixner4df83742016-07-13 17:17:03 +00001340 [CPUHP_AP_RCUTREE_ONLINE] = {
1341 .name = "RCU-tree online",
1342 .startup = rcutree_online_cpu,
1343 .teardown = rcutree_offline_cpu,
1344 },
Thomas Gleixner00e16c32016-07-13 17:16:09 +00001345
Thomas Gleixnerd10ef6f2016-03-08 10:36:13 +01001346 /*
1347 * Online/down_prepare notifiers. Will be removed once the notifiers
1348 * are converted to states.
1349 */
Thomas Gleixner1cf4f622016-02-26 18:43:39 +00001350 [CPUHP_AP_NOTIFY_ONLINE] = {
1351 .name = "notify:online",
1352 .startup = notify_online,
1353 .teardown = notify_down_prepare,
Sebastian Andrzej Siewior3b9d6da2016-04-08 14:40:15 +02001354 .skip_onerr = true,
Thomas Gleixner1cf4f622016-02-26 18:43:39 +00001355 },
Thomas Gleixner4baa0af2016-02-26 18:43:29 +00001356#endif
Thomas Gleixnerd10ef6f2016-03-08 10:36:13 +01001357 /*
1358 * The dynamically registered state space is here
1359 */
1360
Thomas Gleixneraaddd7d2016-03-10 12:54:19 +01001361#ifdef CONFIG_SMP
1362 /* Last state is scheduler control setting the cpu active */
1363 [CPUHP_AP_ACTIVE] = {
1364 .name = "sched:active",
1365 .startup = sched_cpu_activate,
1366 .teardown = sched_cpu_deactivate,
1367 },
1368#endif
1369
Thomas Gleixnerd10ef6f2016-03-08 10:36:13 +01001370 /* CPU is fully up and running. */
Thomas Gleixner4baa0af2016-02-26 18:43:29 +00001371 [CPUHP_ONLINE] = {
1372 .name = "online",
1373 .startup = NULL,
1374 .teardown = NULL,
1375 },
1376};
1377
Thomas Gleixner5b7aa872016-02-26 18:43:33 +00001378/* Sanity check for callbacks */
1379static int cpuhp_cb_check(enum cpuhp_state state)
1380{
1381 if (state <= CPUHP_OFFLINE || state >= CPUHP_ONLINE)
1382 return -EINVAL;
1383 return 0;
1384}
1385
Thomas Gleixner5b7aa872016-02-26 18:43:33 +00001386static void cpuhp_store_callbacks(enum cpuhp_state state,
1387 const char *name,
1388 int (*startup)(unsigned int cpu),
1389 int (*teardown)(unsigned int cpu))
1390{
1391 /* (Un)Install the callbacks for further cpu hotplug operations */
1392 struct cpuhp_step *sp;
1393
1394 mutex_lock(&cpuhp_state_mutex);
1395 sp = cpuhp_get_step(state);
1396 sp->startup = startup;
1397 sp->teardown = teardown;
1398 sp->name = name;
1399 mutex_unlock(&cpuhp_state_mutex);
1400}
1401
1402static void *cpuhp_get_teardown_cb(enum cpuhp_state state)
1403{
1404 return cpuhp_get_step(state)->teardown;
1405}
1406
Thomas Gleixner5b7aa872016-02-26 18:43:33 +00001407/*
1408 * Call the startup/teardown function for a step either on the AP or
1409 * on the current CPU.
1410 */
Thomas Gleixnera7246322016-08-12 19:49:38 +02001411static int cpuhp_issue_call(int cpu, enum cpuhp_state state, bool bringup)
Thomas Gleixner5b7aa872016-02-26 18:43:33 +00001412{
Thomas Gleixnera7246322016-08-12 19:49:38 +02001413 struct cpuhp_step *sp = cpuhp_get_step(state);
Thomas Gleixner5b7aa872016-02-26 18:43:33 +00001414 int ret;
1415
Thomas Gleixnera7246322016-08-12 19:49:38 +02001416 if ((bringup && !sp->startup) || (!bringup && !sp->teardown))
Thomas Gleixner5b7aa872016-02-26 18:43:33 +00001417 return 0;
Thomas Gleixner5b7aa872016-02-26 18:43:33 +00001418 /*
1419 * The non AP bound callbacks can fail on bringup. On teardown
1420 * e.g. module removal we crash for now.
1421 */
Thomas Gleixner1cf4f622016-02-26 18:43:39 +00001422#ifdef CONFIG_SMP
1423 if (cpuhp_is_ap_state(state))
Thomas Gleixnera7246322016-08-12 19:49:38 +02001424 ret = cpuhp_invoke_ap_callback(cpu, state, bringup);
Thomas Gleixner1cf4f622016-02-26 18:43:39 +00001425 else
Thomas Gleixnera7246322016-08-12 19:49:38 +02001426 ret = cpuhp_invoke_callback(cpu, state, bringup);
Thomas Gleixner1cf4f622016-02-26 18:43:39 +00001427#else
Thomas Gleixnera7246322016-08-12 19:49:38 +02001428 ret = cpuhp_invoke_callback(cpu, state, bringup);
Thomas Gleixner1cf4f622016-02-26 18:43:39 +00001429#endif
Thomas Gleixner5b7aa872016-02-26 18:43:33 +00001430 BUG_ON(ret && !bringup);
1431 return ret;
1432}
1433
1434/*
1435 * Called from __cpuhp_setup_state on a recoverable failure.
1436 *
1437 * Note: The teardown callbacks for rollback are not allowed to fail!
1438 */
Thomas Gleixnera7246322016-08-12 19:49:38 +02001439static void cpuhp_rollback_install(int failedcpu, enum cpuhp_state state)
Thomas Gleixner5b7aa872016-02-26 18:43:33 +00001440{
1441 int cpu;
1442
Thomas Gleixner5b7aa872016-02-26 18:43:33 +00001443 /* Roll back the already executed steps on the other cpus */
1444 for_each_present_cpu(cpu) {
1445 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1446 int cpustate = st->state;
1447
1448 if (cpu >= failedcpu)
1449 break;
1450
1451 /* Did we invoke the startup call on that cpu ? */
1452 if (cpustate >= state)
Thomas Gleixnera7246322016-08-12 19:49:38 +02001453 cpuhp_issue_call(cpu, state, false);
Thomas Gleixner5b7aa872016-02-26 18:43:33 +00001454 }
1455}
1456
1457/*
1458 * Returns a free for dynamic slot assignment of the Online state. The states
1459 * are protected by the cpuhp_slot_states mutex and an empty slot is identified
1460 * by having no name assigned.
1461 */
1462static int cpuhp_reserve_state(enum cpuhp_state state)
1463{
1464 enum cpuhp_state i;
1465
1466 mutex_lock(&cpuhp_state_mutex);
Thomas Gleixner1cf4f622016-02-26 18:43:39 +00001467 for (i = CPUHP_AP_ONLINE_DYN; i <= CPUHP_AP_ONLINE_DYN_END; i++) {
1468 if (cpuhp_ap_states[i].name)
Thomas Gleixner5b7aa872016-02-26 18:43:33 +00001469 continue;
1470
Thomas Gleixner1cf4f622016-02-26 18:43:39 +00001471 cpuhp_ap_states[i].name = "Reserved";
Thomas Gleixner5b7aa872016-02-26 18:43:33 +00001472 mutex_unlock(&cpuhp_state_mutex);
1473 return i;
1474 }
1475 mutex_unlock(&cpuhp_state_mutex);
1476 WARN(1, "No more dynamic states available for CPU hotplug\n");
1477 return -ENOSPC;
1478}
1479
1480/**
1481 * __cpuhp_setup_state - Setup the callbacks for an hotplug machine state
1482 * @state: The state to setup
1483 * @invoke: If true, the startup function is invoked for cpus where
1484 * cpu state >= @state
1485 * @startup: startup callback function
1486 * @teardown: teardown callback function
1487 *
1488 * Returns 0 if successful, otherwise a proper error code
1489 */
1490int __cpuhp_setup_state(enum cpuhp_state state,
1491 const char *name, bool invoke,
1492 int (*startup)(unsigned int cpu),
1493 int (*teardown)(unsigned int cpu))
1494{
1495 int cpu, ret = 0;
1496 int dyn_state = 0;
1497
1498 if (cpuhp_cb_check(state) || !name)
1499 return -EINVAL;
1500
1501 get_online_cpus();
1502
1503 /* currently assignments for the ONLINE state are possible */
Thomas Gleixner1cf4f622016-02-26 18:43:39 +00001504 if (state == CPUHP_AP_ONLINE_DYN) {
Thomas Gleixner5b7aa872016-02-26 18:43:33 +00001505 dyn_state = 1;
1506 ret = cpuhp_reserve_state(state);
1507 if (ret < 0)
1508 goto out;
1509 state = ret;
1510 }
1511
1512 cpuhp_store_callbacks(state, name, startup, teardown);
1513
1514 if (!invoke || !startup)
1515 goto out;
1516
1517 /*
1518 * Try to call the startup callback for each present cpu
1519 * depending on the hotplug state of the cpu.
1520 */
1521 for_each_present_cpu(cpu) {
1522 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1523 int cpustate = st->state;
1524
1525 if (cpustate < state)
1526 continue;
1527
Thomas Gleixnera7246322016-08-12 19:49:38 +02001528 ret = cpuhp_issue_call(cpu, state, true);
Thomas Gleixner5b7aa872016-02-26 18:43:33 +00001529 if (ret) {
Thomas Gleixnera7246322016-08-12 19:49:38 +02001530 if (teardown)
1531 cpuhp_rollback_install(cpu, state);
Thomas Gleixner5b7aa872016-02-26 18:43:33 +00001532 cpuhp_store_callbacks(state, NULL, NULL, NULL);
1533 goto out;
1534 }
1535 }
1536out:
1537 put_online_cpus();
1538 if (!ret && dyn_state)
1539 return state;
1540 return ret;
1541}
1542EXPORT_SYMBOL(__cpuhp_setup_state);
1543
1544/**
1545 * __cpuhp_remove_state - Remove the callbacks for an hotplug machine state
1546 * @state: The state to remove
1547 * @invoke: If true, the teardown function is invoked for cpus where
1548 * cpu state >= @state
1549 *
1550 * The teardown callback is currently not allowed to fail. Think
1551 * about module removal!
1552 */
1553void __cpuhp_remove_state(enum cpuhp_state state, bool invoke)
1554{
Thomas Gleixner5b7aa872016-02-26 18:43:33 +00001555 int cpu;
1556
1557 BUG_ON(cpuhp_cb_check(state));
1558
1559 get_online_cpus();
1560
Thomas Gleixnera7246322016-08-12 19:49:38 +02001561 if (!invoke || !cpuhp_get_teardown_cb(state))
Thomas Gleixner5b7aa872016-02-26 18:43:33 +00001562 goto remove;
1563
1564 /*
1565 * Call the teardown callback for each present cpu depending
1566 * on the hotplug state of the cpu. This function is not
1567 * allowed to fail currently!
1568 */
1569 for_each_present_cpu(cpu) {
1570 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1571 int cpustate = st->state;
1572
1573 if (cpustate >= state)
Thomas Gleixnera7246322016-08-12 19:49:38 +02001574 cpuhp_issue_call(cpu, state, false);
Thomas Gleixner5b7aa872016-02-26 18:43:33 +00001575 }
1576remove:
1577 cpuhp_store_callbacks(state, NULL, NULL, NULL);
1578 put_online_cpus();
1579}
1580EXPORT_SYMBOL(__cpuhp_remove_state);
1581
Thomas Gleixner98f8cdc2016-02-26 18:43:31 +00001582#if defined(CONFIG_SYSFS) && defined(CONFIG_HOTPLUG_CPU)
1583static ssize_t show_cpuhp_state(struct device *dev,
1584 struct device_attribute *attr, char *buf)
1585{
1586 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);
1587
1588 return sprintf(buf, "%d\n", st->state);
1589}
1590static DEVICE_ATTR(state, 0444, show_cpuhp_state, NULL);
1591
Thomas Gleixner757c9892016-02-26 18:43:32 +00001592static ssize_t write_cpuhp_target(struct device *dev,
1593 struct device_attribute *attr,
1594 const char *buf, size_t count)
1595{
1596 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);
1597 struct cpuhp_step *sp;
1598 int target, ret;
1599
1600 ret = kstrtoint(buf, 10, &target);
1601 if (ret)
1602 return ret;
1603
1604#ifdef CONFIG_CPU_HOTPLUG_STATE_CONTROL
1605 if (target < CPUHP_OFFLINE || target > CPUHP_ONLINE)
1606 return -EINVAL;
1607#else
1608 if (target != CPUHP_OFFLINE && target != CPUHP_ONLINE)
1609 return -EINVAL;
1610#endif
1611
1612 ret = lock_device_hotplug_sysfs();
1613 if (ret)
1614 return ret;
1615
1616 mutex_lock(&cpuhp_state_mutex);
1617 sp = cpuhp_get_step(target);
1618 ret = !sp->name || sp->cant_stop ? -EINVAL : 0;
1619 mutex_unlock(&cpuhp_state_mutex);
1620 if (ret)
1621 return ret;
1622
1623 if (st->state < target)
1624 ret = do_cpu_up(dev->id, target);
1625 else
1626 ret = do_cpu_down(dev->id, target);
1627
1628 unlock_device_hotplug();
1629 return ret ? ret : count;
1630}
1631
Thomas Gleixner98f8cdc2016-02-26 18:43:31 +00001632static ssize_t show_cpuhp_target(struct device *dev,
1633 struct device_attribute *attr, char *buf)
1634{
1635 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);
1636
1637 return sprintf(buf, "%d\n", st->target);
1638}
Thomas Gleixner757c9892016-02-26 18:43:32 +00001639static DEVICE_ATTR(target, 0644, show_cpuhp_target, write_cpuhp_target);
Thomas Gleixner98f8cdc2016-02-26 18:43:31 +00001640
1641static struct attribute *cpuhp_cpu_attrs[] = {
1642 &dev_attr_state.attr,
1643 &dev_attr_target.attr,
1644 NULL
1645};
1646
1647static struct attribute_group cpuhp_cpu_attr_group = {
1648 .attrs = cpuhp_cpu_attrs,
1649 .name = "hotplug",
1650 NULL
1651};
1652
1653static ssize_t show_cpuhp_states(struct device *dev,
1654 struct device_attribute *attr, char *buf)
1655{
1656 ssize_t cur, res = 0;
1657 int i;
1658
1659 mutex_lock(&cpuhp_state_mutex);
Thomas Gleixner757c9892016-02-26 18:43:32 +00001660 for (i = CPUHP_OFFLINE; i <= CPUHP_ONLINE; i++) {
Thomas Gleixner98f8cdc2016-02-26 18:43:31 +00001661 struct cpuhp_step *sp = cpuhp_get_step(i);
1662
1663 if (sp->name) {
1664 cur = sprintf(buf, "%3d: %s\n", i, sp->name);
1665 buf += cur;
1666 res += cur;
1667 }
1668 }
1669 mutex_unlock(&cpuhp_state_mutex);
1670 return res;
1671}
1672static DEVICE_ATTR(states, 0444, show_cpuhp_states, NULL);
1673
1674static struct attribute *cpuhp_cpu_root_attrs[] = {
1675 &dev_attr_states.attr,
1676 NULL
1677};
1678
1679static struct attribute_group cpuhp_cpu_root_attr_group = {
1680 .attrs = cpuhp_cpu_root_attrs,
1681 .name = "hotplug",
1682 NULL
1683};
1684
1685static int __init cpuhp_sysfs_init(void)
1686{
1687 int cpu, ret;
1688
1689 ret = sysfs_create_group(&cpu_subsys.dev_root->kobj,
1690 &cpuhp_cpu_root_attr_group);
1691 if (ret)
1692 return ret;
1693
1694 for_each_possible_cpu(cpu) {
1695 struct device *dev = get_cpu_device(cpu);
1696
1697 if (!dev)
1698 continue;
1699 ret = sysfs_create_group(&dev->kobj, &cpuhp_cpu_attr_group);
1700 if (ret)
1701 return ret;
1702 }
1703 return 0;
1704}
1705device_initcall(cpuhp_sysfs_init);
1706#endif
1707
Linus Torvaldse56b3bc2008-07-28 11:32:33 -07001708/*
1709 * cpu_bit_bitmap[] is a special, "compressed" data structure that
1710 * represents all NR_CPUS bits binary values of 1<<nr.
1711 *
Rusty Russelle0b582e2009-01-01 10:12:28 +10301712 * It is used by cpumask_of() to get a constant address to a CPU
Linus Torvaldse56b3bc2008-07-28 11:32:33 -07001713 * mask value that has a single bit set only.
1714 */
Mike Travisb8d317d2008-07-24 18:21:29 -07001715
Linus Torvaldse56b3bc2008-07-28 11:32:33 -07001716/* cpu_bit_bitmap[0] is empty - so we can back into it */
Michael Rodriguez4d519852011-03-22 16:34:07 -07001717#define MASK_DECLARE_1(x) [x+1][0] = (1UL << (x))
Linus Torvaldse56b3bc2008-07-28 11:32:33 -07001718#define MASK_DECLARE_2(x) MASK_DECLARE_1(x), MASK_DECLARE_1(x+1)
1719#define MASK_DECLARE_4(x) MASK_DECLARE_2(x), MASK_DECLARE_2(x+2)
1720#define MASK_DECLARE_8(x) MASK_DECLARE_4(x), MASK_DECLARE_4(x+4)
Mike Travisb8d317d2008-07-24 18:21:29 -07001721
Linus Torvaldse56b3bc2008-07-28 11:32:33 -07001722const unsigned long cpu_bit_bitmap[BITS_PER_LONG+1][BITS_TO_LONGS(NR_CPUS)] = {
Mike Travisb8d317d2008-07-24 18:21:29 -07001723
Linus Torvaldse56b3bc2008-07-28 11:32:33 -07001724 MASK_DECLARE_8(0), MASK_DECLARE_8(8),
1725 MASK_DECLARE_8(16), MASK_DECLARE_8(24),
1726#if BITS_PER_LONG > 32
1727 MASK_DECLARE_8(32), MASK_DECLARE_8(40),
1728 MASK_DECLARE_8(48), MASK_DECLARE_8(56),
Mike Travisb8d317d2008-07-24 18:21:29 -07001729#endif
1730};
Linus Torvaldse56b3bc2008-07-28 11:32:33 -07001731EXPORT_SYMBOL_GPL(cpu_bit_bitmap);
Rusty Russell2d3854a2008-11-05 13:39:10 +11001732
1733const DECLARE_BITMAP(cpu_all_bits, NR_CPUS) = CPU_BITS_ALL;
1734EXPORT_SYMBOL(cpu_all_bits);
Rusty Russellb3199c02008-12-30 09:05:14 +10301735
1736#ifdef CONFIG_INIT_ALL_POSSIBLE
Rasmus Villemoes4b804c82016-01-20 15:00:19 -08001737struct cpumask __cpu_possible_mask __read_mostly
Rasmus Villemoesc4c54dd2016-01-20 15:00:16 -08001738 = {CPU_BITS_ALL};
Rusty Russellb3199c02008-12-30 09:05:14 +10301739#else
Rasmus Villemoes4b804c82016-01-20 15:00:19 -08001740struct cpumask __cpu_possible_mask __read_mostly;
Rusty Russellb3199c02008-12-30 09:05:14 +10301741#endif
Rasmus Villemoes4b804c82016-01-20 15:00:19 -08001742EXPORT_SYMBOL(__cpu_possible_mask);
Rusty Russellb3199c02008-12-30 09:05:14 +10301743
Rasmus Villemoes4b804c82016-01-20 15:00:19 -08001744struct cpumask __cpu_online_mask __read_mostly;
1745EXPORT_SYMBOL(__cpu_online_mask);
Rusty Russellb3199c02008-12-30 09:05:14 +10301746
Rasmus Villemoes4b804c82016-01-20 15:00:19 -08001747struct cpumask __cpu_present_mask __read_mostly;
1748EXPORT_SYMBOL(__cpu_present_mask);
Rusty Russellb3199c02008-12-30 09:05:14 +10301749
Rasmus Villemoes4b804c82016-01-20 15:00:19 -08001750struct cpumask __cpu_active_mask __read_mostly;
1751EXPORT_SYMBOL(__cpu_active_mask);
Rusty Russell3fa41522008-12-30 09:05:16 +10301752
Rusty Russell3fa41522008-12-30 09:05:16 +10301753void init_cpu_present(const struct cpumask *src)
1754{
Rasmus Villemoesc4c54dd2016-01-20 15:00:16 -08001755 cpumask_copy(&__cpu_present_mask, src);
Rusty Russell3fa41522008-12-30 09:05:16 +10301756}
1757
1758void init_cpu_possible(const struct cpumask *src)
1759{
Rasmus Villemoesc4c54dd2016-01-20 15:00:16 -08001760 cpumask_copy(&__cpu_possible_mask, src);
Rusty Russell3fa41522008-12-30 09:05:16 +10301761}
1762
1763void init_cpu_online(const struct cpumask *src)
1764{
Rasmus Villemoesc4c54dd2016-01-20 15:00:16 -08001765 cpumask_copy(&__cpu_online_mask, src);
Rusty Russell3fa41522008-12-30 09:05:16 +10301766}
Thomas Gleixnercff7d372016-02-26 18:43:28 +00001767
1768/*
1769 * Activate the first processor.
1770 */
1771void __init boot_cpu_init(void)
1772{
1773 int cpu = smp_processor_id();
1774
1775 /* Mark the boot cpu "present", "online" etc for SMP and UP case */
1776 set_cpu_online(cpu, true);
1777 set_cpu_active(cpu, true);
1778 set_cpu_present(cpu, true);
1779 set_cpu_possible(cpu, true);
1780}
1781
1782/*
1783 * Must be called _AFTER_ setting up the per_cpu areas
1784 */
1785void __init boot_cpu_state_init(void)
1786{
1787 per_cpu_ptr(&cpuhp_state, smp_processor_id())->state = CPUHP_ONLINE;
1788}