blob: e220e565ea9838fc4a9943e56e0270b6b405728c [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/* CPU control.
2 * (C) 2001, 2002, 2003, 2004 Rusty Russell
3 *
4 * This code is licenced under the GPL.
5 */
6#include <linux/proc_fs.h>
7#include <linux/smp.h>
8#include <linux/init.h>
9#include <linux/notifier.h>
10#include <linux/sched.h>
11#include <linux/unistd.h>
12#include <linux/cpu.h>
Anton Vorontsovcb792952012-05-31 16:26:22 -070013#include <linux/oom.h>
14#include <linux/rcupdate.h>
Paul Gortmaker9984de12011-05-23 14:51:41 -040015#include <linux/export.h>
Anton Vorontsove4cc2f82012-05-31 16:26:26 -070016#include <linux/bug.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070017#include <linux/kthread.h>
18#include <linux/stop_machine.h>
Ingo Molnar81615b622006-06-26 00:24:32 -070019#include <linux/mutex.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090020#include <linux/gfp.h>
Srivatsa S. Bhat79cfbdf2011-11-03 00:59:25 +010021#include <linux/suspend.h>
Gautham R. Shenoya19423b2014-03-11 02:04:03 +053022#include <linux/lockdep.h>
Preeti U Murthy345527b2015-03-30 14:59:19 +053023#include <linux/tick.h>
Thomas Gleixnera8994182015-07-05 17:12:30 +000024#include <linux/irq.h>
Thomas Gleixner4cb28ce2016-02-26 18:43:38 +000025#include <linux/smpboot.h>
Thomas Gleixnercff7d372016-02-26 18:43:28 +000026
Todd E Brandtbb3632c2014-06-06 05:40:17 -070027#include <trace/events/power.h>
Thomas Gleixnercff7d372016-02-26 18:43:28 +000028#define CREATE_TRACE_POINTS
29#include <trace/events/cpuhp.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070030
Thomas Gleixner38498a62012-04-20 13:05:44 +000031#include "smpboot.h"
32
Thomas Gleixnercff7d372016-02-26 18:43:28 +000033/**
34 * cpuhp_cpu_state - Per cpu hotplug state storage
35 * @state: The current cpu state
36 * @target: The target state
Thomas Gleixner4cb28ce2016-02-26 18:43:38 +000037 * @thread: Pointer to the hotplug thread
38 * @should_run: Thread should execute
39 * @cb_stat: The state for a single callback (install/uninstall)
40 * @cb: Single callback function (install/uninstall)
41 * @result: Result of the operation
42 * @done: Signal completion to the issuer of the task
Thomas Gleixnercff7d372016-02-26 18:43:28 +000043 */
44struct cpuhp_cpu_state {
45 enum cpuhp_state state;
46 enum cpuhp_state target;
Thomas Gleixner4cb28ce2016-02-26 18:43:38 +000047#ifdef CONFIG_SMP
48 struct task_struct *thread;
49 bool should_run;
50 enum cpuhp_state cb_state;
51 int (*cb)(unsigned int cpu);
52 int result;
53 struct completion done;
54#endif
Thomas Gleixnercff7d372016-02-26 18:43:28 +000055};
56
57static DEFINE_PER_CPU(struct cpuhp_cpu_state, cpuhp_state);
58
59/**
60 * cpuhp_step - Hotplug state machine step
61 * @name: Name of the step
62 * @startup: Startup function of the step
63 * @teardown: Teardown function of the step
64 * @skip_onerr: Do not invoke the functions on error rollback
65 * Will go away once the notifiers are gone
Thomas Gleixner757c9892016-02-26 18:43:32 +000066 * @cant_stop: Bringup/teardown can't be stopped at this step
Thomas Gleixnercff7d372016-02-26 18:43:28 +000067 */
68struct cpuhp_step {
69 const char *name;
70 int (*startup)(unsigned int cpu);
71 int (*teardown)(unsigned int cpu);
72 bool skip_onerr;
Thomas Gleixner757c9892016-02-26 18:43:32 +000073 bool cant_stop;
Thomas Gleixnercff7d372016-02-26 18:43:28 +000074};
75
Thomas Gleixner98f8cdc2016-02-26 18:43:31 +000076static DEFINE_MUTEX(cpuhp_state_mutex);
Thomas Gleixnercff7d372016-02-26 18:43:28 +000077static struct cpuhp_step cpuhp_bp_states[];
Thomas Gleixner4baa0af2016-02-26 18:43:29 +000078static struct cpuhp_step cpuhp_ap_states[];
Thomas Gleixnercff7d372016-02-26 18:43:28 +000079
80/**
81 * cpuhp_invoke_callback _ Invoke the callbacks for a given state
82 * @cpu: The cpu for which the callback should be invoked
83 * @step: The step in the state machine
84 * @cb: The callback function to invoke
85 *
86 * Called from cpu hotplug and from the state register machinery
87 */
88static int cpuhp_invoke_callback(unsigned int cpu, enum cpuhp_state step,
89 int (*cb)(unsigned int))
90{
91 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
92 int ret = 0;
93
94 if (cb) {
95 trace_cpuhp_enter(cpu, st->target, step, cb);
96 ret = cb(cpu);
97 trace_cpuhp_exit(cpu, st->state, step, ret);
98 }
99 return ret;
100}
101
Rusty Russell98a79d62008-12-13 21:19:41 +1030102#ifdef CONFIG_SMP
Rusty Russellb3199c02008-12-30 09:05:14 +1030103/* Serializes the updates to cpu_online_mask, cpu_present_mask */
Linus Torvaldsaa953872006-07-23 12:12:16 -0700104static DEFINE_MUTEX(cpu_add_remove_lock);
Thomas Gleixner090e77c2016-02-26 18:43:23 +0000105bool cpuhp_tasks_frozen;
106EXPORT_SYMBOL_GPL(cpuhp_tasks_frozen);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700107
Lai Jiangshan79a6cde2010-05-26 14:43:36 -0700108/*
Srivatsa S. Bhat93ae4f92014-03-11 02:04:14 +0530109 * The following two APIs (cpu_maps_update_begin/done) must be used when
110 * attempting to serialize the updates to cpu_online_mask & cpu_present_mask.
111 * The APIs cpu_notifier_register_begin/done() must be used to protect CPU
112 * hotplug callback (un)registration performed using __register_cpu_notifier()
113 * or __unregister_cpu_notifier().
Lai Jiangshan79a6cde2010-05-26 14:43:36 -0700114 */
115void cpu_maps_update_begin(void)
116{
117 mutex_lock(&cpu_add_remove_lock);
118}
Srivatsa S. Bhat93ae4f92014-03-11 02:04:14 +0530119EXPORT_SYMBOL(cpu_notifier_register_begin);
Lai Jiangshan79a6cde2010-05-26 14:43:36 -0700120
121void cpu_maps_update_done(void)
122{
123 mutex_unlock(&cpu_add_remove_lock);
124}
Srivatsa S. Bhat93ae4f92014-03-11 02:04:14 +0530125EXPORT_SYMBOL(cpu_notifier_register_done);
Lai Jiangshan79a6cde2010-05-26 14:43:36 -0700126
Daniel J Blueman5c113fb2010-06-01 12:15:11 +0100127static RAW_NOTIFIER_HEAD(cpu_chain);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700128
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -0700129/* If set, cpu_up and cpu_down will return -EBUSY and do nothing.
130 * Should always be manipulated under cpu_add_remove_lock
131 */
132static int cpu_hotplug_disabled;
133
Lai Jiangshan79a6cde2010-05-26 14:43:36 -0700134#ifdef CONFIG_HOTPLUG_CPU
135
Gautham R Shenoyd2219382008-01-25 21:08:01 +0100136static struct {
137 struct task_struct *active_writer;
David Hildenbrand87af9e72014-12-12 10:11:44 +0100138 /* wait queue to wake up the active_writer */
139 wait_queue_head_t wq;
140 /* verifies that no writer will get active while readers are active */
141 struct mutex lock;
Gautham R Shenoyd2219382008-01-25 21:08:01 +0100142 /*
143 * Also blocks the new readers during
144 * an ongoing cpu hotplug operation.
145 */
David Hildenbrand87af9e72014-12-12 10:11:44 +0100146 atomic_t refcount;
Gautham R. Shenoya19423b2014-03-11 02:04:03 +0530147
148#ifdef CONFIG_DEBUG_LOCK_ALLOC
149 struct lockdep_map dep_map;
150#endif
Linus Torvalds31950eb2009-06-22 21:18:12 -0700151} cpu_hotplug = {
152 .active_writer = NULL,
David Hildenbrand87af9e72014-12-12 10:11:44 +0100153 .wq = __WAIT_QUEUE_HEAD_INITIALIZER(cpu_hotplug.wq),
Linus Torvalds31950eb2009-06-22 21:18:12 -0700154 .lock = __MUTEX_INITIALIZER(cpu_hotplug.lock),
Gautham R. Shenoya19423b2014-03-11 02:04:03 +0530155#ifdef CONFIG_DEBUG_LOCK_ALLOC
156 .dep_map = {.name = "cpu_hotplug.lock" },
157#endif
Linus Torvalds31950eb2009-06-22 21:18:12 -0700158};
Gautham R Shenoyd2219382008-01-25 21:08:01 +0100159
Gautham R. Shenoya19423b2014-03-11 02:04:03 +0530160/* Lockdep annotations for get/put_online_cpus() and cpu_hotplug_begin/end() */
161#define cpuhp_lock_acquire_read() lock_map_acquire_read(&cpu_hotplug.dep_map)
Paul E. McKenneydd56af42014-08-25 20:25:06 -0700162#define cpuhp_lock_acquire_tryread() \
163 lock_map_acquire_tryread(&cpu_hotplug.dep_map)
Gautham R. Shenoya19423b2014-03-11 02:04:03 +0530164#define cpuhp_lock_acquire() lock_map_acquire(&cpu_hotplug.dep_map)
165#define cpuhp_lock_release() lock_map_release(&cpu_hotplug.dep_map)
166
Paul E. McKenney62db99f2014-10-22 14:51:49 -0700167
Gautham R Shenoy86ef5c92008-01-25 21:08:02 +0100168void get_online_cpus(void)
Ashok Raja9d9baa2005-11-28 13:43:46 -0800169{
Gautham R Shenoyd2219382008-01-25 21:08:01 +0100170 might_sleep();
171 if (cpu_hotplug.active_writer == current)
Linus Torvaldsaa953872006-07-23 12:12:16 -0700172 return;
Gautham R. Shenoya19423b2014-03-11 02:04:03 +0530173 cpuhp_lock_acquire_read();
Gautham R Shenoyd2219382008-01-25 21:08:01 +0100174 mutex_lock(&cpu_hotplug.lock);
David Hildenbrand87af9e72014-12-12 10:11:44 +0100175 atomic_inc(&cpu_hotplug.refcount);
Gautham R Shenoyd2219382008-01-25 21:08:01 +0100176 mutex_unlock(&cpu_hotplug.lock);
Ashok Raja9d9baa2005-11-28 13:43:46 -0800177}
Gautham R Shenoy86ef5c92008-01-25 21:08:02 +0100178EXPORT_SYMBOL_GPL(get_online_cpus);
Ashok Raj90d45d12005-11-08 21:34:24 -0800179
Gautham R Shenoy86ef5c92008-01-25 21:08:02 +0100180void put_online_cpus(void)
Ashok Raja9d9baa2005-11-28 13:43:46 -0800181{
David Hildenbrand87af9e72014-12-12 10:11:44 +0100182 int refcount;
183
Gautham R Shenoyd2219382008-01-25 21:08:01 +0100184 if (cpu_hotplug.active_writer == current)
Linus Torvaldsaa953872006-07-23 12:12:16 -0700185 return;
Srivatsa S. Bhat075663d2012-10-08 16:28:20 -0700186
David Hildenbrand87af9e72014-12-12 10:11:44 +0100187 refcount = atomic_dec_return(&cpu_hotplug.refcount);
188 if (WARN_ON(refcount < 0)) /* try to fix things up */
189 atomic_inc(&cpu_hotplug.refcount);
Srivatsa S. Bhat075663d2012-10-08 16:28:20 -0700190
David Hildenbrand87af9e72014-12-12 10:11:44 +0100191 if (refcount <= 0 && waitqueue_active(&cpu_hotplug.wq))
192 wake_up(&cpu_hotplug.wq);
193
Gautham R. Shenoya19423b2014-03-11 02:04:03 +0530194 cpuhp_lock_release();
Gautham R Shenoyd2219382008-01-25 21:08:01 +0100195
Ashok Raja9d9baa2005-11-28 13:43:46 -0800196}
Gautham R Shenoy86ef5c92008-01-25 21:08:02 +0100197EXPORT_SYMBOL_GPL(put_online_cpus);
Ashok Raja9d9baa2005-11-28 13:43:46 -0800198
Gautham R Shenoyd2219382008-01-25 21:08:01 +0100199/*
200 * This ensures that the hotplug operation can begin only when the
201 * refcount goes to zero.
202 *
203 * Note that during a cpu-hotplug operation, the new readers, if any,
204 * will be blocked by the cpu_hotplug.lock
205 *
Oleg Nesterovd2ba7e22008-04-29 01:00:29 -0700206 * Since cpu_hotplug_begin() is always called after invoking
207 * cpu_maps_update_begin(), we can be sure that only one writer is active.
Gautham R Shenoyd2219382008-01-25 21:08:01 +0100208 *
209 * Note that theoretically, there is a possibility of a livelock:
210 * - Refcount goes to zero, last reader wakes up the sleeping
211 * writer.
212 * - Last reader unlocks the cpu_hotplug.lock.
213 * - A new reader arrives at this moment, bumps up the refcount.
214 * - The writer acquires the cpu_hotplug.lock finds the refcount
215 * non zero and goes to sleep again.
216 *
217 * However, this is very difficult to achieve in practice since
Gautham R Shenoy86ef5c92008-01-25 21:08:02 +0100218 * get_online_cpus() not an api which is called all that often.
Gautham R Shenoyd2219382008-01-25 21:08:01 +0100219 *
220 */
Toshi Kanib9d10be2013-08-12 09:45:53 -0600221void cpu_hotplug_begin(void)
Gautham R Shenoyd2219382008-01-25 21:08:01 +0100222{
David Hildenbrand87af9e72014-12-12 10:11:44 +0100223 DEFINE_WAIT(wait);
Oleg Nesterovd2ba7e22008-04-29 01:00:29 -0700224
David Hildenbrand87af9e72014-12-12 10:11:44 +0100225 cpu_hotplug.active_writer = current;
Gautham R. Shenoya19423b2014-03-11 02:04:03 +0530226 cpuhp_lock_acquire();
David Hildenbrand87af9e72014-12-12 10:11:44 +0100227
Oleg Nesterovd2ba7e22008-04-29 01:00:29 -0700228 for (;;) {
229 mutex_lock(&cpu_hotplug.lock);
David Hildenbrand87af9e72014-12-12 10:11:44 +0100230 prepare_to_wait(&cpu_hotplug.wq, &wait, TASK_UNINTERRUPTIBLE);
231 if (likely(!atomic_read(&cpu_hotplug.refcount)))
232 break;
Gautham R Shenoyd2219382008-01-25 21:08:01 +0100233 mutex_unlock(&cpu_hotplug.lock);
234 schedule();
Gautham R Shenoyd2219382008-01-25 21:08:01 +0100235 }
David Hildenbrand87af9e72014-12-12 10:11:44 +0100236 finish_wait(&cpu_hotplug.wq, &wait);
Gautham R Shenoyd2219382008-01-25 21:08:01 +0100237}
238
Toshi Kanib9d10be2013-08-12 09:45:53 -0600239void cpu_hotplug_done(void)
Gautham R Shenoyd2219382008-01-25 21:08:01 +0100240{
241 cpu_hotplug.active_writer = NULL;
242 mutex_unlock(&cpu_hotplug.lock);
Gautham R. Shenoya19423b2014-03-11 02:04:03 +0530243 cpuhp_lock_release();
Gautham R Shenoyd2219382008-01-25 21:08:01 +0100244}
Lai Jiangshan79a6cde2010-05-26 14:43:36 -0700245
Srivatsa S. Bhat16e53db2013-06-12 14:04:36 -0700246/*
247 * Wait for currently running CPU hotplug operations to complete (if any) and
248 * disable future CPU hotplug (from sysfs). The 'cpu_add_remove_lock' protects
249 * the 'cpu_hotplug_disabled' flag. The same lock is also acquired by the
250 * hotplug path before performing hotplug operations. So acquiring that lock
251 * guarantees mutual exclusion from any currently running hotplug operations.
252 */
253void cpu_hotplug_disable(void)
254{
255 cpu_maps_update_begin();
Vitaly Kuznetsov89af7ba2015-08-05 00:52:46 -0700256 cpu_hotplug_disabled++;
Srivatsa S. Bhat16e53db2013-06-12 14:04:36 -0700257 cpu_maps_update_done();
258}
Vitaly Kuznetsov32145c42015-08-05 00:52:47 -0700259EXPORT_SYMBOL_GPL(cpu_hotplug_disable);
Srivatsa S. Bhat16e53db2013-06-12 14:04:36 -0700260
261void cpu_hotplug_enable(void)
262{
263 cpu_maps_update_begin();
Vitaly Kuznetsov89af7ba2015-08-05 00:52:46 -0700264 WARN_ON(--cpu_hotplug_disabled < 0);
Srivatsa S. Bhat16e53db2013-06-12 14:04:36 -0700265 cpu_maps_update_done();
266}
Vitaly Kuznetsov32145c42015-08-05 00:52:47 -0700267EXPORT_SYMBOL_GPL(cpu_hotplug_enable);
Toshi Kanib9d10be2013-08-12 09:45:53 -0600268#endif /* CONFIG_HOTPLUG_CPU */
Lai Jiangshan79a6cde2010-05-26 14:43:36 -0700269
Linus Torvalds1da177e2005-04-16 15:20:36 -0700270/* Need to know about CPUs going up/down? */
Mathias Krause71cf5ae2015-07-19 20:06:22 +0200271int register_cpu_notifier(struct notifier_block *nb)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700272{
Neil Brownbd5349c2006-10-17 00:10:35 -0700273 int ret;
Gautham R Shenoyd2219382008-01-25 21:08:01 +0100274 cpu_maps_update_begin();
Neil Brownbd5349c2006-10-17 00:10:35 -0700275 ret = raw_notifier_chain_register(&cpu_chain, nb);
Gautham R Shenoyd2219382008-01-25 21:08:01 +0100276 cpu_maps_update_done();
Neil Brownbd5349c2006-10-17 00:10:35 -0700277 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700278}
Chandra Seetharaman65edc682006-06-27 02:54:08 -0700279
Mathias Krause71cf5ae2015-07-19 20:06:22 +0200280int __register_cpu_notifier(struct notifier_block *nb)
Srivatsa S. Bhat93ae4f92014-03-11 02:04:14 +0530281{
282 return raw_notifier_chain_register(&cpu_chain, nb);
283}
284
Thomas Gleixner090e77c2016-02-26 18:43:23 +0000285static int __cpu_notify(unsigned long val, unsigned int cpu, int nr_to_call,
Akinobu Mitae9fb7632010-05-26 14:43:28 -0700286 int *nr_calls)
287{
Thomas Gleixner090e77c2016-02-26 18:43:23 +0000288 unsigned long mod = cpuhp_tasks_frozen ? CPU_TASKS_FROZEN : 0;
289 void *hcpu = (void *)(long)cpu;
290
Akinobu Mitae6bde732010-05-26 14:43:29 -0700291 int ret;
292
Thomas Gleixner090e77c2016-02-26 18:43:23 +0000293 ret = __raw_notifier_call_chain(&cpu_chain, val | mod, hcpu, nr_to_call,
Akinobu Mitae9fb7632010-05-26 14:43:28 -0700294 nr_calls);
Akinobu Mitae6bde732010-05-26 14:43:29 -0700295
296 return notifier_to_errno(ret);
Akinobu Mitae9fb7632010-05-26 14:43:28 -0700297}
298
Thomas Gleixner090e77c2016-02-26 18:43:23 +0000299static int cpu_notify(unsigned long val, unsigned int cpu)
Akinobu Mitae9fb7632010-05-26 14:43:28 -0700300{
Thomas Gleixner090e77c2016-02-26 18:43:23 +0000301 return __cpu_notify(val, cpu, -1, NULL);
Akinobu Mitae9fb7632010-05-26 14:43:28 -0700302}
303
Thomas Gleixnerba997462016-02-26 18:43:24 +0000304/* Notifier wrappers for transitioning to state machine */
305static int notify_prepare(unsigned int cpu)
306{
307 int nr_calls = 0;
308 int ret;
309
310 ret = __cpu_notify(CPU_UP_PREPARE, cpu, -1, &nr_calls);
311 if (ret) {
312 nr_calls--;
313 printk(KERN_WARNING "%s: attempt to bring up CPU %u failed\n",
314 __func__, cpu);
315 __cpu_notify(CPU_UP_CANCELED, cpu, nr_calls, NULL);
316 }
317 return ret;
318}
319
320static int notify_online(unsigned int cpu)
321{
322 cpu_notify(CPU_ONLINE, cpu);
323 return 0;
324}
325
Thomas Gleixner4baa0af2016-02-26 18:43:29 +0000326static int notify_starting(unsigned int cpu)
327{
328 cpu_notify(CPU_STARTING, cpu);
329 return 0;
330}
331
Thomas Gleixnerba997462016-02-26 18:43:24 +0000332static int bringup_cpu(unsigned int cpu)
333{
334 struct task_struct *idle = idle_thread_get(cpu);
335 int ret;
336
337 /* Arch-specific enabling code. */
338 ret = __cpu_up(cpu, idle);
339 if (ret) {
340 cpu_notify(CPU_UP_CANCELED, cpu);
341 return ret;
342 }
343 BUG_ON(!cpu_online(cpu));
344 return 0;
345}
346
Thomas Gleixner2e1a3482016-02-26 18:43:37 +0000347/*
348 * Hotplug state machine related functions
349 */
350static void undo_cpu_down(unsigned int cpu, struct cpuhp_cpu_state *st,
351 struct cpuhp_step *steps)
352{
353 for (st->state++; st->state < st->target; st->state++) {
354 struct cpuhp_step *step = steps + st->state;
355
356 if (!step->skip_onerr)
357 cpuhp_invoke_callback(cpu, st->state, step->startup);
358 }
359}
360
361static int cpuhp_down_callbacks(unsigned int cpu, struct cpuhp_cpu_state *st,
362 struct cpuhp_step *steps, enum cpuhp_state target)
363{
364 enum cpuhp_state prev_state = st->state;
365 int ret = 0;
366
367 for (; st->state > target; st->state--) {
368 struct cpuhp_step *step = steps + st->state;
369
370 ret = cpuhp_invoke_callback(cpu, st->state, step->teardown);
371 if (ret) {
372 st->target = prev_state;
373 undo_cpu_down(cpu, st, steps);
374 break;
375 }
376 }
377 return ret;
378}
379
380static void undo_cpu_up(unsigned int cpu, struct cpuhp_cpu_state *st,
381 struct cpuhp_step *steps)
382{
383 for (st->state--; st->state > st->target; st->state--) {
384 struct cpuhp_step *step = steps + st->state;
385
386 if (!step->skip_onerr)
387 cpuhp_invoke_callback(cpu, st->state, step->teardown);
388 }
389}
390
391static int cpuhp_up_callbacks(unsigned int cpu, struct cpuhp_cpu_state *st,
392 struct cpuhp_step *steps, enum cpuhp_state target)
393{
394 enum cpuhp_state prev_state = st->state;
395 int ret = 0;
396
397 while (st->state < target) {
398 struct cpuhp_step *step;
399
400 st->state++;
401 step = steps + st->state;
402 ret = cpuhp_invoke_callback(cpu, st->state, step->startup);
403 if (ret) {
404 st->target = prev_state;
405 undo_cpu_up(cpu, st, steps);
406 break;
407 }
408 }
409 return ret;
410}
411
Thomas Gleixner4cb28ce2016-02-26 18:43:38 +0000412/*
413 * The cpu hotplug threads manage the bringup and teardown of the cpus
414 */
415static void cpuhp_create(unsigned int cpu)
416{
417 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
418
419 init_completion(&st->done);
420}
421
422static int cpuhp_should_run(unsigned int cpu)
423{
424 struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);
425
426 return st->should_run;
427}
428
429/* Execute the teardown callbacks. Used to be CPU_DOWN_PREPARE */
430static int cpuhp_ap_offline(unsigned int cpu, struct cpuhp_cpu_state *st)
431{
Thomas Gleixner1cf4f622016-02-26 18:43:39 +0000432 enum cpuhp_state target = max((int)st->target, CPUHP_TEARDOWN_CPU);
Thomas Gleixner4cb28ce2016-02-26 18:43:38 +0000433
434 return cpuhp_down_callbacks(cpu, st, cpuhp_ap_states, target);
435}
436
437/* Execute the online startup callbacks. Used to be CPU_ONLINE */
438static int cpuhp_ap_online(unsigned int cpu, struct cpuhp_cpu_state *st)
439{
440 return cpuhp_up_callbacks(cpu, st, cpuhp_ap_states, st->target);
441}
442
443/*
444 * Execute teardown/startup callbacks on the plugged cpu. Also used to invoke
445 * callbacks when a state gets [un]installed at runtime.
446 */
447static void cpuhp_thread_fun(unsigned int cpu)
448{
449 struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);
450 int ret = 0;
451
452 /*
453 * Paired with the mb() in cpuhp_kick_ap_work and
454 * cpuhp_invoke_ap_callback, so the work set is consistent visible.
455 */
456 smp_mb();
457 if (!st->should_run)
458 return;
459
460 st->should_run = false;
461
462 /* Single callback invocation for [un]install ? */
463 if (st->cb) {
464 if (st->cb_state < CPUHP_AP_ONLINE) {
465 local_irq_disable();
466 ret = cpuhp_invoke_callback(cpu, st->cb_state, st->cb);
467 local_irq_enable();
468 } else {
469 ret = cpuhp_invoke_callback(cpu, st->cb_state, st->cb);
470 }
471 } else {
Thomas Gleixner1cf4f622016-02-26 18:43:39 +0000472 /* Cannot happen .... */
473 BUG_ON(st->state < CPUHP_KICK_AP_THREAD);
474
Thomas Gleixner4cb28ce2016-02-26 18:43:38 +0000475 /* Regular hotplug work */
476 if (st->state < st->target)
477 ret = cpuhp_ap_online(cpu, st);
478 else if (st->state > st->target)
479 ret = cpuhp_ap_offline(cpu, st);
480 }
481 st->result = ret;
482 complete(&st->done);
483}
484
485/* Invoke a single callback on a remote cpu */
486static int cpuhp_invoke_ap_callback(int cpu, enum cpuhp_state state,
487 int (*cb)(unsigned int))
488{
489 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
490
491 if (!cpu_online(cpu))
492 return 0;
493
494 st->cb_state = state;
495 st->cb = cb;
496 /*
497 * Make sure the above stores are visible before should_run becomes
498 * true. Paired with the mb() above in cpuhp_thread_fun()
499 */
500 smp_mb();
501 st->should_run = true;
502 wake_up_process(st->thread);
503 wait_for_completion(&st->done);
504 return st->result;
505}
506
507/* Regular hotplug invocation of the AP hotplug thread */
Thomas Gleixner1cf4f622016-02-26 18:43:39 +0000508static void __cpuhp_kick_ap_work(struct cpuhp_cpu_state *st)
Thomas Gleixner4cb28ce2016-02-26 18:43:38 +0000509{
Thomas Gleixner4cb28ce2016-02-26 18:43:38 +0000510 st->result = 0;
511 st->cb = NULL;
512 /*
513 * Make sure the above stores are visible before should_run becomes
514 * true. Paired with the mb() above in cpuhp_thread_fun()
515 */
516 smp_mb();
517 st->should_run = true;
518 wake_up_process(st->thread);
Thomas Gleixner1cf4f622016-02-26 18:43:39 +0000519}
520
521static int cpuhp_kick_ap_work(unsigned int cpu)
522{
523 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
524 enum cpuhp_state state = st->state;
525
526 trace_cpuhp_enter(cpu, st->target, state, cpuhp_kick_ap_work);
527 __cpuhp_kick_ap_work(st);
Thomas Gleixner4cb28ce2016-02-26 18:43:38 +0000528 wait_for_completion(&st->done);
529 trace_cpuhp_exit(cpu, st->state, state, st->result);
530 return st->result;
531}
532
533static struct smp_hotplug_thread cpuhp_threads = {
534 .store = &cpuhp_state.thread,
535 .create = &cpuhp_create,
536 .thread_should_run = cpuhp_should_run,
537 .thread_fn = cpuhp_thread_fun,
538 .thread_comm = "cpuhp/%u",
539 .selfparking = true,
540};
541
542void __init cpuhp_threads_init(void)
543{
544 BUG_ON(smpboot_register_percpu_thread(&cpuhp_threads));
545 kthread_unpark(this_cpu_read(cpuhp_state.thread));
546}
547
Linus Torvalds00b9b0a2010-05-27 10:32:08 -0700548#ifdef CONFIG_HOTPLUG_CPU
Linus Torvalds1da177e2005-04-16 15:20:36 -0700549EXPORT_SYMBOL(register_cpu_notifier);
Srivatsa S. Bhat93ae4f92014-03-11 02:04:14 +0530550EXPORT_SYMBOL(__register_cpu_notifier);
Mathias Krause71cf5ae2015-07-19 20:06:22 +0200551void unregister_cpu_notifier(struct notifier_block *nb)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700552{
Gautham R Shenoyd2219382008-01-25 21:08:01 +0100553 cpu_maps_update_begin();
Neil Brownbd5349c2006-10-17 00:10:35 -0700554 raw_notifier_chain_unregister(&cpu_chain, nb);
Gautham R Shenoyd2219382008-01-25 21:08:01 +0100555 cpu_maps_update_done();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700556}
557EXPORT_SYMBOL(unregister_cpu_notifier);
558
Mathias Krause71cf5ae2015-07-19 20:06:22 +0200559void __unregister_cpu_notifier(struct notifier_block *nb)
Srivatsa S. Bhat93ae4f92014-03-11 02:04:14 +0530560{
561 raw_notifier_chain_unregister(&cpu_chain, nb);
562}
563EXPORT_SYMBOL(__unregister_cpu_notifier);
564
Anton Vorontsove4cc2f82012-05-31 16:26:26 -0700565/**
566 * clear_tasks_mm_cpumask - Safely clear tasks' mm_cpumask for a CPU
567 * @cpu: a CPU id
568 *
569 * This function walks all processes, finds a valid mm struct for each one and
570 * then clears a corresponding bit in mm's cpumask. While this all sounds
571 * trivial, there are various non-obvious corner cases, which this function
572 * tries to solve in a safe manner.
573 *
574 * Also note that the function uses a somewhat relaxed locking scheme, so it may
575 * be called only for an already offlined CPU.
576 */
Anton Vorontsovcb792952012-05-31 16:26:22 -0700577void clear_tasks_mm_cpumask(int cpu)
578{
579 struct task_struct *p;
580
581 /*
582 * This function is called after the cpu is taken down and marked
583 * offline, so its not like new tasks will ever get this cpu set in
584 * their mm mask. -- Peter Zijlstra
585 * Thus, we may use rcu_read_lock() here, instead of grabbing
586 * full-fledged tasklist_lock.
587 */
Anton Vorontsove4cc2f82012-05-31 16:26:26 -0700588 WARN_ON(cpu_online(cpu));
Anton Vorontsovcb792952012-05-31 16:26:22 -0700589 rcu_read_lock();
590 for_each_process(p) {
591 struct task_struct *t;
592
Anton Vorontsove4cc2f82012-05-31 16:26:26 -0700593 /*
594 * Main thread might exit, but other threads may still have
595 * a valid mm. Find one.
596 */
Anton Vorontsovcb792952012-05-31 16:26:22 -0700597 t = find_lock_task_mm(p);
598 if (!t)
599 continue;
600 cpumask_clear_cpu(cpu, mm_cpumask(t->mm));
601 task_unlock(t);
602 }
603 rcu_read_unlock();
604}
605
Kirill Tkhaib728ca02014-06-25 12:19:55 +0400606static inline void check_for_tasks(int dead_cpu)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700607{
Kirill Tkhaib728ca02014-06-25 12:19:55 +0400608 struct task_struct *g, *p;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700609
Oleg Nesterova75a6062015-09-10 15:07:50 +0200610 read_lock(&tasklist_lock);
611 for_each_process_thread(g, p) {
Kirill Tkhaib728ca02014-06-25 12:19:55 +0400612 if (!p->on_rq)
613 continue;
614 /*
615 * We do the check with unlocked task_rq(p)->lock.
616 * Order the reading to do not warn about a task,
617 * which was running on this cpu in the past, and
618 * it's just been woken on another cpu.
619 */
620 rmb();
621 if (task_cpu(p) != dead_cpu)
622 continue;
623
624 pr_warn("Task %s (pid=%d) is on cpu %d (state=%ld, flags=%x)\n",
625 p->comm, task_pid_nr(p), dead_cpu, p->state, p->flags);
Oleg Nesterova75a6062015-09-10 15:07:50 +0200626 }
627 read_unlock(&tasklist_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700628}
629
Thomas Gleixner98458172016-02-26 18:43:25 +0000630static void cpu_notify_nofail(unsigned long val, unsigned int cpu)
631{
632 BUG_ON(cpu_notify(val, cpu));
633}
634
635static int notify_down_prepare(unsigned int cpu)
636{
637 int err, nr_calls = 0;
638
639 err = __cpu_notify(CPU_DOWN_PREPARE, cpu, -1, &nr_calls);
640 if (err) {
641 nr_calls--;
642 __cpu_notify(CPU_DOWN_FAILED, cpu, nr_calls, NULL);
643 pr_warn("%s: attempt to take down CPU %u failed\n",
644 __func__, cpu);
645 }
646 return err;
647}
648
Thomas Gleixner4baa0af2016-02-26 18:43:29 +0000649static int notify_dying(unsigned int cpu)
650{
651 cpu_notify(CPU_DYING, cpu);
652 return 0;
653}
654
Linus Torvalds1da177e2005-04-16 15:20:36 -0700655/* Take this CPU down. */
Mathias Krause71cf5ae2015-07-19 20:06:22 +0200656static int take_cpu_down(void *_param)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700657{
Thomas Gleixner4baa0af2016-02-26 18:43:29 +0000658 struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);
659 enum cpuhp_state target = max((int)st->target, CPUHP_AP_OFFLINE);
Thomas Gleixner090e77c2016-02-26 18:43:23 +0000660 int err, cpu = smp_processor_id();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700661
Linus Torvalds1da177e2005-04-16 15:20:36 -0700662 /* Ensure this CPU doesn't handle any more interrupts. */
663 err = __cpu_disable();
664 if (err < 0)
Zwane Mwaikambof3705132005-06-25 14:54:50 -0700665 return err;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700666
Thomas Gleixner4baa0af2016-02-26 18:43:29 +0000667 /* Invoke the former CPU_DYING callbacks */
668 for (; st->state > target; st->state--) {
669 struct cpuhp_step *step = cpuhp_ap_states + st->state;
670
671 cpuhp_invoke_callback(cpu, st->state, step->teardown);
672 }
Thomas Gleixner52c063d2015-04-03 02:37:24 +0200673 /* Give up timekeeping duties */
674 tick_handover_do_timer();
Thomas Gleixner14e568e2013-01-31 12:11:14 +0000675 /* Park the stopper thread */
Thomas Gleixner090e77c2016-02-26 18:43:23 +0000676 stop_machine_park(cpu);
Zwane Mwaikambof3705132005-06-25 14:54:50 -0700677 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700678}
679
Thomas Gleixner98458172016-02-26 18:43:25 +0000680static int takedown_cpu(unsigned int cpu)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700681{
Thomas Gleixner98458172016-02-26 18:43:25 +0000682 int err;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700683
Peter Zijlstra6acce3e2013-10-11 14:38:20 +0200684 /*
685 * By now we've cleared cpu_active_mask, wait for all preempt-disabled
686 * and RCU users of this state to go away such that all new such users
687 * will observe it.
688 *
689 * For CONFIG_PREEMPT we have preemptible RCU and its sync_rcu() might
Paul E. McKenney779de6c2015-06-10 13:34:41 -0700690 * not imply sync_sched(), so wait for both.
Michael wang106dd5a2013-11-13 11:10:56 +0800691 *
692 * Do sync before park smpboot threads to take care the rcu boost case.
Peter Zijlstra6acce3e2013-10-11 14:38:20 +0200693 */
Paul E. McKenney779de6c2015-06-10 13:34:41 -0700694 if (IS_ENABLED(CONFIG_PREEMPT))
695 synchronize_rcu_mult(call_rcu, call_rcu_sched);
696 else
697 synchronize_rcu();
Peter Zijlstra6acce3e2013-10-11 14:38:20 +0200698
Thomas Gleixner1cf4f622016-02-26 18:43:39 +0000699 /* Park the hotplug thread */
700 kthread_park(per_cpu_ptr(&cpuhp_state, cpu)->thread);
701
Peter Zijlstra6acce3e2013-10-11 14:38:20 +0200702 /*
Thomas Gleixnera8994182015-07-05 17:12:30 +0000703 * Prevent irq alloc/free while the dying cpu reorganizes the
704 * interrupt affinities.
705 */
706 irq_lock_sparse();
707
708 /*
Peter Zijlstra6acce3e2013-10-11 14:38:20 +0200709 * So now all preempt/rcu users must observe !cpu_active().
710 */
Thomas Gleixner090e77c2016-02-26 18:43:23 +0000711 err = stop_machine(take_cpu_down, NULL, cpumask_of(cpu));
Rusty Russell04321582008-07-28 12:16:29 -0500712 if (err) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700713 /* CPU didn't die: tell everyone. Can't complain. */
Thomas Gleixner090e77c2016-02-26 18:43:23 +0000714 cpu_notify_nofail(CPU_DOWN_FAILED, cpu);
Thomas Gleixnera8994182015-07-05 17:12:30 +0000715 irq_unlock_sparse();
Thomas Gleixner98458172016-02-26 18:43:25 +0000716 return err;
Satoru Takeuchi8fa1d7d2006-10-28 10:38:57 -0700717 }
Rusty Russell04321582008-07-28 12:16:29 -0500718 BUG_ON(cpu_online(cpu));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700719
Peter Zijlstra48c5ccae2010-11-13 19:32:29 +0100720 /*
721 * The migration_call() CPU_DYING callback will have removed all
722 * runnable tasks from the cpu, there's only the idle task left now
723 * that the migration thread is done doing the stop_machine thing.
Peter Zijlstra51a96c72010-11-19 20:37:53 +0100724 *
725 * Wait for the stop thread to go away.
Peter Zijlstra48c5ccae2010-11-13 19:32:29 +0100726 */
Paul E. McKenney528a25b2015-01-28 14:09:43 -0800727 while (!per_cpu(cpu_dead_idle, cpu))
Peter Zijlstra51a96c72010-11-19 20:37:53 +0100728 cpu_relax();
Paul E. McKenney528a25b2015-01-28 14:09:43 -0800729 smp_mb(); /* Read from cpu_dead_idle before __cpu_die(). */
730 per_cpu(cpu_dead_idle, cpu) = false;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700731
Thomas Gleixnera8994182015-07-05 17:12:30 +0000732 /* Interrupts are moved away from the dying cpu, reenable alloc/free */
733 irq_unlock_sparse();
734
Preeti U Murthy345527b2015-03-30 14:59:19 +0530735 hotplug_cpu__broadcast_tick_pull(cpu);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700736 /* This actually kills the CPU. */
737 __cpu_die(cpu);
738
Thomas Gleixnera49b1162015-04-03 02:38:05 +0200739 tick_cleanup_dead_cpu(cpu);
Thomas Gleixner98458172016-02-26 18:43:25 +0000740 return 0;
741}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700742
Thomas Gleixner98458172016-02-26 18:43:25 +0000743static int notify_dead(unsigned int cpu)
744{
745 cpu_notify_nofail(CPU_DEAD, cpu);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700746 check_for_tasks(cpu);
Thomas Gleixner98458172016-02-26 18:43:25 +0000747 return 0;
748}
749
Thomas Gleixnercff7d372016-02-26 18:43:28 +0000750#else
751#define notify_down_prepare NULL
752#define takedown_cpu NULL
753#define notify_dead NULL
Thomas Gleixner4baa0af2016-02-26 18:43:29 +0000754#define notify_dying NULL
Thomas Gleixnercff7d372016-02-26 18:43:28 +0000755#endif
756
757#ifdef CONFIG_HOTPLUG_CPU
Thomas Gleixnercff7d372016-02-26 18:43:28 +0000758
Thomas Gleixner98458172016-02-26 18:43:25 +0000759/* Requires cpu_add_remove_lock to be held */
Thomas Gleixneraf1f4042016-02-26 18:43:30 +0000760static int __ref _cpu_down(unsigned int cpu, int tasks_frozen,
761 enum cpuhp_state target)
Thomas Gleixner98458172016-02-26 18:43:25 +0000762{
Thomas Gleixnercff7d372016-02-26 18:43:28 +0000763 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
764 int prev_state, ret = 0;
765 bool hasdied = false;
Thomas Gleixner98458172016-02-26 18:43:25 +0000766
767 if (num_online_cpus() == 1)
768 return -EBUSY;
769
Thomas Gleixner757c9892016-02-26 18:43:32 +0000770 if (!cpu_present(cpu))
Thomas Gleixner98458172016-02-26 18:43:25 +0000771 return -EINVAL;
772
773 cpu_hotplug_begin();
774
775 cpuhp_tasks_frozen = tasks_frozen;
776
Thomas Gleixnercff7d372016-02-26 18:43:28 +0000777 prev_state = st->state;
Thomas Gleixneraf1f4042016-02-26 18:43:30 +0000778 st->target = target;
Thomas Gleixner1cf4f622016-02-26 18:43:39 +0000779 /*
780 * If the current CPU state is in the range of the AP hotplug thread,
781 * then we need to kick the thread.
782 */
783 if (st->state >= CPUHP_KICK_AP_THREAD) {
784 ret = cpuhp_kick_ap_work(cpu);
785 /*
786 * The AP side has done the error rollback already. Just
787 * return the error code..
788 */
789 if (ret)
790 goto out;
791
792 /*
793 * We might have stopped still in the range of the AP hotplug
794 * thread. Nothing to do anymore.
795 */
796 if (st->state >= CPUHP_KICK_AP_THREAD)
797 goto out;
798 }
799 /*
800 * The AP brought itself down below CPUHP_KICK_AP_THREAD. So we need
801 * to do the further cleanups.
802 */
Thomas Gleixner2e1a3482016-02-26 18:43:37 +0000803 ret = cpuhp_down_callbacks(cpu, st, cpuhp_bp_states, target);
Thomas Gleixner98458172016-02-26 18:43:25 +0000804
Thomas Gleixnercff7d372016-02-26 18:43:28 +0000805 hasdied = prev_state != st->state && st->state == CPUHP_OFFLINE;
Thomas Gleixner1cf4f622016-02-26 18:43:39 +0000806out:
Gautham R Shenoyd2219382008-01-25 21:08:01 +0100807 cpu_hotplug_done();
Thomas Gleixnercff7d372016-02-26 18:43:28 +0000808 /* This post dead nonsense must die */
809 if (!ret && hasdied)
Thomas Gleixner090e77c2016-02-26 18:43:23 +0000810 cpu_notify_nofail(CPU_POST_DEAD, cpu);
Thomas Gleixnercff7d372016-02-26 18:43:28 +0000811 return ret;
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -0700812}
813
Thomas Gleixneraf1f4042016-02-26 18:43:30 +0000814static int do_cpu_down(unsigned int cpu, enum cpuhp_state target)
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -0700815{
Heiko Carstens9ea09af2008-12-22 12:36:30 +0100816 int err;
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -0700817
Gautham R Shenoyd2219382008-01-25 21:08:01 +0100818 cpu_maps_update_begin();
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -0700819
Max Krasnyanskye761b772008-07-15 04:43:49 -0700820 if (cpu_hotplug_disabled) {
821 err = -EBUSY;
822 goto out;
823 }
824
Thomas Gleixneraf1f4042016-02-26 18:43:30 +0000825 err = _cpu_down(cpu, 0, target);
Max Krasnyanskye761b772008-07-15 04:43:49 -0700826
Max Krasnyanskye761b772008-07-15 04:43:49 -0700827out:
Gautham R Shenoyd2219382008-01-25 21:08:01 +0100828 cpu_maps_update_done();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700829 return err;
830}
Thomas Gleixneraf1f4042016-02-26 18:43:30 +0000831int cpu_down(unsigned int cpu)
832{
833 return do_cpu_down(cpu, CPUHP_OFFLINE);
834}
Zhang Ruib62b8ef2008-04-29 02:35:56 -0400835EXPORT_SYMBOL(cpu_down);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700836#endif /*CONFIG_HOTPLUG_CPU*/
837
Thomas Gleixner4baa0af2016-02-26 18:43:29 +0000838/**
839 * notify_cpu_starting(cpu) - call the CPU_STARTING notifiers
840 * @cpu: cpu that just started
841 *
842 * This function calls the cpu_chain notifiers with CPU_STARTING.
843 * It must be called by the arch code on the new cpu, before the new cpu
844 * enables interrupts and before the "boot" cpu returns from __cpu_up().
845 */
846void notify_cpu_starting(unsigned int cpu)
847{
848 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
849 enum cpuhp_state target = min((int)st->target, CPUHP_AP_ONLINE);
850
851 while (st->state < target) {
852 struct cpuhp_step *step;
853
854 st->state++;
855 step = cpuhp_ap_states + st->state;
856 cpuhp_invoke_callback(cpu, st->state, step->startup);
857 }
858}
859
Thomas Gleixner949338e2016-02-26 18:43:35 +0000860/*
861 * Called from the idle task. We need to set active here, so we can kick off
862 * the stopper thread.
863 */
864static int cpuhp_set_cpu_active(unsigned int cpu)
865{
Thomas Gleixner1cf4f622016-02-26 18:43:39 +0000866 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
867
Thomas Gleixner949338e2016-02-26 18:43:35 +0000868 /* The cpu is marked online, set it active now */
869 set_cpu_active(cpu, true);
Thomas Gleixner1cf4f622016-02-26 18:43:39 +0000870 /* Unpark the stopper thread and the hotplug thread */
Thomas Gleixner949338e2016-02-26 18:43:35 +0000871 stop_machine_unpark(cpu);
Thomas Gleixner1cf4f622016-02-26 18:43:39 +0000872 kthread_unpark(st->thread);
Thomas Gleixner949338e2016-02-26 18:43:35 +0000873 return 0;
874}
875
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -0700876/* Requires cpu_add_remove_lock to be held */
Thomas Gleixneraf1f4042016-02-26 18:43:30 +0000877static int _cpu_up(unsigned int cpu, int tasks_frozen, enum cpuhp_state target)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700878{
Thomas Gleixnercff7d372016-02-26 18:43:28 +0000879 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
Suresh Siddha3bb5d2e2012-04-20 17:08:50 -0700880 struct task_struct *idle;
Thomas Gleixner2e1a3482016-02-26 18:43:37 +0000881 int ret = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700882
Gautham R Shenoyd2219382008-01-25 21:08:01 +0100883 cpu_hotplug_begin();
Thomas Gleixner38498a62012-04-20 13:05:44 +0000884
Thomas Gleixner757c9892016-02-26 18:43:32 +0000885 if (!cpu_present(cpu)) {
Yasuaki Ishimatsu5e5041f2012-10-23 01:30:54 +0200886 ret = -EINVAL;
887 goto out;
888 }
889
Thomas Gleixner757c9892016-02-26 18:43:32 +0000890 /*
891 * The caller of do_cpu_up might have raced with another
892 * caller. Ignore it for now.
893 */
894 if (st->state >= target)
Thomas Gleixner38498a62012-04-20 13:05:44 +0000895 goto out;
Thomas Gleixner757c9892016-02-26 18:43:32 +0000896
897 if (st->state == CPUHP_OFFLINE) {
898 /* Let it fail before we try to bring the cpu up */
899 idle = idle_thread_get(cpu);
900 if (IS_ERR(idle)) {
901 ret = PTR_ERR(idle);
902 goto out;
903 }
Suresh Siddha3bb5d2e2012-04-20 17:08:50 -0700904 }
Thomas Gleixner38498a62012-04-20 13:05:44 +0000905
Thomas Gleixnerba997462016-02-26 18:43:24 +0000906 cpuhp_tasks_frozen = tasks_frozen;
907
Thomas Gleixneraf1f4042016-02-26 18:43:30 +0000908 st->target = target;
Thomas Gleixner1cf4f622016-02-26 18:43:39 +0000909 /*
910 * If the current CPU state is in the range of the AP hotplug thread,
911 * then we need to kick the thread once more.
912 */
913 if (st->state >= CPUHP_KICK_AP_THREAD) {
914 ret = cpuhp_kick_ap_work(cpu);
915 /*
916 * The AP side has done the error rollback already. Just
917 * return the error code..
918 */
919 if (ret)
920 goto out;
921 }
922
923 /*
924 * Try to reach the target state. We max out on the BP at
925 * CPUHP_KICK_AP_THREAD. After that the AP hotplug thread is
926 * responsible for bringing it up to the target state.
927 */
928 target = min((int)target, CPUHP_KICK_AP_THREAD);
Thomas Gleixner2e1a3482016-02-26 18:43:37 +0000929 ret = cpuhp_up_callbacks(cpu, st, cpuhp_bp_states, target);
Thomas Gleixner38498a62012-04-20 13:05:44 +0000930out:
Gautham R Shenoyd2219382008-01-25 21:08:01 +0100931 cpu_hotplug_done();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700932 return ret;
933}
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -0700934
Thomas Gleixneraf1f4042016-02-26 18:43:30 +0000935static int do_cpu_up(unsigned int cpu, enum cpuhp_state target)
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -0700936{
937 int err = 0;
minskey guocf234222010-05-24 14:32:41 -0700938
Rusty Russelle0b582e2009-01-01 10:12:28 +1030939 if (!cpu_possible(cpu)) {
Fabian Frederick84117da2014-06-04 16:11:17 -0700940 pr_err("can't online cpu %d because it is not configured as may-hotadd at boot time\n",
941 cpu);
Chen Gong87d5e0232010-03-05 13:42:38 -0800942#if defined(CONFIG_IA64)
Fabian Frederick84117da2014-06-04 16:11:17 -0700943 pr_err("please check additional_cpus= boot parameter\n");
KAMEZAWA Hiroyuki73e753a2007-10-18 23:40:47 -0700944#endif
945 return -EINVAL;
946 }
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -0700947
Toshi Kani01b0f192013-11-12 15:07:25 -0800948 err = try_online_node(cpu_to_node(cpu));
949 if (err)
950 return err;
minskey guocf234222010-05-24 14:32:41 -0700951
Gautham R Shenoyd2219382008-01-25 21:08:01 +0100952 cpu_maps_update_begin();
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -0700953
Max Krasnyanskye761b772008-07-15 04:43:49 -0700954 if (cpu_hotplug_disabled) {
955 err = -EBUSY;
956 goto out;
957 }
958
Thomas Gleixneraf1f4042016-02-26 18:43:30 +0000959 err = _cpu_up(cpu, 0, target);
Max Krasnyanskye761b772008-07-15 04:43:49 -0700960out:
Gautham R Shenoyd2219382008-01-25 21:08:01 +0100961 cpu_maps_update_done();
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -0700962 return err;
963}
Thomas Gleixneraf1f4042016-02-26 18:43:30 +0000964
965int cpu_up(unsigned int cpu)
966{
967 return do_cpu_up(cpu, CPUHP_ONLINE);
968}
Paul E. McKenneya513f6b2011-12-11 21:54:45 -0800969EXPORT_SYMBOL_GPL(cpu_up);
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -0700970
Rafael J. Wysockif3de4be2007-08-30 23:56:29 -0700971#ifdef CONFIG_PM_SLEEP_SMP
Rusty Russelle0b582e2009-01-01 10:12:28 +1030972static cpumask_var_t frozen_cpus;
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -0700973
974int disable_nonboot_cpus(void)
975{
Rafael J. Wysockie9a5f422010-05-27 22:16:22 +0200976 int cpu, first_cpu, error = 0;
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -0700977
Gautham R Shenoyd2219382008-01-25 21:08:01 +0100978 cpu_maps_update_begin();
Rusty Russelle0b582e2009-01-01 10:12:28 +1030979 first_cpu = cpumask_first(cpu_online_mask);
Xiaotian Feng9ee349a2009-12-16 18:04:32 +0100980 /*
981 * We take down all of the non-boot CPUs in one shot to avoid races
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -0700982 * with the userspace trying to use the CPU hotplug at the same time
983 */
Rusty Russelle0b582e2009-01-01 10:12:28 +1030984 cpumask_clear(frozen_cpus);
Peter Zijlstra6ad4c182009-11-25 13:31:39 +0100985
Fabian Frederick84117da2014-06-04 16:11:17 -0700986 pr_info("Disabling non-boot CPUs ...\n");
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -0700987 for_each_online_cpu(cpu) {
988 if (cpu == first_cpu)
989 continue;
Todd E Brandtbb3632c2014-06-06 05:40:17 -0700990 trace_suspend_resume(TPS("CPU_OFF"), cpu, true);
Thomas Gleixneraf1f4042016-02-26 18:43:30 +0000991 error = _cpu_down(cpu, 1, CPUHP_OFFLINE);
Todd E Brandtbb3632c2014-06-06 05:40:17 -0700992 trace_suspend_resume(TPS("CPU_OFF"), cpu, false);
Mike Travisfeae3202009-11-17 18:22:13 -0600993 if (!error)
Rusty Russelle0b582e2009-01-01 10:12:28 +1030994 cpumask_set_cpu(cpu, frozen_cpus);
Mike Travisfeae3202009-11-17 18:22:13 -0600995 else {
Fabian Frederick84117da2014-06-04 16:11:17 -0700996 pr_err("Error taking CPU%d down: %d\n", cpu, error);
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -0700997 break;
998 }
999 }
Joseph Cihula86886e52009-06-30 19:31:07 -07001000
Vitaly Kuznetsov89af7ba2015-08-05 00:52:46 -07001001 if (!error)
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -07001002 BUG_ON(num_online_cpus() > 1);
Vitaly Kuznetsov89af7ba2015-08-05 00:52:46 -07001003 else
Fabian Frederick84117da2014-06-04 16:11:17 -07001004 pr_err("Non-boot CPUs are not disabled\n");
Vitaly Kuznetsov89af7ba2015-08-05 00:52:46 -07001005
1006 /*
1007 * Make sure the CPUs won't be enabled by someone else. We need to do
1008 * this even in case of failure as all disable_nonboot_cpus() users are
1009 * supposed to do enable_nonboot_cpus() on the failure path.
1010 */
1011 cpu_hotplug_disabled++;
1012
Gautham R Shenoyd2219382008-01-25 21:08:01 +01001013 cpu_maps_update_done();
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -07001014 return error;
1015}
1016
Suresh Siddhad0af9ee2009-08-19 18:05:36 -07001017void __weak arch_enable_nonboot_cpus_begin(void)
1018{
1019}
1020
1021void __weak arch_enable_nonboot_cpus_end(void)
1022{
1023}
1024
Mathias Krause71cf5ae2015-07-19 20:06:22 +02001025void enable_nonboot_cpus(void)
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -07001026{
1027 int cpu, error;
1028
1029 /* Allow everyone to use the CPU hotplug again */
Gautham R Shenoyd2219382008-01-25 21:08:01 +01001030 cpu_maps_update_begin();
Vitaly Kuznetsov89af7ba2015-08-05 00:52:46 -07001031 WARN_ON(--cpu_hotplug_disabled < 0);
Rusty Russelle0b582e2009-01-01 10:12:28 +10301032 if (cpumask_empty(frozen_cpus))
Rafael J. Wysocki1d64b9c2007-04-01 23:49:49 -07001033 goto out;
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -07001034
Fabian Frederick84117da2014-06-04 16:11:17 -07001035 pr_info("Enabling non-boot CPUs ...\n");
Suresh Siddhad0af9ee2009-08-19 18:05:36 -07001036
1037 arch_enable_nonboot_cpus_begin();
1038
Rusty Russelle0b582e2009-01-01 10:12:28 +10301039 for_each_cpu(cpu, frozen_cpus) {
Todd E Brandtbb3632c2014-06-06 05:40:17 -07001040 trace_suspend_resume(TPS("CPU_ON"), cpu, true);
Thomas Gleixneraf1f4042016-02-26 18:43:30 +00001041 error = _cpu_up(cpu, 1, CPUHP_ONLINE);
Todd E Brandtbb3632c2014-06-06 05:40:17 -07001042 trace_suspend_resume(TPS("CPU_ON"), cpu, false);
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -07001043 if (!error) {
Fabian Frederick84117da2014-06-04 16:11:17 -07001044 pr_info("CPU%d is up\n", cpu);
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -07001045 continue;
1046 }
Fabian Frederick84117da2014-06-04 16:11:17 -07001047 pr_warn("Error taking CPU%d up: %d\n", cpu, error);
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -07001048 }
Suresh Siddhad0af9ee2009-08-19 18:05:36 -07001049
1050 arch_enable_nonboot_cpus_end();
1051
Rusty Russelle0b582e2009-01-01 10:12:28 +10301052 cpumask_clear(frozen_cpus);
Rafael J. Wysocki1d64b9c2007-04-01 23:49:49 -07001053out:
Gautham R Shenoyd2219382008-01-25 21:08:01 +01001054 cpu_maps_update_done();
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -07001055}
Rusty Russelle0b582e2009-01-01 10:12:28 +10301056
Fenghua Yud7268a32011-11-15 21:59:31 +01001057static int __init alloc_frozen_cpus(void)
Rusty Russelle0b582e2009-01-01 10:12:28 +10301058{
1059 if (!alloc_cpumask_var(&frozen_cpus, GFP_KERNEL|__GFP_ZERO))
1060 return -ENOMEM;
1061 return 0;
1062}
1063core_initcall(alloc_frozen_cpus);
Srivatsa S. Bhat79cfbdf2011-11-03 00:59:25 +01001064
1065/*
Srivatsa S. Bhat79cfbdf2011-11-03 00:59:25 +01001066 * When callbacks for CPU hotplug notifications are being executed, we must
1067 * ensure that the state of the system with respect to the tasks being frozen
1068 * or not, as reported by the notification, remains unchanged *throughout the
1069 * duration* of the execution of the callbacks.
1070 * Hence we need to prevent the freezer from racing with regular CPU hotplug.
1071 *
1072 * This synchronization is implemented by mutually excluding regular CPU
1073 * hotplug and Suspend/Hibernate call paths by hooking onto the Suspend/
1074 * Hibernate notifications.
1075 */
1076static int
1077cpu_hotplug_pm_callback(struct notifier_block *nb,
1078 unsigned long action, void *ptr)
1079{
1080 switch (action) {
1081
1082 case PM_SUSPEND_PREPARE:
1083 case PM_HIBERNATION_PREPARE:
Srivatsa S. Bhat16e53db2013-06-12 14:04:36 -07001084 cpu_hotplug_disable();
Srivatsa S. Bhat79cfbdf2011-11-03 00:59:25 +01001085 break;
1086
1087 case PM_POST_SUSPEND:
1088 case PM_POST_HIBERNATION:
Srivatsa S. Bhat16e53db2013-06-12 14:04:36 -07001089 cpu_hotplug_enable();
Srivatsa S. Bhat79cfbdf2011-11-03 00:59:25 +01001090 break;
1091
1092 default:
1093 return NOTIFY_DONE;
1094 }
1095
1096 return NOTIFY_OK;
1097}
1098
1099
Fenghua Yud7268a32011-11-15 21:59:31 +01001100static int __init cpu_hotplug_pm_sync_init(void)
Srivatsa S. Bhat79cfbdf2011-11-03 00:59:25 +01001101{
Fenghua Yu6e32d472012-11-13 11:32:43 -08001102 /*
1103 * cpu_hotplug_pm_callback has higher priority than x86
1104 * bsp_pm_callback which depends on cpu_hotplug_pm_callback
1105 * to disable cpu hotplug to avoid cpu hotplug race.
1106 */
Srivatsa S. Bhat79cfbdf2011-11-03 00:59:25 +01001107 pm_notifier(cpu_hotplug_pm_callback, 0);
1108 return 0;
1109}
1110core_initcall(cpu_hotplug_pm_sync_init);
1111
Rafael J. Wysockif3de4be2007-08-30 23:56:29 -07001112#endif /* CONFIG_PM_SLEEP_SMP */
Max Krasnyansky68f4f1e2008-05-29 11:17:02 -07001113
1114#endif /* CONFIG_SMP */
Mike Travisb8d317d2008-07-24 18:21:29 -07001115
Thomas Gleixnercff7d372016-02-26 18:43:28 +00001116/* Boot processor state steps */
1117static struct cpuhp_step cpuhp_bp_states[] = {
1118 [CPUHP_OFFLINE] = {
1119 .name = "offline",
1120 .startup = NULL,
1121 .teardown = NULL,
1122 },
1123#ifdef CONFIG_SMP
1124 [CPUHP_CREATE_THREADS]= {
1125 .name = "threads:create",
1126 .startup = smpboot_create_threads,
1127 .teardown = NULL,
Thomas Gleixner757c9892016-02-26 18:43:32 +00001128 .cant_stop = true,
Thomas Gleixnercff7d372016-02-26 18:43:28 +00001129 },
1130 [CPUHP_NOTIFY_PREPARE] = {
1131 .name = "notify:prepare",
1132 .startup = notify_prepare,
1133 .teardown = notify_dead,
1134 .skip_onerr = true,
Thomas Gleixner757c9892016-02-26 18:43:32 +00001135 .cant_stop = true,
Thomas Gleixnercff7d372016-02-26 18:43:28 +00001136 },
1137 [CPUHP_BRINGUP_CPU] = {
1138 .name = "cpu:bringup",
1139 .startup = bringup_cpu,
Thomas Gleixner4baa0af2016-02-26 18:43:29 +00001140 .teardown = NULL,
Thomas Gleixner757c9892016-02-26 18:43:32 +00001141 .cant_stop = true,
Thomas Gleixner4baa0af2016-02-26 18:43:29 +00001142 },
1143 [CPUHP_TEARDOWN_CPU] = {
1144 .name = "cpu:teardown",
1145 .startup = NULL,
Thomas Gleixnercff7d372016-02-26 18:43:28 +00001146 .teardown = takedown_cpu,
Thomas Gleixner757c9892016-02-26 18:43:32 +00001147 .cant_stop = true,
Thomas Gleixnercff7d372016-02-26 18:43:28 +00001148 },
Thomas Gleixner949338e2016-02-26 18:43:35 +00001149 [CPUHP_CPU_SET_ACTIVE] = {
1150 .name = "cpu:active",
1151 .startup = cpuhp_set_cpu_active,
1152 .teardown = NULL,
1153 },
Thomas Gleixner1cf4f622016-02-26 18:43:39 +00001154 [CPUHP_KICK_AP_THREAD] = {
1155 .name = "cpuhp:kickthread",
1156 .startup = cpuhp_kick_ap_work,
1157 .teardown = cpuhp_kick_ap_work,
Thomas Gleixnercff7d372016-02-26 18:43:28 +00001158 },
1159#endif
Thomas Gleixner1cf4f622016-02-26 18:43:39 +00001160 [CPUHP_BP_ONLINE] = {
Thomas Gleixnercff7d372016-02-26 18:43:28 +00001161 .name = "online",
1162 .startup = NULL,
1163 .teardown = NULL,
1164 },
1165};
1166
Thomas Gleixner4baa0af2016-02-26 18:43:29 +00001167/* Application processor state steps */
1168static struct cpuhp_step cpuhp_ap_states[] = {
1169#ifdef CONFIG_SMP
1170 [CPUHP_AP_NOTIFY_STARTING] = {
1171 .name = "notify:starting",
1172 .startup = notify_starting,
1173 .teardown = notify_dying,
1174 .skip_onerr = true,
Thomas Gleixner757c9892016-02-26 18:43:32 +00001175 .cant_stop = true,
Thomas Gleixner4baa0af2016-02-26 18:43:29 +00001176 },
Thomas Gleixner1cf4f622016-02-26 18:43:39 +00001177 [CPUHP_AP_SMPBOOT_THREADS] = {
1178 .name = "smpboot:threads",
1179 .startup = smpboot_unpark_threads,
1180 .teardown = smpboot_park_threads,
1181 },
1182 [CPUHP_AP_NOTIFY_ONLINE] = {
1183 .name = "notify:online",
1184 .startup = notify_online,
1185 .teardown = notify_down_prepare,
1186 },
Thomas Gleixner4baa0af2016-02-26 18:43:29 +00001187#endif
1188 [CPUHP_ONLINE] = {
1189 .name = "online",
1190 .startup = NULL,
1191 .teardown = NULL,
1192 },
1193};
1194
Thomas Gleixner5b7aa872016-02-26 18:43:33 +00001195/* Sanity check for callbacks */
1196static int cpuhp_cb_check(enum cpuhp_state state)
1197{
1198 if (state <= CPUHP_OFFLINE || state >= CPUHP_ONLINE)
1199 return -EINVAL;
1200 return 0;
1201}
1202
Thomas Gleixner98f8cdc2016-02-26 18:43:31 +00001203static bool cpuhp_is_ap_state(enum cpuhp_state state)
1204{
Thomas Gleixner1cf4f622016-02-26 18:43:39 +00001205 if (state >= CPUHP_AP_OFFLINE && state <= CPUHP_AP_ONLINE)
1206 return true;
1207 return state > CPUHP_BP_ONLINE;
Thomas Gleixner98f8cdc2016-02-26 18:43:31 +00001208}
1209
1210static struct cpuhp_step *cpuhp_get_step(enum cpuhp_state state)
1211{
1212 struct cpuhp_step *sp;
1213
1214 sp = cpuhp_is_ap_state(state) ? cpuhp_ap_states : cpuhp_bp_states;
1215 return sp + state;
1216}
1217
Thomas Gleixner5b7aa872016-02-26 18:43:33 +00001218static void cpuhp_store_callbacks(enum cpuhp_state state,
1219 const char *name,
1220 int (*startup)(unsigned int cpu),
1221 int (*teardown)(unsigned int cpu))
1222{
1223 /* (Un)Install the callbacks for further cpu hotplug operations */
1224 struct cpuhp_step *sp;
1225
1226 mutex_lock(&cpuhp_state_mutex);
1227 sp = cpuhp_get_step(state);
1228 sp->startup = startup;
1229 sp->teardown = teardown;
1230 sp->name = name;
1231 mutex_unlock(&cpuhp_state_mutex);
1232}
1233
1234static void *cpuhp_get_teardown_cb(enum cpuhp_state state)
1235{
1236 return cpuhp_get_step(state)->teardown;
1237}
1238
Thomas Gleixner5b7aa872016-02-26 18:43:33 +00001239/*
1240 * Call the startup/teardown function for a step either on the AP or
1241 * on the current CPU.
1242 */
1243static int cpuhp_issue_call(int cpu, enum cpuhp_state state,
1244 int (*cb)(unsigned int), bool bringup)
1245{
1246 int ret;
1247
1248 if (!cb)
1249 return 0;
Thomas Gleixner5b7aa872016-02-26 18:43:33 +00001250 /*
1251 * The non AP bound callbacks can fail on bringup. On teardown
1252 * e.g. module removal we crash for now.
1253 */
Thomas Gleixner1cf4f622016-02-26 18:43:39 +00001254#ifdef CONFIG_SMP
1255 if (cpuhp_is_ap_state(state))
1256 ret = cpuhp_invoke_ap_callback(cpu, state, cb);
1257 else
1258 ret = cpuhp_invoke_callback(cpu, state, cb);
1259#else
1260 ret = cpuhp_invoke_callback(cpu, state, cb);
1261#endif
Thomas Gleixner5b7aa872016-02-26 18:43:33 +00001262 BUG_ON(ret && !bringup);
1263 return ret;
1264}
1265
1266/*
1267 * Called from __cpuhp_setup_state on a recoverable failure.
1268 *
1269 * Note: The teardown callbacks for rollback are not allowed to fail!
1270 */
1271static void cpuhp_rollback_install(int failedcpu, enum cpuhp_state state,
1272 int (*teardown)(unsigned int cpu))
1273{
1274 int cpu;
1275
1276 if (!teardown)
1277 return;
1278
1279 /* Roll back the already executed steps on the other cpus */
1280 for_each_present_cpu(cpu) {
1281 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1282 int cpustate = st->state;
1283
1284 if (cpu >= failedcpu)
1285 break;
1286
1287 /* Did we invoke the startup call on that cpu ? */
1288 if (cpustate >= state)
1289 cpuhp_issue_call(cpu, state, teardown, false);
1290 }
1291}
1292
1293/*
1294 * Returns a free for dynamic slot assignment of the Online state. The states
1295 * are protected by the cpuhp_slot_states mutex and an empty slot is identified
1296 * by having no name assigned.
1297 */
1298static int cpuhp_reserve_state(enum cpuhp_state state)
1299{
1300 enum cpuhp_state i;
1301
1302 mutex_lock(&cpuhp_state_mutex);
Thomas Gleixner1cf4f622016-02-26 18:43:39 +00001303 for (i = CPUHP_AP_ONLINE_DYN; i <= CPUHP_AP_ONLINE_DYN_END; i++) {
1304 if (cpuhp_ap_states[i].name)
Thomas Gleixner5b7aa872016-02-26 18:43:33 +00001305 continue;
1306
Thomas Gleixner1cf4f622016-02-26 18:43:39 +00001307 cpuhp_ap_states[i].name = "Reserved";
Thomas Gleixner5b7aa872016-02-26 18:43:33 +00001308 mutex_unlock(&cpuhp_state_mutex);
1309 return i;
1310 }
1311 mutex_unlock(&cpuhp_state_mutex);
1312 WARN(1, "No more dynamic states available for CPU hotplug\n");
1313 return -ENOSPC;
1314}
1315
1316/**
1317 * __cpuhp_setup_state - Setup the callbacks for an hotplug machine state
1318 * @state: The state to setup
1319 * @invoke: If true, the startup function is invoked for cpus where
1320 * cpu state >= @state
1321 * @startup: startup callback function
1322 * @teardown: teardown callback function
1323 *
1324 * Returns 0 if successful, otherwise a proper error code
1325 */
1326int __cpuhp_setup_state(enum cpuhp_state state,
1327 const char *name, bool invoke,
1328 int (*startup)(unsigned int cpu),
1329 int (*teardown)(unsigned int cpu))
1330{
1331 int cpu, ret = 0;
1332 int dyn_state = 0;
1333
1334 if (cpuhp_cb_check(state) || !name)
1335 return -EINVAL;
1336
1337 get_online_cpus();
1338
1339 /* currently assignments for the ONLINE state are possible */
Thomas Gleixner1cf4f622016-02-26 18:43:39 +00001340 if (state == CPUHP_AP_ONLINE_DYN) {
Thomas Gleixner5b7aa872016-02-26 18:43:33 +00001341 dyn_state = 1;
1342 ret = cpuhp_reserve_state(state);
1343 if (ret < 0)
1344 goto out;
1345 state = ret;
1346 }
1347
1348 cpuhp_store_callbacks(state, name, startup, teardown);
1349
1350 if (!invoke || !startup)
1351 goto out;
1352
1353 /*
1354 * Try to call the startup callback for each present cpu
1355 * depending on the hotplug state of the cpu.
1356 */
1357 for_each_present_cpu(cpu) {
1358 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1359 int cpustate = st->state;
1360
1361 if (cpustate < state)
1362 continue;
1363
1364 ret = cpuhp_issue_call(cpu, state, startup, true);
1365 if (ret) {
1366 cpuhp_rollback_install(cpu, state, teardown);
1367 cpuhp_store_callbacks(state, NULL, NULL, NULL);
1368 goto out;
1369 }
1370 }
1371out:
1372 put_online_cpus();
1373 if (!ret && dyn_state)
1374 return state;
1375 return ret;
1376}
1377EXPORT_SYMBOL(__cpuhp_setup_state);
1378
1379/**
1380 * __cpuhp_remove_state - Remove the callbacks for an hotplug machine state
1381 * @state: The state to remove
1382 * @invoke: If true, the teardown function is invoked for cpus where
1383 * cpu state >= @state
1384 *
1385 * The teardown callback is currently not allowed to fail. Think
1386 * about module removal!
1387 */
1388void __cpuhp_remove_state(enum cpuhp_state state, bool invoke)
1389{
1390 int (*teardown)(unsigned int cpu) = cpuhp_get_teardown_cb(state);
1391 int cpu;
1392
1393 BUG_ON(cpuhp_cb_check(state));
1394
1395 get_online_cpus();
1396
1397 if (!invoke || !teardown)
1398 goto remove;
1399
1400 /*
1401 * Call the teardown callback for each present cpu depending
1402 * on the hotplug state of the cpu. This function is not
1403 * allowed to fail currently!
1404 */
1405 for_each_present_cpu(cpu) {
1406 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1407 int cpustate = st->state;
1408
1409 if (cpustate >= state)
1410 cpuhp_issue_call(cpu, state, teardown, false);
1411 }
1412remove:
1413 cpuhp_store_callbacks(state, NULL, NULL, NULL);
1414 put_online_cpus();
1415}
1416EXPORT_SYMBOL(__cpuhp_remove_state);
1417
Thomas Gleixner98f8cdc2016-02-26 18:43:31 +00001418#if defined(CONFIG_SYSFS) && defined(CONFIG_HOTPLUG_CPU)
1419static ssize_t show_cpuhp_state(struct device *dev,
1420 struct device_attribute *attr, char *buf)
1421{
1422 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);
1423
1424 return sprintf(buf, "%d\n", st->state);
1425}
1426static DEVICE_ATTR(state, 0444, show_cpuhp_state, NULL);
1427
Thomas Gleixner757c9892016-02-26 18:43:32 +00001428static ssize_t write_cpuhp_target(struct device *dev,
1429 struct device_attribute *attr,
1430 const char *buf, size_t count)
1431{
1432 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);
1433 struct cpuhp_step *sp;
1434 int target, ret;
1435
1436 ret = kstrtoint(buf, 10, &target);
1437 if (ret)
1438 return ret;
1439
1440#ifdef CONFIG_CPU_HOTPLUG_STATE_CONTROL
1441 if (target < CPUHP_OFFLINE || target > CPUHP_ONLINE)
1442 return -EINVAL;
1443#else
1444 if (target != CPUHP_OFFLINE && target != CPUHP_ONLINE)
1445 return -EINVAL;
1446#endif
1447
1448 ret = lock_device_hotplug_sysfs();
1449 if (ret)
1450 return ret;
1451
1452 mutex_lock(&cpuhp_state_mutex);
1453 sp = cpuhp_get_step(target);
1454 ret = !sp->name || sp->cant_stop ? -EINVAL : 0;
1455 mutex_unlock(&cpuhp_state_mutex);
1456 if (ret)
1457 return ret;
1458
1459 if (st->state < target)
1460 ret = do_cpu_up(dev->id, target);
1461 else
1462 ret = do_cpu_down(dev->id, target);
1463
1464 unlock_device_hotplug();
1465 return ret ? ret : count;
1466}
1467
Thomas Gleixner98f8cdc2016-02-26 18:43:31 +00001468static ssize_t show_cpuhp_target(struct device *dev,
1469 struct device_attribute *attr, char *buf)
1470{
1471 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);
1472
1473 return sprintf(buf, "%d\n", st->target);
1474}
Thomas Gleixner757c9892016-02-26 18:43:32 +00001475static DEVICE_ATTR(target, 0644, show_cpuhp_target, write_cpuhp_target);
Thomas Gleixner98f8cdc2016-02-26 18:43:31 +00001476
1477static struct attribute *cpuhp_cpu_attrs[] = {
1478 &dev_attr_state.attr,
1479 &dev_attr_target.attr,
1480 NULL
1481};
1482
1483static struct attribute_group cpuhp_cpu_attr_group = {
1484 .attrs = cpuhp_cpu_attrs,
1485 .name = "hotplug",
1486 NULL
1487};
1488
1489static ssize_t show_cpuhp_states(struct device *dev,
1490 struct device_attribute *attr, char *buf)
1491{
1492 ssize_t cur, res = 0;
1493 int i;
1494
1495 mutex_lock(&cpuhp_state_mutex);
Thomas Gleixner757c9892016-02-26 18:43:32 +00001496 for (i = CPUHP_OFFLINE; i <= CPUHP_ONLINE; i++) {
Thomas Gleixner98f8cdc2016-02-26 18:43:31 +00001497 struct cpuhp_step *sp = cpuhp_get_step(i);
1498
1499 if (sp->name) {
1500 cur = sprintf(buf, "%3d: %s\n", i, sp->name);
1501 buf += cur;
1502 res += cur;
1503 }
1504 }
1505 mutex_unlock(&cpuhp_state_mutex);
1506 return res;
1507}
1508static DEVICE_ATTR(states, 0444, show_cpuhp_states, NULL);
1509
1510static struct attribute *cpuhp_cpu_root_attrs[] = {
1511 &dev_attr_states.attr,
1512 NULL
1513};
1514
1515static struct attribute_group cpuhp_cpu_root_attr_group = {
1516 .attrs = cpuhp_cpu_root_attrs,
1517 .name = "hotplug",
1518 NULL
1519};
1520
1521static int __init cpuhp_sysfs_init(void)
1522{
1523 int cpu, ret;
1524
1525 ret = sysfs_create_group(&cpu_subsys.dev_root->kobj,
1526 &cpuhp_cpu_root_attr_group);
1527 if (ret)
1528 return ret;
1529
1530 for_each_possible_cpu(cpu) {
1531 struct device *dev = get_cpu_device(cpu);
1532
1533 if (!dev)
1534 continue;
1535 ret = sysfs_create_group(&dev->kobj, &cpuhp_cpu_attr_group);
1536 if (ret)
1537 return ret;
1538 }
1539 return 0;
1540}
1541device_initcall(cpuhp_sysfs_init);
1542#endif
1543
Linus Torvaldse56b3bc2008-07-28 11:32:33 -07001544/*
1545 * cpu_bit_bitmap[] is a special, "compressed" data structure that
1546 * represents all NR_CPUS bits binary values of 1<<nr.
1547 *
Rusty Russelle0b582e2009-01-01 10:12:28 +10301548 * It is used by cpumask_of() to get a constant address to a CPU
Linus Torvaldse56b3bc2008-07-28 11:32:33 -07001549 * mask value that has a single bit set only.
1550 */
Mike Travisb8d317d2008-07-24 18:21:29 -07001551
Linus Torvaldse56b3bc2008-07-28 11:32:33 -07001552/* cpu_bit_bitmap[0] is empty - so we can back into it */
Michael Rodriguez4d519852011-03-22 16:34:07 -07001553#define MASK_DECLARE_1(x) [x+1][0] = (1UL << (x))
Linus Torvaldse56b3bc2008-07-28 11:32:33 -07001554#define MASK_DECLARE_2(x) MASK_DECLARE_1(x), MASK_DECLARE_1(x+1)
1555#define MASK_DECLARE_4(x) MASK_DECLARE_2(x), MASK_DECLARE_2(x+2)
1556#define MASK_DECLARE_8(x) MASK_DECLARE_4(x), MASK_DECLARE_4(x+4)
Mike Travisb8d317d2008-07-24 18:21:29 -07001557
Linus Torvaldse56b3bc2008-07-28 11:32:33 -07001558const unsigned long cpu_bit_bitmap[BITS_PER_LONG+1][BITS_TO_LONGS(NR_CPUS)] = {
Mike Travisb8d317d2008-07-24 18:21:29 -07001559
Linus Torvaldse56b3bc2008-07-28 11:32:33 -07001560 MASK_DECLARE_8(0), MASK_DECLARE_8(8),
1561 MASK_DECLARE_8(16), MASK_DECLARE_8(24),
1562#if BITS_PER_LONG > 32
1563 MASK_DECLARE_8(32), MASK_DECLARE_8(40),
1564 MASK_DECLARE_8(48), MASK_DECLARE_8(56),
Mike Travisb8d317d2008-07-24 18:21:29 -07001565#endif
1566};
Linus Torvaldse56b3bc2008-07-28 11:32:33 -07001567EXPORT_SYMBOL_GPL(cpu_bit_bitmap);
Rusty Russell2d3854a2008-11-05 13:39:10 +11001568
1569const DECLARE_BITMAP(cpu_all_bits, NR_CPUS) = CPU_BITS_ALL;
1570EXPORT_SYMBOL(cpu_all_bits);
Rusty Russellb3199c02008-12-30 09:05:14 +10301571
1572#ifdef CONFIG_INIT_ALL_POSSIBLE
Rasmus Villemoes4b804c82016-01-20 15:00:19 -08001573struct cpumask __cpu_possible_mask __read_mostly
Rasmus Villemoesc4c54dd2016-01-20 15:00:16 -08001574 = {CPU_BITS_ALL};
Rusty Russellb3199c02008-12-30 09:05:14 +10301575#else
Rasmus Villemoes4b804c82016-01-20 15:00:19 -08001576struct cpumask __cpu_possible_mask __read_mostly;
Rusty Russellb3199c02008-12-30 09:05:14 +10301577#endif
Rasmus Villemoes4b804c82016-01-20 15:00:19 -08001578EXPORT_SYMBOL(__cpu_possible_mask);
Rusty Russellb3199c02008-12-30 09:05:14 +10301579
Rasmus Villemoes4b804c82016-01-20 15:00:19 -08001580struct cpumask __cpu_online_mask __read_mostly;
1581EXPORT_SYMBOL(__cpu_online_mask);
Rusty Russellb3199c02008-12-30 09:05:14 +10301582
Rasmus Villemoes4b804c82016-01-20 15:00:19 -08001583struct cpumask __cpu_present_mask __read_mostly;
1584EXPORT_SYMBOL(__cpu_present_mask);
Rusty Russellb3199c02008-12-30 09:05:14 +10301585
Rasmus Villemoes4b804c82016-01-20 15:00:19 -08001586struct cpumask __cpu_active_mask __read_mostly;
1587EXPORT_SYMBOL(__cpu_active_mask);
Rusty Russell3fa41522008-12-30 09:05:16 +10301588
Rusty Russell3fa41522008-12-30 09:05:16 +10301589void init_cpu_present(const struct cpumask *src)
1590{
Rasmus Villemoesc4c54dd2016-01-20 15:00:16 -08001591 cpumask_copy(&__cpu_present_mask, src);
Rusty Russell3fa41522008-12-30 09:05:16 +10301592}
1593
1594void init_cpu_possible(const struct cpumask *src)
1595{
Rasmus Villemoesc4c54dd2016-01-20 15:00:16 -08001596 cpumask_copy(&__cpu_possible_mask, src);
Rusty Russell3fa41522008-12-30 09:05:16 +10301597}
1598
1599void init_cpu_online(const struct cpumask *src)
1600{
Rasmus Villemoesc4c54dd2016-01-20 15:00:16 -08001601 cpumask_copy(&__cpu_online_mask, src);
Rusty Russell3fa41522008-12-30 09:05:16 +10301602}
Thomas Gleixnercff7d372016-02-26 18:43:28 +00001603
1604/*
1605 * Activate the first processor.
1606 */
1607void __init boot_cpu_init(void)
1608{
1609 int cpu = smp_processor_id();
1610
1611 /* Mark the boot cpu "present", "online" etc for SMP and UP case */
1612 set_cpu_online(cpu, true);
1613 set_cpu_active(cpu, true);
1614 set_cpu_present(cpu, true);
1615 set_cpu_possible(cpu, true);
1616}
1617
1618/*
1619 * Must be called _AFTER_ setting up the per_cpu areas
1620 */
1621void __init boot_cpu_state_init(void)
1622{
1623 per_cpu_ptr(&cpuhp_state, smp_processor_id())->state = CPUHP_ONLINE;
1624}