blob: 042fd7e8e03098882b165787b163a597c4c92aa8 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/* CPU control.
2 * (C) 2001, 2002, 2003, 2004 Rusty Russell
3 *
4 * This code is licenced under the GPL.
5 */
6#include <linux/proc_fs.h>
7#include <linux/smp.h>
8#include <linux/init.h>
9#include <linux/notifier.h>
10#include <linux/sched.h>
11#include <linux/unistd.h>
12#include <linux/cpu.h>
Anton Vorontsovcb792952012-05-31 16:26:22 -070013#include <linux/oom.h>
14#include <linux/rcupdate.h>
Paul Gortmaker9984de12011-05-23 14:51:41 -040015#include <linux/export.h>
Anton Vorontsove4cc2f82012-05-31 16:26:26 -070016#include <linux/bug.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070017#include <linux/kthread.h>
18#include <linux/stop_machine.h>
Ingo Molnar81615b622006-06-26 00:24:32 -070019#include <linux/mutex.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090020#include <linux/gfp.h>
Srivatsa S. Bhat79cfbdf2011-11-03 00:59:25 +010021#include <linux/suspend.h>
Gautham R. Shenoya19423b2014-03-11 02:04:03 +053022#include <linux/lockdep.h>
Preeti U Murthy345527b2015-03-30 14:59:19 +053023#include <linux/tick.h>
Thomas Gleixnera8994182015-07-05 17:12:30 +000024#include <linux/irq.h>
Thomas Gleixner4cb28ce2016-02-26 18:43:38 +000025#include <linux/smpboot.h>
Richard Weinbergere6d49892016-08-18 14:57:17 +020026#include <linux/relay.h>
Sebastian Andrzej Siewior6731d4f2016-08-23 14:53:19 +020027#include <linux/slab.h>
Thomas Gleixnercff7d372016-02-26 18:43:28 +000028
Todd E Brandtbb3632c2014-06-06 05:40:17 -070029#include <trace/events/power.h>
Thomas Gleixnercff7d372016-02-26 18:43:28 +000030#define CREATE_TRACE_POINTS
31#include <trace/events/cpuhp.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070032
Thomas Gleixner38498a62012-04-20 13:05:44 +000033#include "smpboot.h"
34
Thomas Gleixnercff7d372016-02-26 18:43:28 +000035/**
36 * cpuhp_cpu_state - Per cpu hotplug state storage
37 * @state: The current cpu state
38 * @target: The target state
Thomas Gleixner4cb28ce2016-02-26 18:43:38 +000039 * @thread: Pointer to the hotplug thread
40 * @should_run: Thread should execute
Sebastian Andrzej Siewior3b9d6da2016-04-08 14:40:15 +020041 * @rollback: Perform a rollback
Thomas Gleixnera7246322016-08-12 19:49:38 +020042 * @single: Single callback invocation
43 * @bringup: Single callback bringup or teardown selector
44 * @cb_state: The state for a single callback (install/uninstall)
Thomas Gleixner4cb28ce2016-02-26 18:43:38 +000045 * @result: Result of the operation
46 * @done: Signal completion to the issuer of the task
Thomas Gleixnercff7d372016-02-26 18:43:28 +000047 */
48struct cpuhp_cpu_state {
49 enum cpuhp_state state;
50 enum cpuhp_state target;
Thomas Gleixner4cb28ce2016-02-26 18:43:38 +000051#ifdef CONFIG_SMP
52 struct task_struct *thread;
53 bool should_run;
Sebastian Andrzej Siewior3b9d6da2016-04-08 14:40:15 +020054 bool rollback;
Thomas Gleixnera7246322016-08-12 19:49:38 +020055 bool single;
56 bool bringup;
Thomas Gleixnercf392d12016-08-12 19:49:39 +020057 struct hlist_node *node;
Thomas Gleixner4cb28ce2016-02-26 18:43:38 +000058 enum cpuhp_state cb_state;
Thomas Gleixner4cb28ce2016-02-26 18:43:38 +000059 int result;
60 struct completion done;
61#endif
Thomas Gleixnercff7d372016-02-26 18:43:28 +000062};
63
64static DEFINE_PER_CPU(struct cpuhp_cpu_state, cpuhp_state);
65
66/**
67 * cpuhp_step - Hotplug state machine step
68 * @name: Name of the step
69 * @startup: Startup function of the step
70 * @teardown: Teardown function of the step
71 * @skip_onerr: Do not invoke the functions on error rollback
72 * Will go away once the notifiers are gone
Thomas Gleixner757c9892016-02-26 18:43:32 +000073 * @cant_stop: Bringup/teardown can't be stopped at this step
Thomas Gleixnercff7d372016-02-26 18:43:28 +000074 */
75struct cpuhp_step {
Thomas Gleixnercf392d12016-08-12 19:49:39 +020076 const char *name;
77 union {
Thomas Gleixner3c1627e2016-09-05 15:28:36 +020078 int (*single)(unsigned int cpu);
79 int (*multi)(unsigned int cpu,
80 struct hlist_node *node);
81 } startup;
Thomas Gleixnercf392d12016-08-12 19:49:39 +020082 union {
Thomas Gleixner3c1627e2016-09-05 15:28:36 +020083 int (*single)(unsigned int cpu);
84 int (*multi)(unsigned int cpu,
85 struct hlist_node *node);
86 } teardown;
Thomas Gleixnercf392d12016-08-12 19:49:39 +020087 struct hlist_head list;
88 bool skip_onerr;
89 bool cant_stop;
90 bool multi_instance;
Thomas Gleixnercff7d372016-02-26 18:43:28 +000091};
92
Thomas Gleixner98f8cdc2016-02-26 18:43:31 +000093static DEFINE_MUTEX(cpuhp_state_mutex);
Thomas Gleixnercff7d372016-02-26 18:43:28 +000094static struct cpuhp_step cpuhp_bp_states[];
Thomas Gleixner4baa0af2016-02-26 18:43:29 +000095static struct cpuhp_step cpuhp_ap_states[];
Thomas Gleixnercff7d372016-02-26 18:43:28 +000096
Thomas Gleixnera7246322016-08-12 19:49:38 +020097static bool cpuhp_is_ap_state(enum cpuhp_state state)
98{
99 /*
100 * The extra check for CPUHP_TEARDOWN_CPU is only for documentation
101 * purposes as that state is handled explicitly in cpu_down.
102 */
103 return state > CPUHP_BRINGUP_CPU && state != CPUHP_TEARDOWN_CPU;
104}
105
106static struct cpuhp_step *cpuhp_get_step(enum cpuhp_state state)
107{
108 struct cpuhp_step *sp;
109
110 sp = cpuhp_is_ap_state(state) ? cpuhp_ap_states : cpuhp_bp_states;
111 return sp + state;
112}
113
Thomas Gleixnercff7d372016-02-26 18:43:28 +0000114/**
115 * cpuhp_invoke_callback _ Invoke the callbacks for a given state
116 * @cpu: The cpu for which the callback should be invoked
117 * @step: The step in the state machine
Thomas Gleixnera7246322016-08-12 19:49:38 +0200118 * @bringup: True if the bringup callback should be invoked
Thomas Gleixnercff7d372016-02-26 18:43:28 +0000119 *
Thomas Gleixnercf392d12016-08-12 19:49:39 +0200120 * Called from cpu hotplug and from the state register machinery.
Thomas Gleixnercff7d372016-02-26 18:43:28 +0000121 */
Thomas Gleixnera7246322016-08-12 19:49:38 +0200122static int cpuhp_invoke_callback(unsigned int cpu, enum cpuhp_state state,
Thomas Gleixnercf392d12016-08-12 19:49:39 +0200123 bool bringup, struct hlist_node *node)
Thomas Gleixnercff7d372016-02-26 18:43:28 +0000124{
125 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
Thomas Gleixnera7246322016-08-12 19:49:38 +0200126 struct cpuhp_step *step = cpuhp_get_step(state);
Thomas Gleixnercf392d12016-08-12 19:49:39 +0200127 int (*cbm)(unsigned int cpu, struct hlist_node *node);
128 int (*cb)(unsigned int cpu);
129 int ret, cnt;
Thomas Gleixnercff7d372016-02-26 18:43:28 +0000130
Thomas Gleixnercf392d12016-08-12 19:49:39 +0200131 if (!step->multi_instance) {
Thomas Gleixner3c1627e2016-09-05 15:28:36 +0200132 cb = bringup ? step->startup.single : step->teardown.single;
Thomas Gleixnercf392d12016-08-12 19:49:39 +0200133 if (!cb)
134 return 0;
Thomas Gleixnera7246322016-08-12 19:49:38 +0200135 trace_cpuhp_enter(cpu, st->target, state, cb);
Thomas Gleixnercff7d372016-02-26 18:43:28 +0000136 ret = cb(cpu);
Thomas Gleixnera7246322016-08-12 19:49:38 +0200137 trace_cpuhp_exit(cpu, st->state, state, ret);
Thomas Gleixnercf392d12016-08-12 19:49:39 +0200138 return ret;
139 }
Thomas Gleixner3c1627e2016-09-05 15:28:36 +0200140 cbm = bringup ? step->startup.multi : step->teardown.multi;
Thomas Gleixnercf392d12016-08-12 19:49:39 +0200141 if (!cbm)
142 return 0;
143
144 /* Single invocation for instance add/remove */
145 if (node) {
146 trace_cpuhp_multi_enter(cpu, st->target, state, cbm, node);
147 ret = cbm(cpu, node);
148 trace_cpuhp_exit(cpu, st->state, state, ret);
149 return ret;
150 }
151
152 /* State transition. Invoke on all instances */
153 cnt = 0;
154 hlist_for_each(node, &step->list) {
155 trace_cpuhp_multi_enter(cpu, st->target, state, cbm, node);
156 ret = cbm(cpu, node);
157 trace_cpuhp_exit(cpu, st->state, state, ret);
158 if (ret)
159 goto err;
160 cnt++;
161 }
162 return 0;
163err:
164 /* Rollback the instances if one failed */
Thomas Gleixner3c1627e2016-09-05 15:28:36 +0200165 cbm = !bringup ? step->startup.multi : step->teardown.multi;
Thomas Gleixnercf392d12016-08-12 19:49:39 +0200166 if (!cbm)
167 return ret;
168
169 hlist_for_each(node, &step->list) {
170 if (!cnt--)
171 break;
172 cbm(cpu, node);
Thomas Gleixnercff7d372016-02-26 18:43:28 +0000173 }
174 return ret;
175}
176
Rusty Russell98a79d62008-12-13 21:19:41 +1030177#ifdef CONFIG_SMP
Rusty Russellb3199c02008-12-30 09:05:14 +1030178/* Serializes the updates to cpu_online_mask, cpu_present_mask */
Linus Torvaldsaa953872006-07-23 12:12:16 -0700179static DEFINE_MUTEX(cpu_add_remove_lock);
Thomas Gleixner090e77c2016-02-26 18:43:23 +0000180bool cpuhp_tasks_frozen;
181EXPORT_SYMBOL_GPL(cpuhp_tasks_frozen);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700182
Lai Jiangshan79a6cde2010-05-26 14:43:36 -0700183/*
Srivatsa S. Bhat93ae4f92014-03-11 02:04:14 +0530184 * The following two APIs (cpu_maps_update_begin/done) must be used when
185 * attempting to serialize the updates to cpu_online_mask & cpu_present_mask.
Lai Jiangshan79a6cde2010-05-26 14:43:36 -0700186 */
187void cpu_maps_update_begin(void)
188{
189 mutex_lock(&cpu_add_remove_lock);
190}
191
192void cpu_maps_update_done(void)
193{
194 mutex_unlock(&cpu_add_remove_lock);
195}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700196
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -0700197/* If set, cpu_up and cpu_down will return -EBUSY and do nothing.
198 * Should always be manipulated under cpu_add_remove_lock
199 */
200static int cpu_hotplug_disabled;
201
Lai Jiangshan79a6cde2010-05-26 14:43:36 -0700202#ifdef CONFIG_HOTPLUG_CPU
203
Gautham R Shenoyd2219382008-01-25 21:08:01 +0100204static struct {
205 struct task_struct *active_writer;
David Hildenbrand87af9e72014-12-12 10:11:44 +0100206 /* wait queue to wake up the active_writer */
207 wait_queue_head_t wq;
208 /* verifies that no writer will get active while readers are active */
209 struct mutex lock;
Gautham R Shenoyd2219382008-01-25 21:08:01 +0100210 /*
211 * Also blocks the new readers during
212 * an ongoing cpu hotplug operation.
213 */
David Hildenbrand87af9e72014-12-12 10:11:44 +0100214 atomic_t refcount;
Gautham R. Shenoya19423b2014-03-11 02:04:03 +0530215
216#ifdef CONFIG_DEBUG_LOCK_ALLOC
217 struct lockdep_map dep_map;
218#endif
Linus Torvalds31950eb2009-06-22 21:18:12 -0700219} cpu_hotplug = {
220 .active_writer = NULL,
David Hildenbrand87af9e72014-12-12 10:11:44 +0100221 .wq = __WAIT_QUEUE_HEAD_INITIALIZER(cpu_hotplug.wq),
Linus Torvalds31950eb2009-06-22 21:18:12 -0700222 .lock = __MUTEX_INITIALIZER(cpu_hotplug.lock),
Gautham R. Shenoya19423b2014-03-11 02:04:03 +0530223#ifdef CONFIG_DEBUG_LOCK_ALLOC
Joonas Lahtinena705e072016-10-12 13:18:56 +0300224 .dep_map = STATIC_LOCKDEP_MAP_INIT("cpu_hotplug.dep_map", &cpu_hotplug.dep_map),
Gautham R. Shenoya19423b2014-03-11 02:04:03 +0530225#endif
Linus Torvalds31950eb2009-06-22 21:18:12 -0700226};
Gautham R Shenoyd2219382008-01-25 21:08:01 +0100227
Gautham R. Shenoya19423b2014-03-11 02:04:03 +0530228/* Lockdep annotations for get/put_online_cpus() and cpu_hotplug_begin/end() */
229#define cpuhp_lock_acquire_read() lock_map_acquire_read(&cpu_hotplug.dep_map)
Paul E. McKenneydd56af42014-08-25 20:25:06 -0700230#define cpuhp_lock_acquire_tryread() \
231 lock_map_acquire_tryread(&cpu_hotplug.dep_map)
Gautham R. Shenoya19423b2014-03-11 02:04:03 +0530232#define cpuhp_lock_acquire() lock_map_acquire(&cpu_hotplug.dep_map)
233#define cpuhp_lock_release() lock_map_release(&cpu_hotplug.dep_map)
234
Paul E. McKenney62db99f2014-10-22 14:51:49 -0700235
Gautham R Shenoy86ef5c92008-01-25 21:08:02 +0100236void get_online_cpus(void)
Ashok Raja9d9baa2005-11-28 13:43:46 -0800237{
Gautham R Shenoyd2219382008-01-25 21:08:01 +0100238 might_sleep();
239 if (cpu_hotplug.active_writer == current)
Linus Torvaldsaa953872006-07-23 12:12:16 -0700240 return;
Gautham R. Shenoya19423b2014-03-11 02:04:03 +0530241 cpuhp_lock_acquire_read();
Gautham R Shenoyd2219382008-01-25 21:08:01 +0100242 mutex_lock(&cpu_hotplug.lock);
David Hildenbrand87af9e72014-12-12 10:11:44 +0100243 atomic_inc(&cpu_hotplug.refcount);
Gautham R Shenoyd2219382008-01-25 21:08:01 +0100244 mutex_unlock(&cpu_hotplug.lock);
Ashok Raja9d9baa2005-11-28 13:43:46 -0800245}
Gautham R Shenoy86ef5c92008-01-25 21:08:02 +0100246EXPORT_SYMBOL_GPL(get_online_cpus);
Ashok Raj90d45d12005-11-08 21:34:24 -0800247
Gautham R Shenoy86ef5c92008-01-25 21:08:02 +0100248void put_online_cpus(void)
Ashok Raja9d9baa2005-11-28 13:43:46 -0800249{
David Hildenbrand87af9e72014-12-12 10:11:44 +0100250 int refcount;
251
Gautham R Shenoyd2219382008-01-25 21:08:01 +0100252 if (cpu_hotplug.active_writer == current)
Linus Torvaldsaa953872006-07-23 12:12:16 -0700253 return;
Srivatsa S. Bhat075663d2012-10-08 16:28:20 -0700254
David Hildenbrand87af9e72014-12-12 10:11:44 +0100255 refcount = atomic_dec_return(&cpu_hotplug.refcount);
256 if (WARN_ON(refcount < 0)) /* try to fix things up */
257 atomic_inc(&cpu_hotplug.refcount);
Srivatsa S. Bhat075663d2012-10-08 16:28:20 -0700258
David Hildenbrand87af9e72014-12-12 10:11:44 +0100259 if (refcount <= 0 && waitqueue_active(&cpu_hotplug.wq))
260 wake_up(&cpu_hotplug.wq);
261
Gautham R. Shenoya19423b2014-03-11 02:04:03 +0530262 cpuhp_lock_release();
Gautham R Shenoyd2219382008-01-25 21:08:01 +0100263
Ashok Raja9d9baa2005-11-28 13:43:46 -0800264}
Gautham R Shenoy86ef5c92008-01-25 21:08:02 +0100265EXPORT_SYMBOL_GPL(put_online_cpus);
Ashok Raja9d9baa2005-11-28 13:43:46 -0800266
Gautham R Shenoyd2219382008-01-25 21:08:01 +0100267/*
268 * This ensures that the hotplug operation can begin only when the
269 * refcount goes to zero.
270 *
271 * Note that during a cpu-hotplug operation, the new readers, if any,
272 * will be blocked by the cpu_hotplug.lock
273 *
Oleg Nesterovd2ba7e22008-04-29 01:00:29 -0700274 * Since cpu_hotplug_begin() is always called after invoking
275 * cpu_maps_update_begin(), we can be sure that only one writer is active.
Gautham R Shenoyd2219382008-01-25 21:08:01 +0100276 *
277 * Note that theoretically, there is a possibility of a livelock:
278 * - Refcount goes to zero, last reader wakes up the sleeping
279 * writer.
280 * - Last reader unlocks the cpu_hotplug.lock.
281 * - A new reader arrives at this moment, bumps up the refcount.
282 * - The writer acquires the cpu_hotplug.lock finds the refcount
283 * non zero and goes to sleep again.
284 *
285 * However, this is very difficult to achieve in practice since
Gautham R Shenoy86ef5c92008-01-25 21:08:02 +0100286 * get_online_cpus() not an api which is called all that often.
Gautham R Shenoyd2219382008-01-25 21:08:01 +0100287 *
288 */
Toshi Kanib9d10be2013-08-12 09:45:53 -0600289void cpu_hotplug_begin(void)
Gautham R Shenoyd2219382008-01-25 21:08:01 +0100290{
David Hildenbrand87af9e72014-12-12 10:11:44 +0100291 DEFINE_WAIT(wait);
Oleg Nesterovd2ba7e22008-04-29 01:00:29 -0700292
David Hildenbrand87af9e72014-12-12 10:11:44 +0100293 cpu_hotplug.active_writer = current;
Gautham R. Shenoya19423b2014-03-11 02:04:03 +0530294 cpuhp_lock_acquire();
David Hildenbrand87af9e72014-12-12 10:11:44 +0100295
Oleg Nesterovd2ba7e22008-04-29 01:00:29 -0700296 for (;;) {
297 mutex_lock(&cpu_hotplug.lock);
David Hildenbrand87af9e72014-12-12 10:11:44 +0100298 prepare_to_wait(&cpu_hotplug.wq, &wait, TASK_UNINTERRUPTIBLE);
299 if (likely(!atomic_read(&cpu_hotplug.refcount)))
300 break;
Gautham R Shenoyd2219382008-01-25 21:08:01 +0100301 mutex_unlock(&cpu_hotplug.lock);
302 schedule();
Gautham R Shenoyd2219382008-01-25 21:08:01 +0100303 }
David Hildenbrand87af9e72014-12-12 10:11:44 +0100304 finish_wait(&cpu_hotplug.wq, &wait);
Gautham R Shenoyd2219382008-01-25 21:08:01 +0100305}
306
Toshi Kanib9d10be2013-08-12 09:45:53 -0600307void cpu_hotplug_done(void)
Gautham R Shenoyd2219382008-01-25 21:08:01 +0100308{
309 cpu_hotplug.active_writer = NULL;
310 mutex_unlock(&cpu_hotplug.lock);
Gautham R. Shenoya19423b2014-03-11 02:04:03 +0530311 cpuhp_lock_release();
Gautham R Shenoyd2219382008-01-25 21:08:01 +0100312}
Lai Jiangshan79a6cde2010-05-26 14:43:36 -0700313
Srivatsa S. Bhat16e53db2013-06-12 14:04:36 -0700314/*
315 * Wait for currently running CPU hotplug operations to complete (if any) and
316 * disable future CPU hotplug (from sysfs). The 'cpu_add_remove_lock' protects
317 * the 'cpu_hotplug_disabled' flag. The same lock is also acquired by the
318 * hotplug path before performing hotplug operations. So acquiring that lock
319 * guarantees mutual exclusion from any currently running hotplug operations.
320 */
321void cpu_hotplug_disable(void)
322{
323 cpu_maps_update_begin();
Vitaly Kuznetsov89af7ba2015-08-05 00:52:46 -0700324 cpu_hotplug_disabled++;
Srivatsa S. Bhat16e53db2013-06-12 14:04:36 -0700325 cpu_maps_update_done();
326}
Vitaly Kuznetsov32145c42015-08-05 00:52:47 -0700327EXPORT_SYMBOL_GPL(cpu_hotplug_disable);
Srivatsa S. Bhat16e53db2013-06-12 14:04:36 -0700328
Lianwei Wang01b41152016-06-09 23:43:28 -0700329static void __cpu_hotplug_enable(void)
330{
331 if (WARN_ONCE(!cpu_hotplug_disabled, "Unbalanced cpu hotplug enable\n"))
332 return;
333 cpu_hotplug_disabled--;
334}
335
Srivatsa S. Bhat16e53db2013-06-12 14:04:36 -0700336void cpu_hotplug_enable(void)
337{
338 cpu_maps_update_begin();
Lianwei Wang01b41152016-06-09 23:43:28 -0700339 __cpu_hotplug_enable();
Srivatsa S. Bhat16e53db2013-06-12 14:04:36 -0700340 cpu_maps_update_done();
341}
Vitaly Kuznetsov32145c42015-08-05 00:52:47 -0700342EXPORT_SYMBOL_GPL(cpu_hotplug_enable);
Toshi Kanib9d10be2013-08-12 09:45:53 -0600343#endif /* CONFIG_HOTPLUG_CPU */
Lai Jiangshan79a6cde2010-05-26 14:43:36 -0700344
Thomas Gleixnerba997462016-02-26 18:43:24 +0000345/* Notifier wrappers for transitioning to state machine */
Thomas Gleixnerba997462016-02-26 18:43:24 +0000346
Thomas Gleixner8df3e072016-02-26 18:43:41 +0000347static int bringup_wait_for_ap(unsigned int cpu)
348{
349 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
350
351 wait_for_completion(&st->done);
352 return st->result;
353}
354
Thomas Gleixnerba997462016-02-26 18:43:24 +0000355static int bringup_cpu(unsigned int cpu)
356{
357 struct task_struct *idle = idle_thread_get(cpu);
358 int ret;
359
Boris Ostrovskyaa877172016-08-03 13:22:28 -0400360 /*
361 * Some architectures have to walk the irq descriptors to
362 * setup the vector space for the cpu which comes online.
363 * Prevent irq alloc/free across the bringup.
364 */
365 irq_lock_sparse();
366
Thomas Gleixnerba997462016-02-26 18:43:24 +0000367 /* Arch-specific enabling code. */
368 ret = __cpu_up(cpu, idle);
Boris Ostrovskyaa877172016-08-03 13:22:28 -0400369 irq_unlock_sparse();
Thomas Gleixner530e9b72016-12-21 20:19:53 +0100370 if (ret)
Thomas Gleixnerba997462016-02-26 18:43:24 +0000371 return ret;
Thomas Gleixner8df3e072016-02-26 18:43:41 +0000372 ret = bringup_wait_for_ap(cpu);
Thomas Gleixnerba997462016-02-26 18:43:24 +0000373 BUG_ON(!cpu_online(cpu));
Thomas Gleixner8df3e072016-02-26 18:43:41 +0000374 return ret;
Thomas Gleixnerba997462016-02-26 18:43:24 +0000375}
376
Thomas Gleixner2e1a3482016-02-26 18:43:37 +0000377/*
378 * Hotplug state machine related functions
379 */
Thomas Gleixnera7246322016-08-12 19:49:38 +0200380static void undo_cpu_down(unsigned int cpu, struct cpuhp_cpu_state *st)
Thomas Gleixner2e1a3482016-02-26 18:43:37 +0000381{
382 for (st->state++; st->state < st->target; st->state++) {
Thomas Gleixnera7246322016-08-12 19:49:38 +0200383 struct cpuhp_step *step = cpuhp_get_step(st->state);
Thomas Gleixner2e1a3482016-02-26 18:43:37 +0000384
385 if (!step->skip_onerr)
Thomas Gleixnercf392d12016-08-12 19:49:39 +0200386 cpuhp_invoke_callback(cpu, st->state, true, NULL);
Thomas Gleixner2e1a3482016-02-26 18:43:37 +0000387 }
388}
389
390static int cpuhp_down_callbacks(unsigned int cpu, struct cpuhp_cpu_state *st,
Thomas Gleixnera7246322016-08-12 19:49:38 +0200391 enum cpuhp_state target)
Thomas Gleixner2e1a3482016-02-26 18:43:37 +0000392{
393 enum cpuhp_state prev_state = st->state;
394 int ret = 0;
395
396 for (; st->state > target; st->state--) {
Thomas Gleixnercf392d12016-08-12 19:49:39 +0200397 ret = cpuhp_invoke_callback(cpu, st->state, false, NULL);
Thomas Gleixner2e1a3482016-02-26 18:43:37 +0000398 if (ret) {
399 st->target = prev_state;
Thomas Gleixnera7246322016-08-12 19:49:38 +0200400 undo_cpu_down(cpu, st);
Thomas Gleixner2e1a3482016-02-26 18:43:37 +0000401 break;
402 }
403 }
404 return ret;
405}
406
Thomas Gleixnera7246322016-08-12 19:49:38 +0200407static void undo_cpu_up(unsigned int cpu, struct cpuhp_cpu_state *st)
Thomas Gleixner2e1a3482016-02-26 18:43:37 +0000408{
409 for (st->state--; st->state > st->target; st->state--) {
Thomas Gleixnera7246322016-08-12 19:49:38 +0200410 struct cpuhp_step *step = cpuhp_get_step(st->state);
Thomas Gleixner2e1a3482016-02-26 18:43:37 +0000411
412 if (!step->skip_onerr)
Thomas Gleixnercf392d12016-08-12 19:49:39 +0200413 cpuhp_invoke_callback(cpu, st->state, false, NULL);
Thomas Gleixner2e1a3482016-02-26 18:43:37 +0000414 }
415}
416
417static int cpuhp_up_callbacks(unsigned int cpu, struct cpuhp_cpu_state *st,
Thomas Gleixnera7246322016-08-12 19:49:38 +0200418 enum cpuhp_state target)
Thomas Gleixner2e1a3482016-02-26 18:43:37 +0000419{
420 enum cpuhp_state prev_state = st->state;
421 int ret = 0;
422
423 while (st->state < target) {
Thomas Gleixner2e1a3482016-02-26 18:43:37 +0000424 st->state++;
Thomas Gleixnercf392d12016-08-12 19:49:39 +0200425 ret = cpuhp_invoke_callback(cpu, st->state, true, NULL);
Thomas Gleixner2e1a3482016-02-26 18:43:37 +0000426 if (ret) {
427 st->target = prev_state;
Thomas Gleixnera7246322016-08-12 19:49:38 +0200428 undo_cpu_up(cpu, st);
Thomas Gleixner2e1a3482016-02-26 18:43:37 +0000429 break;
430 }
431 }
432 return ret;
433}
434
Thomas Gleixner4cb28ce2016-02-26 18:43:38 +0000435/*
436 * The cpu hotplug threads manage the bringup and teardown of the cpus
437 */
438static void cpuhp_create(unsigned int cpu)
439{
440 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
441
442 init_completion(&st->done);
443}
444
445static int cpuhp_should_run(unsigned int cpu)
446{
447 struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);
448
449 return st->should_run;
450}
451
452/* Execute the teardown callbacks. Used to be CPU_DOWN_PREPARE */
453static int cpuhp_ap_offline(unsigned int cpu, struct cpuhp_cpu_state *st)
454{
Thomas Gleixner1cf4f622016-02-26 18:43:39 +0000455 enum cpuhp_state target = max((int)st->target, CPUHP_TEARDOWN_CPU);
Thomas Gleixner4cb28ce2016-02-26 18:43:38 +0000456
Thomas Gleixnera7246322016-08-12 19:49:38 +0200457 return cpuhp_down_callbacks(cpu, st, target);
Thomas Gleixner4cb28ce2016-02-26 18:43:38 +0000458}
459
460/* Execute the online startup callbacks. Used to be CPU_ONLINE */
461static int cpuhp_ap_online(unsigned int cpu, struct cpuhp_cpu_state *st)
462{
Thomas Gleixnera7246322016-08-12 19:49:38 +0200463 return cpuhp_up_callbacks(cpu, st, st->target);
Thomas Gleixner4cb28ce2016-02-26 18:43:38 +0000464}
465
466/*
467 * Execute teardown/startup callbacks on the plugged cpu. Also used to invoke
468 * callbacks when a state gets [un]installed at runtime.
469 */
470static void cpuhp_thread_fun(unsigned int cpu)
471{
472 struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);
473 int ret = 0;
474
475 /*
476 * Paired with the mb() in cpuhp_kick_ap_work and
477 * cpuhp_invoke_ap_callback, so the work set is consistent visible.
478 */
479 smp_mb();
480 if (!st->should_run)
481 return;
482
483 st->should_run = false;
484
485 /* Single callback invocation for [un]install ? */
Thomas Gleixnera7246322016-08-12 19:49:38 +0200486 if (st->single) {
Thomas Gleixner4cb28ce2016-02-26 18:43:38 +0000487 if (st->cb_state < CPUHP_AP_ONLINE) {
488 local_irq_disable();
Thomas Gleixnera7246322016-08-12 19:49:38 +0200489 ret = cpuhp_invoke_callback(cpu, st->cb_state,
Thomas Gleixnercf392d12016-08-12 19:49:39 +0200490 st->bringup, st->node);
Thomas Gleixner4cb28ce2016-02-26 18:43:38 +0000491 local_irq_enable();
492 } else {
Thomas Gleixnera7246322016-08-12 19:49:38 +0200493 ret = cpuhp_invoke_callback(cpu, st->cb_state,
Thomas Gleixnercf392d12016-08-12 19:49:39 +0200494 st->bringup, st->node);
Thomas Gleixner4cb28ce2016-02-26 18:43:38 +0000495 }
Sebastian Andrzej Siewior3b9d6da2016-04-08 14:40:15 +0200496 } else if (st->rollback) {
497 BUG_ON(st->state < CPUHP_AP_ONLINE_IDLE);
498
Thomas Gleixnera7246322016-08-12 19:49:38 +0200499 undo_cpu_down(cpu, st);
Sebastian Andrzej Siewior3b9d6da2016-04-08 14:40:15 +0200500 st->rollback = false;
Thomas Gleixner4cb28ce2016-02-26 18:43:38 +0000501 } else {
Thomas Gleixner1cf4f622016-02-26 18:43:39 +0000502 /* Cannot happen .... */
Thomas Gleixner8df3e072016-02-26 18:43:41 +0000503 BUG_ON(st->state < CPUHP_AP_ONLINE_IDLE);
Thomas Gleixner1cf4f622016-02-26 18:43:39 +0000504
Thomas Gleixner4cb28ce2016-02-26 18:43:38 +0000505 /* Regular hotplug work */
506 if (st->state < st->target)
507 ret = cpuhp_ap_online(cpu, st);
508 else if (st->state > st->target)
509 ret = cpuhp_ap_offline(cpu, st);
510 }
511 st->result = ret;
512 complete(&st->done);
513}
514
515/* Invoke a single callback on a remote cpu */
Thomas Gleixnera7246322016-08-12 19:49:38 +0200516static int
Thomas Gleixnercf392d12016-08-12 19:49:39 +0200517cpuhp_invoke_ap_callback(int cpu, enum cpuhp_state state, bool bringup,
518 struct hlist_node *node)
Thomas Gleixner4cb28ce2016-02-26 18:43:38 +0000519{
520 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
521
522 if (!cpu_online(cpu))
523 return 0;
524
Thomas Gleixner6a4e2452016-07-13 17:16:03 +0000525 /*
526 * If we are up and running, use the hotplug thread. For early calls
527 * we invoke the thread function directly.
528 */
529 if (!st->thread)
Thomas Gleixnercf392d12016-08-12 19:49:39 +0200530 return cpuhp_invoke_callback(cpu, state, bringup, node);
Thomas Gleixner6a4e2452016-07-13 17:16:03 +0000531
Thomas Gleixner4cb28ce2016-02-26 18:43:38 +0000532 st->cb_state = state;
Thomas Gleixnera7246322016-08-12 19:49:38 +0200533 st->single = true;
534 st->bringup = bringup;
Thomas Gleixnercf392d12016-08-12 19:49:39 +0200535 st->node = node;
Thomas Gleixnera7246322016-08-12 19:49:38 +0200536
Thomas Gleixner4cb28ce2016-02-26 18:43:38 +0000537 /*
538 * Make sure the above stores are visible before should_run becomes
539 * true. Paired with the mb() above in cpuhp_thread_fun()
540 */
541 smp_mb();
542 st->should_run = true;
543 wake_up_process(st->thread);
544 wait_for_completion(&st->done);
545 return st->result;
546}
547
548/* Regular hotplug invocation of the AP hotplug thread */
Thomas Gleixner1cf4f622016-02-26 18:43:39 +0000549static void __cpuhp_kick_ap_work(struct cpuhp_cpu_state *st)
Thomas Gleixner4cb28ce2016-02-26 18:43:38 +0000550{
Thomas Gleixner4cb28ce2016-02-26 18:43:38 +0000551 st->result = 0;
Thomas Gleixnera7246322016-08-12 19:49:38 +0200552 st->single = false;
Thomas Gleixner4cb28ce2016-02-26 18:43:38 +0000553 /*
554 * Make sure the above stores are visible before should_run becomes
555 * true. Paired with the mb() above in cpuhp_thread_fun()
556 */
557 smp_mb();
558 st->should_run = true;
559 wake_up_process(st->thread);
Thomas Gleixner1cf4f622016-02-26 18:43:39 +0000560}
561
562static int cpuhp_kick_ap_work(unsigned int cpu)
563{
564 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
565 enum cpuhp_state state = st->state;
566
567 trace_cpuhp_enter(cpu, st->target, state, cpuhp_kick_ap_work);
568 __cpuhp_kick_ap_work(st);
Thomas Gleixner4cb28ce2016-02-26 18:43:38 +0000569 wait_for_completion(&st->done);
570 trace_cpuhp_exit(cpu, st->state, state, st->result);
571 return st->result;
572}
573
574static struct smp_hotplug_thread cpuhp_threads = {
575 .store = &cpuhp_state.thread,
576 .create = &cpuhp_create,
577 .thread_should_run = cpuhp_should_run,
578 .thread_fn = cpuhp_thread_fun,
579 .thread_comm = "cpuhp/%u",
580 .selfparking = true,
581};
582
583void __init cpuhp_threads_init(void)
584{
585 BUG_ON(smpboot_register_percpu_thread(&cpuhp_threads));
586 kthread_unpark(this_cpu_read(cpuhp_state.thread));
587}
588
Michal Hocko777c6e02016-12-07 14:54:38 +0100589#ifdef CONFIG_HOTPLUG_CPU
Anton Vorontsove4cc2f82012-05-31 16:26:26 -0700590/**
591 * clear_tasks_mm_cpumask - Safely clear tasks' mm_cpumask for a CPU
592 * @cpu: a CPU id
593 *
594 * This function walks all processes, finds a valid mm struct for each one and
595 * then clears a corresponding bit in mm's cpumask. While this all sounds
596 * trivial, there are various non-obvious corner cases, which this function
597 * tries to solve in a safe manner.
598 *
599 * Also note that the function uses a somewhat relaxed locking scheme, so it may
600 * be called only for an already offlined CPU.
601 */
Anton Vorontsovcb792952012-05-31 16:26:22 -0700602void clear_tasks_mm_cpumask(int cpu)
603{
604 struct task_struct *p;
605
606 /*
607 * This function is called after the cpu is taken down and marked
608 * offline, so its not like new tasks will ever get this cpu set in
609 * their mm mask. -- Peter Zijlstra
610 * Thus, we may use rcu_read_lock() here, instead of grabbing
611 * full-fledged tasklist_lock.
612 */
Anton Vorontsove4cc2f82012-05-31 16:26:26 -0700613 WARN_ON(cpu_online(cpu));
Anton Vorontsovcb792952012-05-31 16:26:22 -0700614 rcu_read_lock();
615 for_each_process(p) {
616 struct task_struct *t;
617
Anton Vorontsove4cc2f82012-05-31 16:26:26 -0700618 /*
619 * Main thread might exit, but other threads may still have
620 * a valid mm. Find one.
621 */
Anton Vorontsovcb792952012-05-31 16:26:22 -0700622 t = find_lock_task_mm(p);
623 if (!t)
624 continue;
625 cpumask_clear_cpu(cpu, mm_cpumask(t->mm));
626 task_unlock(t);
627 }
628 rcu_read_unlock();
629}
630
Kirill Tkhaib728ca02014-06-25 12:19:55 +0400631static inline void check_for_tasks(int dead_cpu)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700632{
Kirill Tkhaib728ca02014-06-25 12:19:55 +0400633 struct task_struct *g, *p;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700634
Oleg Nesterova75a6062015-09-10 15:07:50 +0200635 read_lock(&tasklist_lock);
636 for_each_process_thread(g, p) {
Kirill Tkhaib728ca02014-06-25 12:19:55 +0400637 if (!p->on_rq)
638 continue;
639 /*
640 * We do the check with unlocked task_rq(p)->lock.
641 * Order the reading to do not warn about a task,
642 * which was running on this cpu in the past, and
643 * it's just been woken on another cpu.
644 */
645 rmb();
646 if (task_cpu(p) != dead_cpu)
647 continue;
648
649 pr_warn("Task %s (pid=%d) is on cpu %d (state=%ld, flags=%x)\n",
650 p->comm, task_pid_nr(p), dead_cpu, p->state, p->flags);
Oleg Nesterova75a6062015-09-10 15:07:50 +0200651 }
652 read_unlock(&tasklist_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700653}
654
655/* Take this CPU down. */
Mathias Krause71cf5ae2015-07-19 20:06:22 +0200656static int take_cpu_down(void *_param)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700657{
Thomas Gleixner4baa0af2016-02-26 18:43:29 +0000658 struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);
659 enum cpuhp_state target = max((int)st->target, CPUHP_AP_OFFLINE);
Thomas Gleixner090e77c2016-02-26 18:43:23 +0000660 int err, cpu = smp_processor_id();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700661
Linus Torvalds1da177e2005-04-16 15:20:36 -0700662 /* Ensure this CPU doesn't handle any more interrupts. */
663 err = __cpu_disable();
664 if (err < 0)
Zwane Mwaikambof3705132005-06-25 14:54:50 -0700665 return err;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700666
Thomas Gleixnera7246322016-08-12 19:49:38 +0200667 /*
668 * We get here while we are in CPUHP_TEARDOWN_CPU state and we must not
669 * do this step again.
670 */
671 WARN_ON(st->state != CPUHP_TEARDOWN_CPU);
672 st->state--;
Thomas Gleixner4baa0af2016-02-26 18:43:29 +0000673 /* Invoke the former CPU_DYING callbacks */
Thomas Gleixnera7246322016-08-12 19:49:38 +0200674 for (; st->state > target; st->state--)
Thomas Gleixnercf392d12016-08-12 19:49:39 +0200675 cpuhp_invoke_callback(cpu, st->state, false, NULL);
Thomas Gleixner4baa0af2016-02-26 18:43:29 +0000676
Thomas Gleixner52c063d2015-04-03 02:37:24 +0200677 /* Give up timekeeping duties */
678 tick_handover_do_timer();
Thomas Gleixner14e568e2013-01-31 12:11:14 +0000679 /* Park the stopper thread */
Thomas Gleixner090e77c2016-02-26 18:43:23 +0000680 stop_machine_park(cpu);
Zwane Mwaikambof3705132005-06-25 14:54:50 -0700681 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700682}
683
Thomas Gleixner98458172016-02-26 18:43:25 +0000684static int takedown_cpu(unsigned int cpu)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700685{
Thomas Gleixnere69aab12016-02-26 18:43:43 +0000686 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
Thomas Gleixner98458172016-02-26 18:43:25 +0000687 int err;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700688
Thomas Gleixner2a58c522016-03-10 20:42:08 +0100689 /* Park the smpboot threads */
Thomas Gleixner1cf4f622016-02-26 18:43:39 +0000690 kthread_park(per_cpu_ptr(&cpuhp_state, cpu)->thread);
Thomas Gleixner2a58c522016-03-10 20:42:08 +0100691 smpboot_park_threads(cpu);
Thomas Gleixner1cf4f622016-02-26 18:43:39 +0000692
Peter Zijlstra6acce3e2013-10-11 14:38:20 +0200693 /*
Thomas Gleixnera8994182015-07-05 17:12:30 +0000694 * Prevent irq alloc/free while the dying cpu reorganizes the
695 * interrupt affinities.
696 */
697 irq_lock_sparse();
698
699 /*
Peter Zijlstra6acce3e2013-10-11 14:38:20 +0200700 * So now all preempt/rcu users must observe !cpu_active().
701 */
Thomas Gleixner090e77c2016-02-26 18:43:23 +0000702 err = stop_machine(take_cpu_down, NULL, cpumask_of(cpu));
Rusty Russell04321582008-07-28 12:16:29 -0500703 if (err) {
Sebastian Andrzej Siewior3b9d6da2016-04-08 14:40:15 +0200704 /* CPU refused to die */
Thomas Gleixnera8994182015-07-05 17:12:30 +0000705 irq_unlock_sparse();
Sebastian Andrzej Siewior3b9d6da2016-04-08 14:40:15 +0200706 /* Unpark the hotplug thread so we can rollback there */
707 kthread_unpark(per_cpu_ptr(&cpuhp_state, cpu)->thread);
Thomas Gleixner98458172016-02-26 18:43:25 +0000708 return err;
Satoru Takeuchi8fa1d7d2006-10-28 10:38:57 -0700709 }
Rusty Russell04321582008-07-28 12:16:29 -0500710 BUG_ON(cpu_online(cpu));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700711
Peter Zijlstra48c5ccae2010-11-13 19:32:29 +0100712 /*
Thomas Gleixneree1e7142016-08-18 14:57:16 +0200713 * The CPUHP_AP_SCHED_MIGRATE_DYING callback will have removed all
Peter Zijlstra48c5ccae2010-11-13 19:32:29 +0100714 * runnable tasks from the cpu, there's only the idle task left now
715 * that the migration thread is done doing the stop_machine thing.
Peter Zijlstra51a96c72010-11-19 20:37:53 +0100716 *
717 * Wait for the stop thread to go away.
Peter Zijlstra48c5ccae2010-11-13 19:32:29 +0100718 */
Thomas Gleixnere69aab12016-02-26 18:43:43 +0000719 wait_for_completion(&st->done);
720 BUG_ON(st->state != CPUHP_AP_IDLE_DEAD);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700721
Thomas Gleixnera8994182015-07-05 17:12:30 +0000722 /* Interrupts are moved away from the dying cpu, reenable alloc/free */
723 irq_unlock_sparse();
724
Preeti U Murthy345527b2015-03-30 14:59:19 +0530725 hotplug_cpu__broadcast_tick_pull(cpu);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700726 /* This actually kills the CPU. */
727 __cpu_die(cpu);
728
Thomas Gleixnera49b1162015-04-03 02:38:05 +0200729 tick_cleanup_dead_cpu(cpu);
Thomas Gleixner98458172016-02-26 18:43:25 +0000730 return 0;
731}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700732
Thomas Gleixner71f87b22016-03-03 10:52:10 +0100733static void cpuhp_complete_idle_dead(void *arg)
734{
735 struct cpuhp_cpu_state *st = arg;
736
737 complete(&st->done);
738}
739
Thomas Gleixnere69aab12016-02-26 18:43:43 +0000740void cpuhp_report_idle_dead(void)
741{
742 struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);
743
744 BUG_ON(st->state != CPUHP_AP_OFFLINE);
Thomas Gleixner27d50c72016-02-26 18:43:44 +0000745 rcu_report_dead(smp_processor_id());
Thomas Gleixner71f87b22016-03-03 10:52:10 +0100746 st->state = CPUHP_AP_IDLE_DEAD;
747 /*
748 * We cannot call complete after rcu_report_dead() so we delegate it
749 * to an online cpu.
750 */
751 smp_call_function_single(cpumask_first(cpu_online_mask),
752 cpuhp_complete_idle_dead, st, 0);
Thomas Gleixnere69aab12016-02-26 18:43:43 +0000753}
754
Thomas Gleixnercff7d372016-02-26 18:43:28 +0000755#else
Thomas Gleixnercff7d372016-02-26 18:43:28 +0000756#define takedown_cpu NULL
Thomas Gleixnercff7d372016-02-26 18:43:28 +0000757#endif
758
759#ifdef CONFIG_HOTPLUG_CPU
Thomas Gleixnercff7d372016-02-26 18:43:28 +0000760
Thomas Gleixner98458172016-02-26 18:43:25 +0000761/* Requires cpu_add_remove_lock to be held */
Thomas Gleixneraf1f4042016-02-26 18:43:30 +0000762static int __ref _cpu_down(unsigned int cpu, int tasks_frozen,
763 enum cpuhp_state target)
Thomas Gleixner98458172016-02-26 18:43:25 +0000764{
Thomas Gleixnercff7d372016-02-26 18:43:28 +0000765 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
766 int prev_state, ret = 0;
767 bool hasdied = false;
Thomas Gleixner98458172016-02-26 18:43:25 +0000768
769 if (num_online_cpus() == 1)
770 return -EBUSY;
771
Thomas Gleixner757c9892016-02-26 18:43:32 +0000772 if (!cpu_present(cpu))
Thomas Gleixner98458172016-02-26 18:43:25 +0000773 return -EINVAL;
774
775 cpu_hotplug_begin();
776
777 cpuhp_tasks_frozen = tasks_frozen;
778
Thomas Gleixnercff7d372016-02-26 18:43:28 +0000779 prev_state = st->state;
Thomas Gleixneraf1f4042016-02-26 18:43:30 +0000780 st->target = target;
Thomas Gleixner1cf4f622016-02-26 18:43:39 +0000781 /*
782 * If the current CPU state is in the range of the AP hotplug thread,
783 * then we need to kick the thread.
784 */
Thomas Gleixner8df3e072016-02-26 18:43:41 +0000785 if (st->state > CPUHP_TEARDOWN_CPU) {
Thomas Gleixner1cf4f622016-02-26 18:43:39 +0000786 ret = cpuhp_kick_ap_work(cpu);
787 /*
788 * The AP side has done the error rollback already. Just
789 * return the error code..
790 */
791 if (ret)
792 goto out;
793
794 /*
795 * We might have stopped still in the range of the AP hotplug
796 * thread. Nothing to do anymore.
797 */
Thomas Gleixner8df3e072016-02-26 18:43:41 +0000798 if (st->state > CPUHP_TEARDOWN_CPU)
Thomas Gleixner1cf4f622016-02-26 18:43:39 +0000799 goto out;
800 }
801 /*
Thomas Gleixner8df3e072016-02-26 18:43:41 +0000802 * The AP brought itself down to CPUHP_TEARDOWN_CPU. So we need
Thomas Gleixner1cf4f622016-02-26 18:43:39 +0000803 * to do the further cleanups.
804 */
Thomas Gleixnera7246322016-08-12 19:49:38 +0200805 ret = cpuhp_down_callbacks(cpu, st, target);
Sebastian Andrzej Siewior3b9d6da2016-04-08 14:40:15 +0200806 if (ret && st->state > CPUHP_TEARDOWN_CPU && st->state < prev_state) {
807 st->target = prev_state;
808 st->rollback = true;
809 cpuhp_kick_ap_work(cpu);
810 }
Thomas Gleixner98458172016-02-26 18:43:25 +0000811
Thomas Gleixnercff7d372016-02-26 18:43:28 +0000812 hasdied = prev_state != st->state && st->state == CPUHP_OFFLINE;
Thomas Gleixner1cf4f622016-02-26 18:43:39 +0000813out:
Gautham R Shenoyd2219382008-01-25 21:08:01 +0100814 cpu_hotplug_done();
Thomas Gleixnercff7d372016-02-26 18:43:28 +0000815 return ret;
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -0700816}
817
Thomas Gleixneraf1f4042016-02-26 18:43:30 +0000818static int do_cpu_down(unsigned int cpu, enum cpuhp_state target)
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -0700819{
Heiko Carstens9ea09af2008-12-22 12:36:30 +0100820 int err;
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -0700821
Gautham R Shenoyd2219382008-01-25 21:08:01 +0100822 cpu_maps_update_begin();
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -0700823
Max Krasnyanskye761b772008-07-15 04:43:49 -0700824 if (cpu_hotplug_disabled) {
825 err = -EBUSY;
826 goto out;
827 }
828
Thomas Gleixneraf1f4042016-02-26 18:43:30 +0000829 err = _cpu_down(cpu, 0, target);
Max Krasnyanskye761b772008-07-15 04:43:49 -0700830
Max Krasnyanskye761b772008-07-15 04:43:49 -0700831out:
Gautham R Shenoyd2219382008-01-25 21:08:01 +0100832 cpu_maps_update_done();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700833 return err;
834}
Thomas Gleixneraf1f4042016-02-26 18:43:30 +0000835int cpu_down(unsigned int cpu)
836{
837 return do_cpu_down(cpu, CPUHP_OFFLINE);
838}
Zhang Ruib62b8ef2008-04-29 02:35:56 -0400839EXPORT_SYMBOL(cpu_down);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700840#endif /*CONFIG_HOTPLUG_CPU*/
841
Thomas Gleixner4baa0af2016-02-26 18:43:29 +0000842/**
Thomas Gleixneree1e7142016-08-18 14:57:16 +0200843 * notify_cpu_starting(cpu) - Invoke the callbacks on the starting CPU
Thomas Gleixner4baa0af2016-02-26 18:43:29 +0000844 * @cpu: cpu that just started
845 *
Thomas Gleixner4baa0af2016-02-26 18:43:29 +0000846 * It must be called by the arch code on the new cpu, before the new cpu
847 * enables interrupts and before the "boot" cpu returns from __cpu_up().
848 */
849void notify_cpu_starting(unsigned int cpu)
850{
851 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
852 enum cpuhp_state target = min((int)st->target, CPUHP_AP_ONLINE);
853
Sebastian Andrzej Siewior0c6d4572016-08-17 14:21:04 +0200854 rcu_cpu_starting(cpu); /* Enables RCU usage on this CPU. */
Thomas Gleixner4baa0af2016-02-26 18:43:29 +0000855 while (st->state < target) {
Thomas Gleixner4baa0af2016-02-26 18:43:29 +0000856 st->state++;
Thomas Gleixnercf392d12016-08-12 19:49:39 +0200857 cpuhp_invoke_callback(cpu, st->state, true, NULL);
Thomas Gleixner4baa0af2016-02-26 18:43:29 +0000858 }
859}
860
Thomas Gleixner949338e2016-02-26 18:43:35 +0000861/*
862 * Called from the idle task. We need to set active here, so we can kick off
Thomas Gleixner8df3e072016-02-26 18:43:41 +0000863 * the stopper thread and unpark the smpboot threads. If the target state is
864 * beyond CPUHP_AP_ONLINE_IDLE we kick cpuhp thread and let it bring up the
865 * cpu further.
Thomas Gleixner949338e2016-02-26 18:43:35 +0000866 */
Thomas Gleixner8df3e072016-02-26 18:43:41 +0000867void cpuhp_online_idle(enum cpuhp_state state)
Thomas Gleixner949338e2016-02-26 18:43:35 +0000868{
Thomas Gleixner8df3e072016-02-26 18:43:41 +0000869 struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);
870 unsigned int cpu = smp_processor_id();
871
872 /* Happens for the boot cpu */
873 if (state != CPUHP_AP_ONLINE_IDLE)
874 return;
875
876 st->state = CPUHP_AP_ONLINE_IDLE;
Thomas Gleixner1cf4f622016-02-26 18:43:39 +0000877
Thomas Gleixner8df3e072016-02-26 18:43:41 +0000878 /* Unpark the stopper thread and the hotplug thread of this cpu */
Thomas Gleixner949338e2016-02-26 18:43:35 +0000879 stop_machine_unpark(cpu);
Thomas Gleixner1cf4f622016-02-26 18:43:39 +0000880 kthread_unpark(st->thread);
Thomas Gleixner8df3e072016-02-26 18:43:41 +0000881
882 /* Should we go further up ? */
883 if (st->target > CPUHP_AP_ONLINE_IDLE)
884 __cpuhp_kick_ap_work(st);
885 else
886 complete(&st->done);
Thomas Gleixner949338e2016-02-26 18:43:35 +0000887}
888
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -0700889/* Requires cpu_add_remove_lock to be held */
Thomas Gleixneraf1f4042016-02-26 18:43:30 +0000890static int _cpu_up(unsigned int cpu, int tasks_frozen, enum cpuhp_state target)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700891{
Thomas Gleixnercff7d372016-02-26 18:43:28 +0000892 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
Suresh Siddha3bb5d2e2012-04-20 17:08:50 -0700893 struct task_struct *idle;
Thomas Gleixner2e1a3482016-02-26 18:43:37 +0000894 int ret = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700895
Gautham R Shenoyd2219382008-01-25 21:08:01 +0100896 cpu_hotplug_begin();
Thomas Gleixner38498a62012-04-20 13:05:44 +0000897
Thomas Gleixner757c9892016-02-26 18:43:32 +0000898 if (!cpu_present(cpu)) {
Yasuaki Ishimatsu5e5041f2012-10-23 01:30:54 +0200899 ret = -EINVAL;
900 goto out;
901 }
902
Thomas Gleixner757c9892016-02-26 18:43:32 +0000903 /*
904 * The caller of do_cpu_up might have raced with another
905 * caller. Ignore it for now.
906 */
907 if (st->state >= target)
Thomas Gleixner38498a62012-04-20 13:05:44 +0000908 goto out;
Thomas Gleixner757c9892016-02-26 18:43:32 +0000909
910 if (st->state == CPUHP_OFFLINE) {
911 /* Let it fail before we try to bring the cpu up */
912 idle = idle_thread_get(cpu);
913 if (IS_ERR(idle)) {
914 ret = PTR_ERR(idle);
915 goto out;
916 }
Suresh Siddha3bb5d2e2012-04-20 17:08:50 -0700917 }
Thomas Gleixner38498a62012-04-20 13:05:44 +0000918
Thomas Gleixnerba997462016-02-26 18:43:24 +0000919 cpuhp_tasks_frozen = tasks_frozen;
920
Thomas Gleixneraf1f4042016-02-26 18:43:30 +0000921 st->target = target;
Thomas Gleixner1cf4f622016-02-26 18:43:39 +0000922 /*
923 * If the current CPU state is in the range of the AP hotplug thread,
924 * then we need to kick the thread once more.
925 */
Thomas Gleixner8df3e072016-02-26 18:43:41 +0000926 if (st->state > CPUHP_BRINGUP_CPU) {
Thomas Gleixner1cf4f622016-02-26 18:43:39 +0000927 ret = cpuhp_kick_ap_work(cpu);
928 /*
929 * The AP side has done the error rollback already. Just
930 * return the error code..
931 */
932 if (ret)
933 goto out;
934 }
935
936 /*
937 * Try to reach the target state. We max out on the BP at
Thomas Gleixner8df3e072016-02-26 18:43:41 +0000938 * CPUHP_BRINGUP_CPU. After that the AP hotplug thread is
Thomas Gleixner1cf4f622016-02-26 18:43:39 +0000939 * responsible for bringing it up to the target state.
940 */
Thomas Gleixner8df3e072016-02-26 18:43:41 +0000941 target = min((int)target, CPUHP_BRINGUP_CPU);
Thomas Gleixnera7246322016-08-12 19:49:38 +0200942 ret = cpuhp_up_callbacks(cpu, st, target);
Thomas Gleixner38498a62012-04-20 13:05:44 +0000943out:
Gautham R Shenoyd2219382008-01-25 21:08:01 +0100944 cpu_hotplug_done();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700945 return ret;
946}
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -0700947
Thomas Gleixneraf1f4042016-02-26 18:43:30 +0000948static int do_cpu_up(unsigned int cpu, enum cpuhp_state target)
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -0700949{
950 int err = 0;
minskey guocf234222010-05-24 14:32:41 -0700951
Rusty Russelle0b582e2009-01-01 10:12:28 +1030952 if (!cpu_possible(cpu)) {
Fabian Frederick84117da2014-06-04 16:11:17 -0700953 pr_err("can't online cpu %d because it is not configured as may-hotadd at boot time\n",
954 cpu);
Chen Gong87d5e0232010-03-05 13:42:38 -0800955#if defined(CONFIG_IA64)
Fabian Frederick84117da2014-06-04 16:11:17 -0700956 pr_err("please check additional_cpus= boot parameter\n");
KAMEZAWA Hiroyuki73e753a2007-10-18 23:40:47 -0700957#endif
958 return -EINVAL;
959 }
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -0700960
Toshi Kani01b0f192013-11-12 15:07:25 -0800961 err = try_online_node(cpu_to_node(cpu));
962 if (err)
963 return err;
minskey guocf234222010-05-24 14:32:41 -0700964
Gautham R Shenoyd2219382008-01-25 21:08:01 +0100965 cpu_maps_update_begin();
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -0700966
Max Krasnyanskye761b772008-07-15 04:43:49 -0700967 if (cpu_hotplug_disabled) {
968 err = -EBUSY;
969 goto out;
970 }
971
Thomas Gleixneraf1f4042016-02-26 18:43:30 +0000972 err = _cpu_up(cpu, 0, target);
Max Krasnyanskye761b772008-07-15 04:43:49 -0700973out:
Gautham R Shenoyd2219382008-01-25 21:08:01 +0100974 cpu_maps_update_done();
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -0700975 return err;
976}
Thomas Gleixneraf1f4042016-02-26 18:43:30 +0000977
978int cpu_up(unsigned int cpu)
979{
980 return do_cpu_up(cpu, CPUHP_ONLINE);
981}
Paul E. McKenneya513f6b2011-12-11 21:54:45 -0800982EXPORT_SYMBOL_GPL(cpu_up);
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -0700983
Rafael J. Wysockif3de4be2007-08-30 23:56:29 -0700984#ifdef CONFIG_PM_SLEEP_SMP
Rusty Russelle0b582e2009-01-01 10:12:28 +1030985static cpumask_var_t frozen_cpus;
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -0700986
James Morsed391e552016-08-17 13:50:25 +0100987int freeze_secondary_cpus(int primary)
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -0700988{
James Morsed391e552016-08-17 13:50:25 +0100989 int cpu, error = 0;
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -0700990
Gautham R Shenoyd2219382008-01-25 21:08:01 +0100991 cpu_maps_update_begin();
James Morsed391e552016-08-17 13:50:25 +0100992 if (!cpu_online(primary))
993 primary = cpumask_first(cpu_online_mask);
Xiaotian Feng9ee349a2009-12-16 18:04:32 +0100994 /*
995 * We take down all of the non-boot CPUs in one shot to avoid races
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -0700996 * with the userspace trying to use the CPU hotplug at the same time
997 */
Rusty Russelle0b582e2009-01-01 10:12:28 +1030998 cpumask_clear(frozen_cpus);
Peter Zijlstra6ad4c182009-11-25 13:31:39 +0100999
Fabian Frederick84117da2014-06-04 16:11:17 -07001000 pr_info("Disabling non-boot CPUs ...\n");
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -07001001 for_each_online_cpu(cpu) {
James Morsed391e552016-08-17 13:50:25 +01001002 if (cpu == primary)
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -07001003 continue;
Todd E Brandtbb3632c2014-06-06 05:40:17 -07001004 trace_suspend_resume(TPS("CPU_OFF"), cpu, true);
Thomas Gleixneraf1f4042016-02-26 18:43:30 +00001005 error = _cpu_down(cpu, 1, CPUHP_OFFLINE);
Todd E Brandtbb3632c2014-06-06 05:40:17 -07001006 trace_suspend_resume(TPS("CPU_OFF"), cpu, false);
Mike Travisfeae3202009-11-17 18:22:13 -06001007 if (!error)
Rusty Russelle0b582e2009-01-01 10:12:28 +10301008 cpumask_set_cpu(cpu, frozen_cpus);
Mike Travisfeae3202009-11-17 18:22:13 -06001009 else {
Fabian Frederick84117da2014-06-04 16:11:17 -07001010 pr_err("Error taking CPU%d down: %d\n", cpu, error);
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -07001011 break;
1012 }
1013 }
Joseph Cihula86886e52009-06-30 19:31:07 -07001014
Vitaly Kuznetsov89af7ba2015-08-05 00:52:46 -07001015 if (!error)
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -07001016 BUG_ON(num_online_cpus() > 1);
Vitaly Kuznetsov89af7ba2015-08-05 00:52:46 -07001017 else
Fabian Frederick84117da2014-06-04 16:11:17 -07001018 pr_err("Non-boot CPUs are not disabled\n");
Vitaly Kuznetsov89af7ba2015-08-05 00:52:46 -07001019
1020 /*
1021 * Make sure the CPUs won't be enabled by someone else. We need to do
1022 * this even in case of failure as all disable_nonboot_cpus() users are
1023 * supposed to do enable_nonboot_cpus() on the failure path.
1024 */
1025 cpu_hotplug_disabled++;
1026
Gautham R Shenoyd2219382008-01-25 21:08:01 +01001027 cpu_maps_update_done();
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -07001028 return error;
1029}
1030
Suresh Siddhad0af9ee2009-08-19 18:05:36 -07001031void __weak arch_enable_nonboot_cpus_begin(void)
1032{
1033}
1034
1035void __weak arch_enable_nonboot_cpus_end(void)
1036{
1037}
1038
Mathias Krause71cf5ae2015-07-19 20:06:22 +02001039void enable_nonboot_cpus(void)
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -07001040{
1041 int cpu, error;
1042
1043 /* Allow everyone to use the CPU hotplug again */
Gautham R Shenoyd2219382008-01-25 21:08:01 +01001044 cpu_maps_update_begin();
Lianwei Wang01b41152016-06-09 23:43:28 -07001045 __cpu_hotplug_enable();
Rusty Russelle0b582e2009-01-01 10:12:28 +10301046 if (cpumask_empty(frozen_cpus))
Rafael J. Wysocki1d64b9c2007-04-01 23:49:49 -07001047 goto out;
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -07001048
Fabian Frederick84117da2014-06-04 16:11:17 -07001049 pr_info("Enabling non-boot CPUs ...\n");
Suresh Siddhad0af9ee2009-08-19 18:05:36 -07001050
1051 arch_enable_nonboot_cpus_begin();
1052
Rusty Russelle0b582e2009-01-01 10:12:28 +10301053 for_each_cpu(cpu, frozen_cpus) {
Todd E Brandtbb3632c2014-06-06 05:40:17 -07001054 trace_suspend_resume(TPS("CPU_ON"), cpu, true);
Thomas Gleixneraf1f4042016-02-26 18:43:30 +00001055 error = _cpu_up(cpu, 1, CPUHP_ONLINE);
Todd E Brandtbb3632c2014-06-06 05:40:17 -07001056 trace_suspend_resume(TPS("CPU_ON"), cpu, false);
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -07001057 if (!error) {
Fabian Frederick84117da2014-06-04 16:11:17 -07001058 pr_info("CPU%d is up\n", cpu);
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -07001059 continue;
1060 }
Fabian Frederick84117da2014-06-04 16:11:17 -07001061 pr_warn("Error taking CPU%d up: %d\n", cpu, error);
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -07001062 }
Suresh Siddhad0af9ee2009-08-19 18:05:36 -07001063
1064 arch_enable_nonboot_cpus_end();
1065
Rusty Russelle0b582e2009-01-01 10:12:28 +10301066 cpumask_clear(frozen_cpus);
Rafael J. Wysocki1d64b9c2007-04-01 23:49:49 -07001067out:
Gautham R Shenoyd2219382008-01-25 21:08:01 +01001068 cpu_maps_update_done();
Rafael J. Wysockie3920fb2006-09-25 23:32:48 -07001069}
Rusty Russelle0b582e2009-01-01 10:12:28 +10301070
Fenghua Yud7268a32011-11-15 21:59:31 +01001071static int __init alloc_frozen_cpus(void)
Rusty Russelle0b582e2009-01-01 10:12:28 +10301072{
1073 if (!alloc_cpumask_var(&frozen_cpus, GFP_KERNEL|__GFP_ZERO))
1074 return -ENOMEM;
1075 return 0;
1076}
1077core_initcall(alloc_frozen_cpus);
Srivatsa S. Bhat79cfbdf2011-11-03 00:59:25 +01001078
1079/*
Srivatsa S. Bhat79cfbdf2011-11-03 00:59:25 +01001080 * When callbacks for CPU hotplug notifications are being executed, we must
1081 * ensure that the state of the system with respect to the tasks being frozen
1082 * or not, as reported by the notification, remains unchanged *throughout the
1083 * duration* of the execution of the callbacks.
1084 * Hence we need to prevent the freezer from racing with regular CPU hotplug.
1085 *
1086 * This synchronization is implemented by mutually excluding regular CPU
1087 * hotplug and Suspend/Hibernate call paths by hooking onto the Suspend/
1088 * Hibernate notifications.
1089 */
1090static int
1091cpu_hotplug_pm_callback(struct notifier_block *nb,
1092 unsigned long action, void *ptr)
1093{
1094 switch (action) {
1095
1096 case PM_SUSPEND_PREPARE:
1097 case PM_HIBERNATION_PREPARE:
Srivatsa S. Bhat16e53db2013-06-12 14:04:36 -07001098 cpu_hotplug_disable();
Srivatsa S. Bhat79cfbdf2011-11-03 00:59:25 +01001099 break;
1100
1101 case PM_POST_SUSPEND:
1102 case PM_POST_HIBERNATION:
Srivatsa S. Bhat16e53db2013-06-12 14:04:36 -07001103 cpu_hotplug_enable();
Srivatsa S. Bhat79cfbdf2011-11-03 00:59:25 +01001104 break;
1105
1106 default:
1107 return NOTIFY_DONE;
1108 }
1109
1110 return NOTIFY_OK;
1111}
1112
1113
Fenghua Yud7268a32011-11-15 21:59:31 +01001114static int __init cpu_hotplug_pm_sync_init(void)
Srivatsa S. Bhat79cfbdf2011-11-03 00:59:25 +01001115{
Fenghua Yu6e32d472012-11-13 11:32:43 -08001116 /*
1117 * cpu_hotplug_pm_callback has higher priority than x86
1118 * bsp_pm_callback which depends on cpu_hotplug_pm_callback
1119 * to disable cpu hotplug to avoid cpu hotplug race.
1120 */
Srivatsa S. Bhat79cfbdf2011-11-03 00:59:25 +01001121 pm_notifier(cpu_hotplug_pm_callback, 0);
1122 return 0;
1123}
1124core_initcall(cpu_hotplug_pm_sync_init);
1125
Rafael J. Wysockif3de4be2007-08-30 23:56:29 -07001126#endif /* CONFIG_PM_SLEEP_SMP */
Max Krasnyansky68f4f1e2008-05-29 11:17:02 -07001127
1128#endif /* CONFIG_SMP */
Mike Travisb8d317d2008-07-24 18:21:29 -07001129
Thomas Gleixnercff7d372016-02-26 18:43:28 +00001130/* Boot processor state steps */
1131static struct cpuhp_step cpuhp_bp_states[] = {
1132 [CPUHP_OFFLINE] = {
1133 .name = "offline",
Thomas Gleixner3c1627e2016-09-05 15:28:36 +02001134 .startup.single = NULL,
1135 .teardown.single = NULL,
Thomas Gleixnercff7d372016-02-26 18:43:28 +00001136 },
1137#ifdef CONFIG_SMP
1138 [CPUHP_CREATE_THREADS]= {
Thomas Gleixner677f6642016-09-06 16:13:48 +02001139 .name = "threads:prepare",
Thomas Gleixner3c1627e2016-09-05 15:28:36 +02001140 .startup.single = smpboot_create_threads,
1141 .teardown.single = NULL,
Thomas Gleixner757c9892016-02-26 18:43:32 +00001142 .cant_stop = true,
Thomas Gleixnercff7d372016-02-26 18:43:28 +00001143 },
Thomas Gleixner00e16c32016-07-13 17:16:09 +00001144 [CPUHP_PERF_PREPARE] = {
Thomas Gleixner3c1627e2016-09-05 15:28:36 +02001145 .name = "perf:prepare",
1146 .startup.single = perf_event_init_cpu,
1147 .teardown.single = perf_event_exit_cpu,
Thomas Gleixner00e16c32016-07-13 17:16:09 +00001148 },
Thomas Gleixner7ee681b2016-07-13 17:16:29 +00001149 [CPUHP_WORKQUEUE_PREP] = {
Thomas Gleixner3c1627e2016-09-05 15:28:36 +02001150 .name = "workqueue:prepare",
1151 .startup.single = workqueue_prepare_cpu,
1152 .teardown.single = NULL,
Thomas Gleixner7ee681b2016-07-13 17:16:29 +00001153 },
Thomas Gleixner27590dc2016-07-15 10:41:04 +02001154 [CPUHP_HRTIMERS_PREPARE] = {
Thomas Gleixner3c1627e2016-09-05 15:28:36 +02001155 .name = "hrtimers:prepare",
1156 .startup.single = hrtimers_prepare_cpu,
1157 .teardown.single = hrtimers_dead_cpu,
Thomas Gleixner27590dc2016-07-15 10:41:04 +02001158 },
Richard Weinberger31487f82016-07-13 17:17:01 +00001159 [CPUHP_SMPCFD_PREPARE] = {
Thomas Gleixner677f6642016-09-06 16:13:48 +02001160 .name = "smpcfd:prepare",
Thomas Gleixner3c1627e2016-09-05 15:28:36 +02001161 .startup.single = smpcfd_prepare_cpu,
1162 .teardown.single = smpcfd_dead_cpu,
Richard Weinberger31487f82016-07-13 17:17:01 +00001163 },
Richard Weinbergere6d49892016-08-18 14:57:17 +02001164 [CPUHP_RELAY_PREPARE] = {
1165 .name = "relay:prepare",
1166 .startup.single = relay_prepare_cpu,
1167 .teardown.single = NULL,
1168 },
Sebastian Andrzej Siewior6731d4f2016-08-23 14:53:19 +02001169 [CPUHP_SLAB_PREPARE] = {
1170 .name = "slab:prepare",
1171 .startup.single = slab_prepare_cpu,
1172 .teardown.single = slab_dead_cpu,
Thomas Gleixnercff7d372016-02-26 18:43:28 +00001173 },
Thomas Gleixner4df83742016-07-13 17:17:03 +00001174 [CPUHP_RCUTREE_PREP] = {
Thomas Gleixner677f6642016-09-06 16:13:48 +02001175 .name = "RCU/tree:prepare",
Thomas Gleixner3c1627e2016-09-05 15:28:36 +02001176 .startup.single = rcutree_prepare_cpu,
1177 .teardown.single = rcutree_dead_cpu,
Thomas Gleixner4df83742016-07-13 17:17:03 +00001178 },
Thomas Gleixnercff7d372016-02-26 18:43:28 +00001179 /*
Richard Cochran4fae16d2016-07-27 11:08:18 +02001180 * On the tear-down path, timers_dead_cpu() must be invoked
1181 * before blk_mq_queue_reinit_notify() from notify_dead(),
1182 * otherwise a RCU stall occurs.
1183 */
1184 [CPUHP_TIMERS_DEAD] = {
Thomas Gleixner3c1627e2016-09-05 15:28:36 +02001185 .name = "timers:dead",
1186 .startup.single = NULL,
1187 .teardown.single = timers_dead_cpu,
Richard Cochran4fae16d2016-07-27 11:08:18 +02001188 },
Thomas Gleixnerd10ef6f2016-03-08 10:36:13 +01001189 /* Kicks the plugged cpu into life */
Thomas Gleixnercff7d372016-02-26 18:43:28 +00001190 [CPUHP_BRINGUP_CPU] = {
1191 .name = "cpu:bringup",
Thomas Gleixner3c1627e2016-09-05 15:28:36 +02001192 .startup.single = bringup_cpu,
1193 .teardown.single = NULL,
Thomas Gleixner757c9892016-02-26 18:43:32 +00001194 .cant_stop = true,
Thomas Gleixner4baa0af2016-02-26 18:43:29 +00001195 },
Richard Weinberger31487f82016-07-13 17:17:01 +00001196 [CPUHP_AP_SMPCFD_DYING] = {
Thomas Gleixner677f6642016-09-06 16:13:48 +02001197 .name = "smpcfd:dying",
Thomas Gleixner3c1627e2016-09-05 15:28:36 +02001198 .startup.single = NULL,
1199 .teardown.single = smpcfd_dying_cpu,
Richard Weinberger31487f82016-07-13 17:17:01 +00001200 },
Thomas Gleixnerd10ef6f2016-03-08 10:36:13 +01001201 /*
1202 * Handled on controll processor until the plugged processor manages
1203 * this itself.
1204 */
Thomas Gleixner4baa0af2016-02-26 18:43:29 +00001205 [CPUHP_TEARDOWN_CPU] = {
1206 .name = "cpu:teardown",
Thomas Gleixner3c1627e2016-09-05 15:28:36 +02001207 .startup.single = NULL,
1208 .teardown.single = takedown_cpu,
Thomas Gleixner757c9892016-02-26 18:43:32 +00001209 .cant_stop = true,
Thomas Gleixnercff7d372016-02-26 18:43:28 +00001210 },
Thomas Gleixnera7c734142016-07-12 21:59:23 +02001211#else
1212 [CPUHP_BRINGUP_CPU] = { },
Thomas Gleixnercff7d372016-02-26 18:43:28 +00001213#endif
Thomas Gleixnercff7d372016-02-26 18:43:28 +00001214};
1215
Thomas Gleixner4baa0af2016-02-26 18:43:29 +00001216/* Application processor state steps */
1217static struct cpuhp_step cpuhp_ap_states[] = {
1218#ifdef CONFIG_SMP
Thomas Gleixnerd10ef6f2016-03-08 10:36:13 +01001219 /* Final state before CPU kills itself */
1220 [CPUHP_AP_IDLE_DEAD] = {
1221 .name = "idle:dead",
1222 },
1223 /*
1224 * Last state before CPU enters the idle loop to die. Transient state
1225 * for synchronization.
1226 */
1227 [CPUHP_AP_OFFLINE] = {
1228 .name = "ap:offline",
1229 .cant_stop = true,
1230 },
Thomas Gleixner9cf72432016-03-10 12:54:09 +01001231 /* First state is scheduler control. Interrupts are disabled */
1232 [CPUHP_AP_SCHED_STARTING] = {
1233 .name = "sched:starting",
Thomas Gleixner3c1627e2016-09-05 15:28:36 +02001234 .startup.single = sched_cpu_starting,
1235 .teardown.single = sched_cpu_dying,
Thomas Gleixner9cf72432016-03-10 12:54:09 +01001236 },
Thomas Gleixner4df83742016-07-13 17:17:03 +00001237 [CPUHP_AP_RCUTREE_DYING] = {
Thomas Gleixner677f6642016-09-06 16:13:48 +02001238 .name = "RCU/tree:dying",
Thomas Gleixner3c1627e2016-09-05 15:28:36 +02001239 .startup.single = NULL,
1240 .teardown.single = rcutree_dying_cpu,
Thomas Gleixner4baa0af2016-02-26 18:43:29 +00001241 },
Thomas Gleixnerd10ef6f2016-03-08 10:36:13 +01001242 /* Entry state on starting. Interrupts enabled from here on. Transient
1243 * state for synchronsization */
1244 [CPUHP_AP_ONLINE] = {
1245 .name = "ap:online",
1246 },
1247 /* Handle smpboot threads park/unpark */
Thomas Gleixner1cf4f622016-02-26 18:43:39 +00001248 [CPUHP_AP_SMPBOOT_THREADS] = {
Thomas Gleixner677f6642016-09-06 16:13:48 +02001249 .name = "smpboot/threads:online",
Thomas Gleixner3c1627e2016-09-05 15:28:36 +02001250 .startup.single = smpboot_unpark_threads,
1251 .teardown.single = NULL,
Thomas Gleixner1cf4f622016-02-26 18:43:39 +00001252 },
Thomas Gleixner00e16c32016-07-13 17:16:09 +00001253 [CPUHP_AP_PERF_ONLINE] = {
Thomas Gleixner3c1627e2016-09-05 15:28:36 +02001254 .name = "perf:online",
1255 .startup.single = perf_event_init_cpu,
1256 .teardown.single = perf_event_exit_cpu,
Thomas Gleixner00e16c32016-07-13 17:16:09 +00001257 },
Thomas Gleixner7ee681b2016-07-13 17:16:29 +00001258 [CPUHP_AP_WORKQUEUE_ONLINE] = {
Thomas Gleixner3c1627e2016-09-05 15:28:36 +02001259 .name = "workqueue:online",
1260 .startup.single = workqueue_online_cpu,
1261 .teardown.single = workqueue_offline_cpu,
Thomas Gleixner7ee681b2016-07-13 17:16:29 +00001262 },
Thomas Gleixner4df83742016-07-13 17:17:03 +00001263 [CPUHP_AP_RCUTREE_ONLINE] = {
Thomas Gleixner677f6642016-09-06 16:13:48 +02001264 .name = "RCU/tree:online",
Thomas Gleixner3c1627e2016-09-05 15:28:36 +02001265 .startup.single = rcutree_online_cpu,
1266 .teardown.single = rcutree_offline_cpu,
Thomas Gleixner4df83742016-07-13 17:17:03 +00001267 },
Thomas Gleixner4baa0af2016-02-26 18:43:29 +00001268#endif
Thomas Gleixnerd10ef6f2016-03-08 10:36:13 +01001269 /*
1270 * The dynamically registered state space is here
1271 */
1272
Thomas Gleixneraaddd7d2016-03-10 12:54:19 +01001273#ifdef CONFIG_SMP
1274 /* Last state is scheduler control setting the cpu active */
1275 [CPUHP_AP_ACTIVE] = {
1276 .name = "sched:active",
Thomas Gleixner3c1627e2016-09-05 15:28:36 +02001277 .startup.single = sched_cpu_activate,
1278 .teardown.single = sched_cpu_deactivate,
Thomas Gleixneraaddd7d2016-03-10 12:54:19 +01001279 },
1280#endif
1281
Thomas Gleixnerd10ef6f2016-03-08 10:36:13 +01001282 /* CPU is fully up and running. */
Thomas Gleixner4baa0af2016-02-26 18:43:29 +00001283 [CPUHP_ONLINE] = {
1284 .name = "online",
Thomas Gleixner3c1627e2016-09-05 15:28:36 +02001285 .startup.single = NULL,
1286 .teardown.single = NULL,
Thomas Gleixner4baa0af2016-02-26 18:43:29 +00001287 },
1288};
1289
Thomas Gleixner5b7aa872016-02-26 18:43:33 +00001290/* Sanity check for callbacks */
1291static int cpuhp_cb_check(enum cpuhp_state state)
1292{
1293 if (state <= CPUHP_OFFLINE || state >= CPUHP_ONLINE)
1294 return -EINVAL;
1295 return 0;
1296}
1297
Thomas Gleixnerdc280d932016-12-21 20:19:49 +01001298/*
1299 * Returns a free for dynamic slot assignment of the Online state. The states
1300 * are protected by the cpuhp_slot_states mutex and an empty slot is identified
1301 * by having no name assigned.
1302 */
1303static int cpuhp_reserve_state(enum cpuhp_state state)
1304{
1305 enum cpuhp_state i;
1306
1307 for (i = CPUHP_AP_ONLINE_DYN; i <= CPUHP_AP_ONLINE_DYN_END; i++) {
1308 if (!cpuhp_ap_states[i].name)
1309 return i;
1310 }
1311 WARN(1, "No more dynamic states available for CPU hotplug\n");
1312 return -ENOSPC;
1313}
1314
1315static int cpuhp_store_callbacks(enum cpuhp_state state, const char *name,
1316 int (*startup)(unsigned int cpu),
1317 int (*teardown)(unsigned int cpu),
1318 bool multi_instance)
Thomas Gleixner5b7aa872016-02-26 18:43:33 +00001319{
1320 /* (Un)Install the callbacks for further cpu hotplug operations */
1321 struct cpuhp_step *sp;
Thomas Gleixnerdc280d932016-12-21 20:19:49 +01001322 int ret = 0;
Thomas Gleixner5b7aa872016-02-26 18:43:33 +00001323
1324 mutex_lock(&cpuhp_state_mutex);
Thomas Gleixnerdc280d932016-12-21 20:19:49 +01001325
1326 if (state == CPUHP_AP_ONLINE_DYN) {
1327 ret = cpuhp_reserve_state(state);
1328 if (ret < 0)
1329 goto out;
1330 state = ret;
1331 }
Thomas Gleixner5b7aa872016-02-26 18:43:33 +00001332 sp = cpuhp_get_step(state);
Thomas Gleixnerdc280d932016-12-21 20:19:49 +01001333 if (name && sp->name) {
1334 ret = -EBUSY;
1335 goto out;
1336 }
Thomas Gleixner3c1627e2016-09-05 15:28:36 +02001337 sp->startup.single = startup;
1338 sp->teardown.single = teardown;
Thomas Gleixner5b7aa872016-02-26 18:43:33 +00001339 sp->name = name;
Thomas Gleixnercf392d12016-08-12 19:49:39 +02001340 sp->multi_instance = multi_instance;
1341 INIT_HLIST_HEAD(&sp->list);
Thomas Gleixnerdc280d932016-12-21 20:19:49 +01001342out:
Thomas Gleixner5b7aa872016-02-26 18:43:33 +00001343 mutex_unlock(&cpuhp_state_mutex);
Thomas Gleixnerdc280d932016-12-21 20:19:49 +01001344 return ret;
Thomas Gleixner5b7aa872016-02-26 18:43:33 +00001345}
1346
1347static void *cpuhp_get_teardown_cb(enum cpuhp_state state)
1348{
Thomas Gleixner3c1627e2016-09-05 15:28:36 +02001349 return cpuhp_get_step(state)->teardown.single;
Thomas Gleixner5b7aa872016-02-26 18:43:33 +00001350}
1351
Thomas Gleixner5b7aa872016-02-26 18:43:33 +00001352/*
1353 * Call the startup/teardown function for a step either on the AP or
1354 * on the current CPU.
1355 */
Thomas Gleixnercf392d12016-08-12 19:49:39 +02001356static int cpuhp_issue_call(int cpu, enum cpuhp_state state, bool bringup,
1357 struct hlist_node *node)
Thomas Gleixner5b7aa872016-02-26 18:43:33 +00001358{
Thomas Gleixnera7246322016-08-12 19:49:38 +02001359 struct cpuhp_step *sp = cpuhp_get_step(state);
Thomas Gleixner5b7aa872016-02-26 18:43:33 +00001360 int ret;
1361
Thomas Gleixner3c1627e2016-09-05 15:28:36 +02001362 if ((bringup && !sp->startup.single) ||
1363 (!bringup && !sp->teardown.single))
Thomas Gleixner5b7aa872016-02-26 18:43:33 +00001364 return 0;
Thomas Gleixner5b7aa872016-02-26 18:43:33 +00001365 /*
1366 * The non AP bound callbacks can fail on bringup. On teardown
1367 * e.g. module removal we crash for now.
1368 */
Thomas Gleixner1cf4f622016-02-26 18:43:39 +00001369#ifdef CONFIG_SMP
1370 if (cpuhp_is_ap_state(state))
Thomas Gleixnercf392d12016-08-12 19:49:39 +02001371 ret = cpuhp_invoke_ap_callback(cpu, state, bringup, node);
Thomas Gleixner1cf4f622016-02-26 18:43:39 +00001372 else
Thomas Gleixnercf392d12016-08-12 19:49:39 +02001373 ret = cpuhp_invoke_callback(cpu, state, bringup, node);
Thomas Gleixner1cf4f622016-02-26 18:43:39 +00001374#else
Thomas Gleixnercf392d12016-08-12 19:49:39 +02001375 ret = cpuhp_invoke_callback(cpu, state, bringup, node);
Thomas Gleixner1cf4f622016-02-26 18:43:39 +00001376#endif
Thomas Gleixner5b7aa872016-02-26 18:43:33 +00001377 BUG_ON(ret && !bringup);
1378 return ret;
1379}
1380
1381/*
1382 * Called from __cpuhp_setup_state on a recoverable failure.
1383 *
1384 * Note: The teardown callbacks for rollback are not allowed to fail!
1385 */
1386static void cpuhp_rollback_install(int failedcpu, enum cpuhp_state state,
Thomas Gleixnercf392d12016-08-12 19:49:39 +02001387 struct hlist_node *node)
Thomas Gleixner5b7aa872016-02-26 18:43:33 +00001388{
1389 int cpu;
1390
Thomas Gleixner5b7aa872016-02-26 18:43:33 +00001391 /* Roll back the already executed steps on the other cpus */
1392 for_each_present_cpu(cpu) {
1393 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1394 int cpustate = st->state;
1395
1396 if (cpu >= failedcpu)
1397 break;
1398
1399 /* Did we invoke the startup call on that cpu ? */
1400 if (cpustate >= state)
Thomas Gleixnercf392d12016-08-12 19:49:39 +02001401 cpuhp_issue_call(cpu, state, false, node);
Thomas Gleixner5b7aa872016-02-26 18:43:33 +00001402 }
1403}
1404
Thomas Gleixnercf392d12016-08-12 19:49:39 +02001405int __cpuhp_state_add_instance(enum cpuhp_state state, struct hlist_node *node,
1406 bool invoke)
1407{
1408 struct cpuhp_step *sp;
1409 int cpu;
1410 int ret;
1411
1412 sp = cpuhp_get_step(state);
1413 if (sp->multi_instance == false)
1414 return -EINVAL;
1415
1416 get_online_cpus();
1417
Thomas Gleixner3c1627e2016-09-05 15:28:36 +02001418 if (!invoke || !sp->startup.multi)
Thomas Gleixnercf392d12016-08-12 19:49:39 +02001419 goto add_node;
1420
1421 /*
1422 * Try to call the startup callback for each present cpu
1423 * depending on the hotplug state of the cpu.
1424 */
1425 for_each_present_cpu(cpu) {
1426 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1427 int cpustate = st->state;
1428
1429 if (cpustate < state)
1430 continue;
1431
1432 ret = cpuhp_issue_call(cpu, state, true, node);
1433 if (ret) {
Thomas Gleixner3c1627e2016-09-05 15:28:36 +02001434 if (sp->teardown.multi)
Thomas Gleixnercf392d12016-08-12 19:49:39 +02001435 cpuhp_rollback_install(cpu, state, node);
1436 goto err;
1437 }
1438 }
1439add_node:
1440 ret = 0;
1441 mutex_lock(&cpuhp_state_mutex);
1442 hlist_add_head(node, &sp->list);
1443 mutex_unlock(&cpuhp_state_mutex);
1444
1445err:
1446 put_online_cpus();
1447 return ret;
1448}
1449EXPORT_SYMBOL_GPL(__cpuhp_state_add_instance);
1450
Thomas Gleixner5b7aa872016-02-26 18:43:33 +00001451/**
1452 * __cpuhp_setup_state - Setup the callbacks for an hotplug machine state
Thomas Gleixnerdc280d932016-12-21 20:19:49 +01001453 * @state: The state to setup
1454 * @invoke: If true, the startup function is invoked for cpus where
1455 * cpu state >= @state
1456 * @startup: startup callback function
1457 * @teardown: teardown callback function
1458 * @multi_instance: State is set up for multiple instances which get
1459 * added afterwards.
Thomas Gleixner5b7aa872016-02-26 18:43:33 +00001460 *
Boris Ostrovsky512f0982016-12-15 10:00:57 -05001461 * Returns:
1462 * On success:
1463 * Positive state number if @state is CPUHP_AP_ONLINE_DYN
1464 * 0 for all other states
1465 * On failure: proper (negative) error code
Thomas Gleixner5b7aa872016-02-26 18:43:33 +00001466 */
1467int __cpuhp_setup_state(enum cpuhp_state state,
1468 const char *name, bool invoke,
1469 int (*startup)(unsigned int cpu),
Thomas Gleixnercf392d12016-08-12 19:49:39 +02001470 int (*teardown)(unsigned int cpu),
1471 bool multi_instance)
Thomas Gleixner5b7aa872016-02-26 18:43:33 +00001472{
1473 int cpu, ret = 0;
Thomas Gleixner5b7aa872016-02-26 18:43:33 +00001474
1475 if (cpuhp_cb_check(state) || !name)
1476 return -EINVAL;
1477
1478 get_online_cpus();
1479
Thomas Gleixnerdc280d932016-12-21 20:19:49 +01001480 ret = cpuhp_store_callbacks(state, name, startup, teardown,
1481 multi_instance);
Thomas Gleixner5b7aa872016-02-26 18:43:33 +00001482
Thomas Gleixnerdc280d932016-12-21 20:19:49 +01001483 if (ret || !invoke || !startup)
Thomas Gleixner5b7aa872016-02-26 18:43:33 +00001484 goto out;
1485
1486 /*
1487 * Try to call the startup callback for each present cpu
1488 * depending on the hotplug state of the cpu.
1489 */
1490 for_each_present_cpu(cpu) {
1491 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1492 int cpustate = st->state;
1493
1494 if (cpustate < state)
1495 continue;
1496
Thomas Gleixnercf392d12016-08-12 19:49:39 +02001497 ret = cpuhp_issue_call(cpu, state, true, NULL);
Thomas Gleixner5b7aa872016-02-26 18:43:33 +00001498 if (ret) {
Thomas Gleixnera7246322016-08-12 19:49:38 +02001499 if (teardown)
Thomas Gleixnercf392d12016-08-12 19:49:39 +02001500 cpuhp_rollback_install(cpu, state, NULL);
1501 cpuhp_store_callbacks(state, NULL, NULL, NULL, false);
Thomas Gleixner5b7aa872016-02-26 18:43:33 +00001502 goto out;
1503 }
1504 }
1505out:
1506 put_online_cpus();
Thomas Gleixnerdc280d932016-12-21 20:19:49 +01001507 /*
1508 * If the requested state is CPUHP_AP_ONLINE_DYN, return the
1509 * dynamically allocated state in case of success.
1510 */
1511 if (!ret && state == CPUHP_AP_ONLINE_DYN)
Thomas Gleixner5b7aa872016-02-26 18:43:33 +00001512 return state;
1513 return ret;
1514}
1515EXPORT_SYMBOL(__cpuhp_setup_state);
1516
Thomas Gleixnercf392d12016-08-12 19:49:39 +02001517int __cpuhp_state_remove_instance(enum cpuhp_state state,
1518 struct hlist_node *node, bool invoke)
1519{
1520 struct cpuhp_step *sp = cpuhp_get_step(state);
1521 int cpu;
1522
1523 BUG_ON(cpuhp_cb_check(state));
1524
1525 if (!sp->multi_instance)
1526 return -EINVAL;
1527
1528 get_online_cpus();
1529 if (!invoke || !cpuhp_get_teardown_cb(state))
1530 goto remove;
1531 /*
1532 * Call the teardown callback for each present cpu depending
1533 * on the hotplug state of the cpu. This function is not
1534 * allowed to fail currently!
1535 */
1536 for_each_present_cpu(cpu) {
1537 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1538 int cpustate = st->state;
1539
1540 if (cpustate >= state)
1541 cpuhp_issue_call(cpu, state, false, node);
1542 }
1543
1544remove:
1545 mutex_lock(&cpuhp_state_mutex);
1546 hlist_del(node);
1547 mutex_unlock(&cpuhp_state_mutex);
1548 put_online_cpus();
1549
1550 return 0;
1551}
1552EXPORT_SYMBOL_GPL(__cpuhp_state_remove_instance);
Thomas Gleixner5b7aa872016-02-26 18:43:33 +00001553/**
1554 * __cpuhp_remove_state - Remove the callbacks for an hotplug machine state
1555 * @state: The state to remove
1556 * @invoke: If true, the teardown function is invoked for cpus where
1557 * cpu state >= @state
1558 *
1559 * The teardown callback is currently not allowed to fail. Think
1560 * about module removal!
1561 */
1562void __cpuhp_remove_state(enum cpuhp_state state, bool invoke)
1563{
Thomas Gleixnercf392d12016-08-12 19:49:39 +02001564 struct cpuhp_step *sp = cpuhp_get_step(state);
Thomas Gleixner5b7aa872016-02-26 18:43:33 +00001565 int cpu;
1566
1567 BUG_ON(cpuhp_cb_check(state));
1568
1569 get_online_cpus();
1570
Thomas Gleixnercf392d12016-08-12 19:49:39 +02001571 if (sp->multi_instance) {
1572 WARN(!hlist_empty(&sp->list),
1573 "Error: Removing state %d which has instances left.\n",
1574 state);
1575 goto remove;
1576 }
1577
Thomas Gleixnera7246322016-08-12 19:49:38 +02001578 if (!invoke || !cpuhp_get_teardown_cb(state))
Thomas Gleixner5b7aa872016-02-26 18:43:33 +00001579 goto remove;
1580
1581 /*
1582 * Call the teardown callback for each present cpu depending
1583 * on the hotplug state of the cpu. This function is not
1584 * allowed to fail currently!
1585 */
1586 for_each_present_cpu(cpu) {
1587 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1588 int cpustate = st->state;
1589
1590 if (cpustate >= state)
Thomas Gleixnercf392d12016-08-12 19:49:39 +02001591 cpuhp_issue_call(cpu, state, false, NULL);
Thomas Gleixner5b7aa872016-02-26 18:43:33 +00001592 }
1593remove:
Thomas Gleixnercf392d12016-08-12 19:49:39 +02001594 cpuhp_store_callbacks(state, NULL, NULL, NULL, false);
Thomas Gleixner5b7aa872016-02-26 18:43:33 +00001595 put_online_cpus();
1596}
1597EXPORT_SYMBOL(__cpuhp_remove_state);
1598
Thomas Gleixner98f8cdc2016-02-26 18:43:31 +00001599#if defined(CONFIG_SYSFS) && defined(CONFIG_HOTPLUG_CPU)
1600static ssize_t show_cpuhp_state(struct device *dev,
1601 struct device_attribute *attr, char *buf)
1602{
1603 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);
1604
1605 return sprintf(buf, "%d\n", st->state);
1606}
1607static DEVICE_ATTR(state, 0444, show_cpuhp_state, NULL);
1608
Thomas Gleixner757c9892016-02-26 18:43:32 +00001609static ssize_t write_cpuhp_target(struct device *dev,
1610 struct device_attribute *attr,
1611 const char *buf, size_t count)
1612{
1613 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);
1614 struct cpuhp_step *sp;
1615 int target, ret;
1616
1617 ret = kstrtoint(buf, 10, &target);
1618 if (ret)
1619 return ret;
1620
1621#ifdef CONFIG_CPU_HOTPLUG_STATE_CONTROL
1622 if (target < CPUHP_OFFLINE || target > CPUHP_ONLINE)
1623 return -EINVAL;
1624#else
1625 if (target != CPUHP_OFFLINE && target != CPUHP_ONLINE)
1626 return -EINVAL;
1627#endif
1628
1629 ret = lock_device_hotplug_sysfs();
1630 if (ret)
1631 return ret;
1632
1633 mutex_lock(&cpuhp_state_mutex);
1634 sp = cpuhp_get_step(target);
1635 ret = !sp->name || sp->cant_stop ? -EINVAL : 0;
1636 mutex_unlock(&cpuhp_state_mutex);
1637 if (ret)
1638 return ret;
1639
1640 if (st->state < target)
1641 ret = do_cpu_up(dev->id, target);
1642 else
1643 ret = do_cpu_down(dev->id, target);
1644
1645 unlock_device_hotplug();
1646 return ret ? ret : count;
1647}
1648
Thomas Gleixner98f8cdc2016-02-26 18:43:31 +00001649static ssize_t show_cpuhp_target(struct device *dev,
1650 struct device_attribute *attr, char *buf)
1651{
1652 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);
1653
1654 return sprintf(buf, "%d\n", st->target);
1655}
Thomas Gleixner757c9892016-02-26 18:43:32 +00001656static DEVICE_ATTR(target, 0644, show_cpuhp_target, write_cpuhp_target);
Thomas Gleixner98f8cdc2016-02-26 18:43:31 +00001657
1658static struct attribute *cpuhp_cpu_attrs[] = {
1659 &dev_attr_state.attr,
1660 &dev_attr_target.attr,
1661 NULL
1662};
1663
1664static struct attribute_group cpuhp_cpu_attr_group = {
1665 .attrs = cpuhp_cpu_attrs,
1666 .name = "hotplug",
1667 NULL
1668};
1669
1670static ssize_t show_cpuhp_states(struct device *dev,
1671 struct device_attribute *attr, char *buf)
1672{
1673 ssize_t cur, res = 0;
1674 int i;
1675
1676 mutex_lock(&cpuhp_state_mutex);
Thomas Gleixner757c9892016-02-26 18:43:32 +00001677 for (i = CPUHP_OFFLINE; i <= CPUHP_ONLINE; i++) {
Thomas Gleixner98f8cdc2016-02-26 18:43:31 +00001678 struct cpuhp_step *sp = cpuhp_get_step(i);
1679
1680 if (sp->name) {
1681 cur = sprintf(buf, "%3d: %s\n", i, sp->name);
1682 buf += cur;
1683 res += cur;
1684 }
1685 }
1686 mutex_unlock(&cpuhp_state_mutex);
1687 return res;
1688}
1689static DEVICE_ATTR(states, 0444, show_cpuhp_states, NULL);
1690
1691static struct attribute *cpuhp_cpu_root_attrs[] = {
1692 &dev_attr_states.attr,
1693 NULL
1694};
1695
1696static struct attribute_group cpuhp_cpu_root_attr_group = {
1697 .attrs = cpuhp_cpu_root_attrs,
1698 .name = "hotplug",
1699 NULL
1700};
1701
1702static int __init cpuhp_sysfs_init(void)
1703{
1704 int cpu, ret;
1705
1706 ret = sysfs_create_group(&cpu_subsys.dev_root->kobj,
1707 &cpuhp_cpu_root_attr_group);
1708 if (ret)
1709 return ret;
1710
1711 for_each_possible_cpu(cpu) {
1712 struct device *dev = get_cpu_device(cpu);
1713
1714 if (!dev)
1715 continue;
1716 ret = sysfs_create_group(&dev->kobj, &cpuhp_cpu_attr_group);
1717 if (ret)
1718 return ret;
1719 }
1720 return 0;
1721}
1722device_initcall(cpuhp_sysfs_init);
1723#endif
1724
Linus Torvaldse56b3bc2008-07-28 11:32:33 -07001725/*
1726 * cpu_bit_bitmap[] is a special, "compressed" data structure that
1727 * represents all NR_CPUS bits binary values of 1<<nr.
1728 *
Rusty Russelle0b582e2009-01-01 10:12:28 +10301729 * It is used by cpumask_of() to get a constant address to a CPU
Linus Torvaldse56b3bc2008-07-28 11:32:33 -07001730 * mask value that has a single bit set only.
1731 */
Mike Travisb8d317d2008-07-24 18:21:29 -07001732
Linus Torvaldse56b3bc2008-07-28 11:32:33 -07001733/* cpu_bit_bitmap[0] is empty - so we can back into it */
Michael Rodriguez4d519852011-03-22 16:34:07 -07001734#define MASK_DECLARE_1(x) [x+1][0] = (1UL << (x))
Linus Torvaldse56b3bc2008-07-28 11:32:33 -07001735#define MASK_DECLARE_2(x) MASK_DECLARE_1(x), MASK_DECLARE_1(x+1)
1736#define MASK_DECLARE_4(x) MASK_DECLARE_2(x), MASK_DECLARE_2(x+2)
1737#define MASK_DECLARE_8(x) MASK_DECLARE_4(x), MASK_DECLARE_4(x+4)
Mike Travisb8d317d2008-07-24 18:21:29 -07001738
Linus Torvaldse56b3bc2008-07-28 11:32:33 -07001739const unsigned long cpu_bit_bitmap[BITS_PER_LONG+1][BITS_TO_LONGS(NR_CPUS)] = {
Mike Travisb8d317d2008-07-24 18:21:29 -07001740
Linus Torvaldse56b3bc2008-07-28 11:32:33 -07001741 MASK_DECLARE_8(0), MASK_DECLARE_8(8),
1742 MASK_DECLARE_8(16), MASK_DECLARE_8(24),
1743#if BITS_PER_LONG > 32
1744 MASK_DECLARE_8(32), MASK_DECLARE_8(40),
1745 MASK_DECLARE_8(48), MASK_DECLARE_8(56),
Mike Travisb8d317d2008-07-24 18:21:29 -07001746#endif
1747};
Linus Torvaldse56b3bc2008-07-28 11:32:33 -07001748EXPORT_SYMBOL_GPL(cpu_bit_bitmap);
Rusty Russell2d3854a2008-11-05 13:39:10 +11001749
1750const DECLARE_BITMAP(cpu_all_bits, NR_CPUS) = CPU_BITS_ALL;
1751EXPORT_SYMBOL(cpu_all_bits);
Rusty Russellb3199c02008-12-30 09:05:14 +10301752
1753#ifdef CONFIG_INIT_ALL_POSSIBLE
Rasmus Villemoes4b804c82016-01-20 15:00:19 -08001754struct cpumask __cpu_possible_mask __read_mostly
Rasmus Villemoesc4c54dd2016-01-20 15:00:16 -08001755 = {CPU_BITS_ALL};
Rusty Russellb3199c02008-12-30 09:05:14 +10301756#else
Rasmus Villemoes4b804c82016-01-20 15:00:19 -08001757struct cpumask __cpu_possible_mask __read_mostly;
Rusty Russellb3199c02008-12-30 09:05:14 +10301758#endif
Rasmus Villemoes4b804c82016-01-20 15:00:19 -08001759EXPORT_SYMBOL(__cpu_possible_mask);
Rusty Russellb3199c02008-12-30 09:05:14 +10301760
Rasmus Villemoes4b804c82016-01-20 15:00:19 -08001761struct cpumask __cpu_online_mask __read_mostly;
1762EXPORT_SYMBOL(__cpu_online_mask);
Rusty Russellb3199c02008-12-30 09:05:14 +10301763
Rasmus Villemoes4b804c82016-01-20 15:00:19 -08001764struct cpumask __cpu_present_mask __read_mostly;
1765EXPORT_SYMBOL(__cpu_present_mask);
Rusty Russellb3199c02008-12-30 09:05:14 +10301766
Rasmus Villemoes4b804c82016-01-20 15:00:19 -08001767struct cpumask __cpu_active_mask __read_mostly;
1768EXPORT_SYMBOL(__cpu_active_mask);
Rusty Russell3fa41522008-12-30 09:05:16 +10301769
Rusty Russell3fa41522008-12-30 09:05:16 +10301770void init_cpu_present(const struct cpumask *src)
1771{
Rasmus Villemoesc4c54dd2016-01-20 15:00:16 -08001772 cpumask_copy(&__cpu_present_mask, src);
Rusty Russell3fa41522008-12-30 09:05:16 +10301773}
1774
1775void init_cpu_possible(const struct cpumask *src)
1776{
Rasmus Villemoesc4c54dd2016-01-20 15:00:16 -08001777 cpumask_copy(&__cpu_possible_mask, src);
Rusty Russell3fa41522008-12-30 09:05:16 +10301778}
1779
1780void init_cpu_online(const struct cpumask *src)
1781{
Rasmus Villemoesc4c54dd2016-01-20 15:00:16 -08001782 cpumask_copy(&__cpu_online_mask, src);
Rusty Russell3fa41522008-12-30 09:05:16 +10301783}
Thomas Gleixnercff7d372016-02-26 18:43:28 +00001784
1785/*
1786 * Activate the first processor.
1787 */
1788void __init boot_cpu_init(void)
1789{
1790 int cpu = smp_processor_id();
1791
1792 /* Mark the boot cpu "present", "online" etc for SMP and UP case */
1793 set_cpu_online(cpu, true);
1794 set_cpu_active(cpu, true);
1795 set_cpu_present(cpu, true);
1796 set_cpu_possible(cpu, true);
1797}
1798
1799/*
1800 * Must be called _AFTER_ setting up the per_cpu areas
1801 */
1802void __init boot_cpu_state_init(void)
1803{
1804 per_cpu_ptr(&cpuhp_state, smp_processor_id())->state = CPUHP_ONLINE;
1805}