blob: eca92aac2e175f3c4150620d148f957a78ad21da [file] [log] [blame]
Paul Mackerras14cf11a2005-09-26 16:04:21 +10001/*
Paul Mackerras14cf11a2005-09-26 16:04:21 +10002 * Derived from "arch/i386/kernel/process.c"
3 * Copyright (C) 1995 Linus Torvalds
4 *
5 * Updated and modified by Cort Dougan (cort@cs.nmt.edu) and
6 * Paul Mackerras (paulus@cs.anu.edu.au)
7 *
8 * PowerPC version
9 * Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
10 *
11 * This program is free software; you can redistribute it and/or
12 * modify it under the terms of the GNU General Public License
13 * as published by the Free Software Foundation; either version
14 * 2 of the License, or (at your option) any later version.
15 */
16
Paul Mackerras14cf11a2005-09-26 16:04:21 +100017#include <linux/errno.h>
18#include <linux/sched.h>
Ingo Molnarb17b0152017-02-08 18:51:35 +010019#include <linux/sched/debug.h>
Ingo Molnar29930022017-02-08 18:51:36 +010020#include <linux/sched/task.h>
Ingo Molnar68db0cf2017-02-08 18:51:37 +010021#include <linux/sched/task_stack.h>
Paul Mackerras14cf11a2005-09-26 16:04:21 +100022#include <linux/kernel.h>
23#include <linux/mm.h>
24#include <linux/smp.h>
Paul Mackerras14cf11a2005-09-26 16:04:21 +100025#include <linux/stddef.h>
26#include <linux/unistd.h>
27#include <linux/ptrace.h>
28#include <linux/slab.h>
29#include <linux/user.h>
30#include <linux/elf.h>
Paul Mackerras14cf11a2005-09-26 16:04:21 +100031#include <linux/prctl.h>
32#include <linux/init_task.h>
Paul Gortmaker4b16f8e2011-07-22 18:24:23 -040033#include <linux/export.h>
Paul Mackerras14cf11a2005-09-26 16:04:21 +100034#include <linux/kallsyms.h>
35#include <linux/mqueue.h>
36#include <linux/hardirq.h>
Paul Mackerras06d67d52005-10-10 22:29:05 +100037#include <linux/utsname.h>
Steven Rostedt6794c782009-02-09 21:10:27 -080038#include <linux/ftrace.h>
Martin Schwidefsky79741dd2008-12-31 15:11:38 +010039#include <linux/kernel_stat.h>
Anton Blanchardd8390882009-02-22 01:50:03 +000040#include <linux/personality.h>
41#include <linux/random.h>
K.Prasad5aae8a52010-06-15 11:35:19 +053042#include <linux/hw_breakpoint.h>
Anton Blanchard7b051f62014-10-13 20:27:15 +110043#include <linux/uaccess.h>
Daniel Axtens7f92bc52016-01-06 11:45:51 +110044#include <linux/elf-randomize.h>
Ram Pai06bb53b2018-01-18 17:50:31 -080045#include <linux/pkeys.h>
Paul Mackerras14cf11a2005-09-26 16:04:21 +100046
47#include <asm/pgtable.h>
Paul Mackerras14cf11a2005-09-26 16:04:21 +100048#include <asm/io.h>
49#include <asm/processor.h>
50#include <asm/mmu.h>
51#include <asm/prom.h>
Michael Ellerman76032de2005-11-07 13:12:03 +110052#include <asm/machdep.h>
Paul Mackerrasc6622f62006-02-24 10:06:59 +110053#include <asm/time.h>
David Howellsae3a1972012-03-28 18:30:02 +010054#include <asm/runlatch.h>
Arnd Bergmanna7f31842006-03-23 00:00:08 +010055#include <asm/syscalls.h>
David Howellsae3a1972012-03-28 18:30:02 +010056#include <asm/switch_to.h>
Michael Neulingfb096922013-02-13 16:21:37 +000057#include <asm/tm.h>
David Howellsae3a1972012-03-28 18:30:02 +010058#include <asm/debug.h>
Paul Mackerras06d67d52005-10-10 22:29:05 +100059#ifdef CONFIG_PPC64
60#include <asm/firmware.h>
Madhavan Srinivasanc2e480b2017-12-20 09:25:42 +053061#include <asm/hw_irq.h>
Paul Mackerras06d67d52005-10-10 22:29:05 +100062#endif
Anton Blanchard7cedd602014-02-04 16:08:51 +110063#include <asm/code-patching.h>
Daniel Axtens7f92bc52016-01-06 11:45:51 +110064#include <asm/exec.h>
Michael Ellerman5d31a962016-03-24 22:04:04 +110065#include <asm/livepatch.h>
Kevin Haob92a2262016-07-23 14:42:40 +053066#include <asm/cpu_has_feature.h>
Daniel Axtens0545d542016-09-06 15:32:43 +100067#include <asm/asm-prototypes.h>
Michael Ellerman5d31a962016-03-24 22:04:04 +110068
Luis Machadod6a61bf2008-07-24 02:10:41 +100069#include <linux/kprobes.h>
70#include <linux/kdebug.h>
Paul Mackerras14cf11a2005-09-26 16:04:21 +100071
Michael Neuling8b3c34c2013-02-13 16:21:32 +000072/* Transactional Memory debug */
73#ifdef TM_DEBUG_SW
74#define TM_DEBUG(x...) printk(KERN_INFO x)
75#else
76#define TM_DEBUG(x...) do { } while(0)
77#endif
78
Paul Mackerras14cf11a2005-09-26 16:04:21 +100079extern unsigned long _get_SP(void);
80
Paul Mackerrasd31626f2014-01-13 15:56:29 +110081#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
Michael Ellerman54820532017-10-12 21:17:18 +110082/*
83 * Are we running in "Suspend disabled" mode? If so we have to block any
84 * sigreturn that would get us into suspended state, and we also warn in some
85 * other paths that we should never reach with suspend disabled.
86 */
87bool tm_suspend_disabled __ro_after_init = false;
88
Anton Blanchardb86fd2b2015-10-29 11:43:58 +110089static void check_if_tm_restore_required(struct task_struct *tsk)
Paul Mackerrasd31626f2014-01-13 15:56:29 +110090{
91 /*
92 * If we are saving the current thread's registers, and the
93 * thread is in a transactional state, set the TIF_RESTORE_TM
94 * bit so that we know to restore the registers before
95 * returning to userspace.
96 */
97 if (tsk == current && tsk->thread.regs &&
98 MSR_TM_ACTIVE(tsk->thread.regs->msr) &&
99 !test_thread_flag(TIF_RESTORE_TM)) {
Anshuman Khandual829023d2015-07-06 16:24:10 +0530100 tsk->thread.ckpt_regs.msr = tsk->thread.regs->msr;
Paul Mackerrasd31626f2014-01-13 15:56:29 +1100101 set_thread_flag(TIF_RESTORE_TM);
102 }
Paul Mackerrasd31626f2014-01-13 15:56:29 +1100103}
Cyril Burdc16b552016-09-23 16:18:08 +1000104
105static inline bool msr_tm_active(unsigned long msr)
106{
107 return MSR_TM_ACTIVE(msr);
108}
Cyril Bura7771172017-11-02 14:09:03 +1100109
110static bool tm_active_with_fp(struct task_struct *tsk)
111{
112 return msr_tm_active(tsk->thread.regs->msr) &&
113 (tsk->thread.ckpt_regs.msr & MSR_FP);
114}
115
116static bool tm_active_with_altivec(struct task_struct *tsk)
117{
118 return msr_tm_active(tsk->thread.regs->msr) &&
119 (tsk->thread.ckpt_regs.msr & MSR_VEC);
120}
Paul Mackerrasd31626f2014-01-13 15:56:29 +1100121#else
Cyril Burdc16b552016-09-23 16:18:08 +1000122static inline bool msr_tm_active(unsigned long msr) { return false; }
Anton Blanchardb86fd2b2015-10-29 11:43:58 +1100123static inline void check_if_tm_restore_required(struct task_struct *tsk) { }
Cyril Bura7771172017-11-02 14:09:03 +1100124static inline bool tm_active_with_fp(struct task_struct *tsk) { return false; }
125static inline bool tm_active_with_altivec(struct task_struct *tsk) { return false; }
Paul Mackerrasd31626f2014-01-13 15:56:29 +1100126#endif /* CONFIG_PPC_TRANSACTIONAL_MEM */
127
Anton Blanchard3eb5d582015-10-29 11:44:06 +1100128bool strict_msr_control;
129EXPORT_SYMBOL(strict_msr_control);
130
131static int __init enable_strict_msr_control(char *str)
132{
133 strict_msr_control = true;
134 pr_info("Enabling strict facility control\n");
135
136 return 0;
137}
138early_param("ppc_strict_facility_enable", enable_strict_msr_control);
139
Cyril Bur3cee0702016-09-23 16:18:10 +1000140unsigned long msr_check_and_set(unsigned long bits)
Anton Blancharda0e72cf2015-10-29 11:44:04 +1100141{
142 unsigned long oldmsr = mfmsr();
143 unsigned long newmsr;
144
145 newmsr = oldmsr | bits;
146
147#ifdef CONFIG_VSX
148 if (cpu_has_feature(CPU_FTR_VSX) && (bits & MSR_FP))
149 newmsr |= MSR_VSX;
150#endif
151
152 if (oldmsr != newmsr)
153 mtmsr_isync(newmsr);
Cyril Bur3cee0702016-09-23 16:18:10 +1000154
155 return newmsr;
Anton Blancharda0e72cf2015-10-29 11:44:04 +1100156}
Simon Guod1c72112018-05-23 15:01:44 +0800157EXPORT_SYMBOL_GPL(msr_check_and_set);
Anton Blancharda0e72cf2015-10-29 11:44:04 +1100158
Anton Blanchard3eb5d582015-10-29 11:44:06 +1100159void __msr_check_and_clear(unsigned long bits)
Anton Blancharda0e72cf2015-10-29 11:44:04 +1100160{
161 unsigned long oldmsr = mfmsr();
162 unsigned long newmsr;
163
164 newmsr = oldmsr & ~bits;
165
166#ifdef CONFIG_VSX
167 if (cpu_has_feature(CPU_FTR_VSX) && (bits & MSR_FP))
168 newmsr &= ~MSR_VSX;
169#endif
170
171 if (oldmsr != newmsr)
172 mtmsr_isync(newmsr);
173}
Anton Blanchard3eb5d582015-10-29 11:44:06 +1100174EXPORT_SYMBOL(__msr_check_and_clear);
Anton Blancharda0e72cf2015-10-29 11:44:04 +1100175
Kevin Hao037f0ee2013-07-14 17:02:05 +0800176#ifdef CONFIG_PPC_FPU
Mathieu Malaterre1cdf0392018-02-25 18:22:23 +0100177static void __giveup_fpu(struct task_struct *tsk)
Cyril Bur87924682016-02-29 17:53:49 +1100178{
Anton Blanchard8eb98032016-05-29 22:03:50 +1000179 unsigned long msr;
180
Cyril Bur87924682016-02-29 17:53:49 +1100181 save_fpu(tsk);
Anton Blanchard8eb98032016-05-29 22:03:50 +1000182 msr = tsk->thread.regs->msr;
183 msr &= ~MSR_FP;
Cyril Bur87924682016-02-29 17:53:49 +1100184#ifdef CONFIG_VSX
185 if (cpu_has_feature(CPU_FTR_VSX))
Anton Blanchard8eb98032016-05-29 22:03:50 +1000186 msr &= ~MSR_VSX;
Cyril Bur87924682016-02-29 17:53:49 +1100187#endif
Anton Blanchard8eb98032016-05-29 22:03:50 +1000188 tsk->thread.regs->msr = msr;
Cyril Bur87924682016-02-29 17:53:49 +1100189}
190
Anton Blanchard98da5812015-10-29 11:44:01 +1100191void giveup_fpu(struct task_struct *tsk)
192{
Anton Blanchard98da5812015-10-29 11:44:01 +1100193 check_if_tm_restore_required(tsk);
194
Anton Blancharda0e72cf2015-10-29 11:44:04 +1100195 msr_check_and_set(MSR_FP);
Anton Blanchard98da5812015-10-29 11:44:01 +1100196 __giveup_fpu(tsk);
Anton Blancharda0e72cf2015-10-29 11:44:04 +1100197 msr_check_and_clear(MSR_FP);
Anton Blanchard98da5812015-10-29 11:44:01 +1100198}
199EXPORT_SYMBOL(giveup_fpu);
200
Paul Mackerras14cf11a2005-09-26 16:04:21 +1000201/*
202 * Make sure the floating-point register state in the
203 * the thread_struct is up to date for task tsk.
204 */
205void flush_fp_to_thread(struct task_struct *tsk)
206{
207 if (tsk->thread.regs) {
208 /*
209 * We need to disable preemption here because if we didn't,
210 * another process could get scheduled after the regs->msr
211 * test but before we have finished saving the FP registers
212 * to the thread_struct. That process could take over the
213 * FPU, and then when we get scheduled again we would store
214 * bogus values for the remaining FP registers.
215 */
216 preempt_disable();
217 if (tsk->thread.regs->msr & MSR_FP) {
Paul Mackerras14cf11a2005-09-26 16:04:21 +1000218 /*
219 * This should only ever be called for current or
220 * for a stopped child process. Since we save away
Anton Blanchardaf1bbc32015-10-29 11:43:57 +1100221 * the FP register state on context switch,
Paul Mackerras14cf11a2005-09-26 16:04:21 +1000222 * there is something wrong if a stopped child appears
223 * to still have its FP state in the CPU registers.
224 */
225 BUG_ON(tsk != current);
Anton Blanchardb86fd2b2015-10-29 11:43:58 +1100226 giveup_fpu(tsk);
Paul Mackerras14cf11a2005-09-26 16:04:21 +1000227 }
228 preempt_enable();
229 }
230}
Paul Mackerrasde56a942011-06-29 00:21:34 +0000231EXPORT_SYMBOL_GPL(flush_fp_to_thread);
Paul Mackerras14cf11a2005-09-26 16:04:21 +1000232
233void enable_kernel_fp(void)
234{
Cyril Bure909fb82016-09-23 16:18:11 +1000235 unsigned long cpumsr;
236
Paul Mackerras14cf11a2005-09-26 16:04:21 +1000237 WARN_ON(preemptible());
238
Cyril Bure909fb82016-09-23 16:18:11 +1000239 cpumsr = msr_check_and_set(MSR_FP);
Anton Blanchard611b0e52015-10-29 11:43:59 +1100240
Anton Blanchardd64d02c2015-12-10 20:04:05 +1100241 if (current->thread.regs && (current->thread.regs->msr & MSR_FP)) {
242 check_if_tm_restore_required(current);
Cyril Bure909fb82016-09-23 16:18:11 +1000243 /*
244 * If a thread has already been reclaimed then the
245 * checkpointed registers are on the CPU but have definitely
246 * been saved by the reclaim code. Don't need to and *cannot*
247 * giveup as this would save to the 'live' structure not the
248 * checkpointed structure.
249 */
250 if(!msr_tm_active(cpumsr) && msr_tm_active(current->thread.regs->msr))
251 return;
Anton Blancharda0e72cf2015-10-29 11:44:04 +1100252 __giveup_fpu(current);
Anton Blanchardd64d02c2015-12-10 20:04:05 +1100253 }
Paul Mackerras14cf11a2005-09-26 16:04:21 +1000254}
255EXPORT_SYMBOL(enable_kernel_fp);
Cyril Bur70fe3d92016-02-29 17:53:47 +1100256
Benjamin Herrenschmidt6a303832017-08-16 16:01:15 +1000257static int restore_fp(struct task_struct *tsk)
258{
Cyril Bura7771172017-11-02 14:09:03 +1100259 if (tsk->thread.load_fp || tm_active_with_fp(tsk)) {
Cyril Bur70fe3d92016-02-29 17:53:47 +1100260 load_fp_state(&current->thread.fp_state);
261 current->thread.load_fp++;
262 return 1;
263 }
264 return 0;
265}
266#else
267static int restore_fp(struct task_struct *tsk) { return 0; }
Anton Blanchardd1e1cf22015-10-29 11:44:11 +1100268#endif /* CONFIG_PPC_FPU */
Paul Mackerras14cf11a2005-09-26 16:04:21 +1000269
Paul Mackerras14cf11a2005-09-26 16:04:21 +1000270#ifdef CONFIG_ALTIVEC
Cyril Bur70fe3d92016-02-29 17:53:47 +1100271#define loadvec(thr) ((thr).load_vec)
272
Cyril Bur6f515d82016-02-29 17:53:50 +1100273static void __giveup_altivec(struct task_struct *tsk)
274{
Anton Blanchard8eb98032016-05-29 22:03:50 +1000275 unsigned long msr;
276
Cyril Bur6f515d82016-02-29 17:53:50 +1100277 save_altivec(tsk);
Anton Blanchard8eb98032016-05-29 22:03:50 +1000278 msr = tsk->thread.regs->msr;
279 msr &= ~MSR_VEC;
Cyril Bur6f515d82016-02-29 17:53:50 +1100280#ifdef CONFIG_VSX
281 if (cpu_has_feature(CPU_FTR_VSX))
Anton Blanchard8eb98032016-05-29 22:03:50 +1000282 msr &= ~MSR_VSX;
Cyril Bur6f515d82016-02-29 17:53:50 +1100283#endif
Anton Blanchard8eb98032016-05-29 22:03:50 +1000284 tsk->thread.regs->msr = msr;
Cyril Bur6f515d82016-02-29 17:53:50 +1100285}
286
Anton Blanchard98da5812015-10-29 11:44:01 +1100287void giveup_altivec(struct task_struct *tsk)
288{
Anton Blanchard98da5812015-10-29 11:44:01 +1100289 check_if_tm_restore_required(tsk);
290
Anton Blancharda0e72cf2015-10-29 11:44:04 +1100291 msr_check_and_set(MSR_VEC);
Anton Blanchard98da5812015-10-29 11:44:01 +1100292 __giveup_altivec(tsk);
Anton Blancharda0e72cf2015-10-29 11:44:04 +1100293 msr_check_and_clear(MSR_VEC);
Anton Blanchard98da5812015-10-29 11:44:01 +1100294}
295EXPORT_SYMBOL(giveup_altivec);
296
Paul Mackerras14cf11a2005-09-26 16:04:21 +1000297void enable_kernel_altivec(void)
298{
Cyril Bure909fb82016-09-23 16:18:11 +1000299 unsigned long cpumsr;
300
Paul Mackerras14cf11a2005-09-26 16:04:21 +1000301 WARN_ON(preemptible());
302
Cyril Bure909fb82016-09-23 16:18:11 +1000303 cpumsr = msr_check_and_set(MSR_VEC);
Anton Blanchard611b0e52015-10-29 11:43:59 +1100304
Anton Blanchardd64d02c2015-12-10 20:04:05 +1100305 if (current->thread.regs && (current->thread.regs->msr & MSR_VEC)) {
306 check_if_tm_restore_required(current);
Cyril Bure909fb82016-09-23 16:18:11 +1000307 /*
308 * If a thread has already been reclaimed then the
309 * checkpointed registers are on the CPU but have definitely
310 * been saved by the reclaim code. Don't need to and *cannot*
311 * giveup as this would save to the 'live' structure not the
312 * checkpointed structure.
313 */
314 if(!msr_tm_active(cpumsr) && msr_tm_active(current->thread.regs->msr))
315 return;
Anton Blancharda0e72cf2015-10-29 11:44:04 +1100316 __giveup_altivec(current);
Anton Blanchardd64d02c2015-12-10 20:04:05 +1100317 }
Paul Mackerras14cf11a2005-09-26 16:04:21 +1000318}
319EXPORT_SYMBOL(enable_kernel_altivec);
320
321/*
322 * Make sure the VMX/Altivec register state in the
323 * the thread_struct is up to date for task tsk.
324 */
325void flush_altivec_to_thread(struct task_struct *tsk)
326{
327 if (tsk->thread.regs) {
328 preempt_disable();
329 if (tsk->thread.regs->msr & MSR_VEC) {
Paul Mackerras14cf11a2005-09-26 16:04:21 +1000330 BUG_ON(tsk != current);
Anton Blanchardb86fd2b2015-10-29 11:43:58 +1100331 giveup_altivec(tsk);
Paul Mackerras14cf11a2005-09-26 16:04:21 +1000332 }
333 preempt_enable();
334 }
335}
Paul Mackerrasde56a942011-06-29 00:21:34 +0000336EXPORT_SYMBOL_GPL(flush_altivec_to_thread);
Cyril Bur70fe3d92016-02-29 17:53:47 +1100337
338static int restore_altivec(struct task_struct *tsk)
339{
Cyril Burdc16b552016-09-23 16:18:08 +1000340 if (cpu_has_feature(CPU_FTR_ALTIVEC) &&
Cyril Bura7771172017-11-02 14:09:03 +1100341 (tsk->thread.load_vec || tm_active_with_altivec(tsk))) {
Cyril Bur70fe3d92016-02-29 17:53:47 +1100342 load_vr_state(&tsk->thread.vr_state);
343 tsk->thread.used_vr = 1;
344 tsk->thread.load_vec++;
345
346 return 1;
347 }
348 return 0;
349}
350#else
351#define loadvec(thr) 0
352static inline int restore_altivec(struct task_struct *tsk) { return 0; }
Paul Mackerras14cf11a2005-09-26 16:04:21 +1000353#endif /* CONFIG_ALTIVEC */
354
Michael Neulingce48b212008-06-25 14:07:18 +1000355#ifdef CONFIG_VSX
Cyril Burbf6a4d52016-02-29 17:53:51 +1100356static void __giveup_vsx(struct task_struct *tsk)
Anton Blancharda7d623d2015-10-29 11:44:02 +1100357{
Benjamin Herrenschmidtdc801082017-08-16 16:01:17 +1000358 unsigned long msr = tsk->thread.regs->msr;
359
360 /*
361 * We should never be ssetting MSR_VSX without also setting
362 * MSR_FP and MSR_VEC
363 */
364 WARN_ON((msr & MSR_VSX) && !((msr & MSR_FP) && (msr & MSR_VEC)));
365
366 /* __giveup_fpu will clear MSR_VSX */
367 if (msr & MSR_FP)
Anton Blancharda7d623d2015-10-29 11:44:02 +1100368 __giveup_fpu(tsk);
Benjamin Herrenschmidtdc801082017-08-16 16:01:17 +1000369 if (msr & MSR_VEC)
Anton Blancharda7d623d2015-10-29 11:44:02 +1100370 __giveup_altivec(tsk);
Cyril Burbf6a4d52016-02-29 17:53:51 +1100371}
372
373static void giveup_vsx(struct task_struct *tsk)
374{
375 check_if_tm_restore_required(tsk);
376
377 msr_check_and_set(MSR_FP|MSR_VEC|MSR_VSX);
Anton Blancharda7d623d2015-10-29 11:44:02 +1100378 __giveup_vsx(tsk);
Anton Blancharda0e72cf2015-10-29 11:44:04 +1100379 msr_check_and_clear(MSR_FP|MSR_VEC|MSR_VSX);
Anton Blancharda7d623d2015-10-29 11:44:02 +1100380}
Cyril Burbf6a4d52016-02-29 17:53:51 +1100381
Michael Neulingce48b212008-06-25 14:07:18 +1000382void enable_kernel_vsx(void)
383{
Cyril Bure909fb82016-09-23 16:18:11 +1000384 unsigned long cpumsr;
385
Michael Neulingce48b212008-06-25 14:07:18 +1000386 WARN_ON(preemptible());
387
Cyril Bure909fb82016-09-23 16:18:11 +1000388 cpumsr = msr_check_and_set(MSR_FP|MSR_VEC|MSR_VSX);
Anton Blanchard611b0e52015-10-29 11:43:59 +1100389
Benjamin Herrenschmidt5a69aec2017-08-16 16:01:14 +1000390 if (current->thread.regs &&
391 (current->thread.regs->msr & (MSR_VSX|MSR_VEC|MSR_FP))) {
Anton Blanchardd64d02c2015-12-10 20:04:05 +1100392 check_if_tm_restore_required(current);
Cyril Bure909fb82016-09-23 16:18:11 +1000393 /*
394 * If a thread has already been reclaimed then the
395 * checkpointed registers are on the CPU but have definitely
396 * been saved by the reclaim code. Don't need to and *cannot*
397 * giveup as this would save to the 'live' structure not the
398 * checkpointed structure.
399 */
400 if(!msr_tm_active(cpumsr) && msr_tm_active(current->thread.regs->msr))
401 return;
Anton Blancharda0e72cf2015-10-29 11:44:04 +1100402 __giveup_vsx(current);
Anton Blanchard611b0e52015-10-29 11:43:59 +1100403 }
Michael Neulingce48b212008-06-25 14:07:18 +1000404}
405EXPORT_SYMBOL(enable_kernel_vsx);
Michael Neulingce48b212008-06-25 14:07:18 +1000406
407void flush_vsx_to_thread(struct task_struct *tsk)
408{
409 if (tsk->thread.regs) {
410 preempt_disable();
Benjamin Herrenschmidt5a69aec2017-08-16 16:01:14 +1000411 if (tsk->thread.regs->msr & (MSR_VSX|MSR_VEC|MSR_FP)) {
Michael Neulingce48b212008-06-25 14:07:18 +1000412 BUG_ON(tsk != current);
Michael Neulingce48b212008-06-25 14:07:18 +1000413 giveup_vsx(tsk);
414 }
415 preempt_enable();
416 }
417}
Paul Mackerrasde56a942011-06-29 00:21:34 +0000418EXPORT_SYMBOL_GPL(flush_vsx_to_thread);
Cyril Bur70fe3d92016-02-29 17:53:47 +1100419
420static int restore_vsx(struct task_struct *tsk)
421{
422 if (cpu_has_feature(CPU_FTR_VSX)) {
423 tsk->thread.used_vsr = 1;
424 return 1;
425 }
426
427 return 0;
428}
429#else
430static inline int restore_vsx(struct task_struct *tsk) { return 0; }
Michael Neulingce48b212008-06-25 14:07:18 +1000431#endif /* CONFIG_VSX */
432
Paul Mackerras14cf11a2005-09-26 16:04:21 +1000433#ifdef CONFIG_SPE
Anton Blanchard98da5812015-10-29 11:44:01 +1100434void giveup_spe(struct task_struct *tsk)
435{
Anton Blanchard98da5812015-10-29 11:44:01 +1100436 check_if_tm_restore_required(tsk);
437
Anton Blancharda0e72cf2015-10-29 11:44:04 +1100438 msr_check_and_set(MSR_SPE);
Anton Blanchard98da5812015-10-29 11:44:01 +1100439 __giveup_spe(tsk);
Anton Blancharda0e72cf2015-10-29 11:44:04 +1100440 msr_check_and_clear(MSR_SPE);
Anton Blanchard98da5812015-10-29 11:44:01 +1100441}
442EXPORT_SYMBOL(giveup_spe);
Paul Mackerras14cf11a2005-09-26 16:04:21 +1000443
444void enable_kernel_spe(void)
445{
446 WARN_ON(preemptible());
447
Anton Blancharda0e72cf2015-10-29 11:44:04 +1100448 msr_check_and_set(MSR_SPE);
Anton Blanchard611b0e52015-10-29 11:43:59 +1100449
Anton Blanchardd64d02c2015-12-10 20:04:05 +1100450 if (current->thread.regs && (current->thread.regs->msr & MSR_SPE)) {
451 check_if_tm_restore_required(current);
Anton Blancharda0e72cf2015-10-29 11:44:04 +1100452 __giveup_spe(current);
Anton Blanchardd64d02c2015-12-10 20:04:05 +1100453 }
Paul Mackerras14cf11a2005-09-26 16:04:21 +1000454}
455EXPORT_SYMBOL(enable_kernel_spe);
456
457void flush_spe_to_thread(struct task_struct *tsk)
458{
459 if (tsk->thread.regs) {
460 preempt_disable();
461 if (tsk->thread.regs->msr & MSR_SPE) {
Paul Mackerras14cf11a2005-09-26 16:04:21 +1000462 BUG_ON(tsk != current);
yu liu685659e2011-06-14 18:34:25 -0500463 tsk->thread.spefscr = mfspr(SPRN_SPEFSCR);
Kumar Gala0ee6c152007-08-28 21:15:53 -0500464 giveup_spe(tsk);
Paul Mackerras14cf11a2005-09-26 16:04:21 +1000465 }
466 preempt_enable();
467 }
468}
Paul Mackerras14cf11a2005-09-26 16:04:21 +1000469#endif /* CONFIG_SPE */
470
Anton Blanchardc2085052015-10-29 11:44:08 +1100471static unsigned long msr_all_available;
472
473static int __init init_msr_all_available(void)
474{
475#ifdef CONFIG_PPC_FPU
476 msr_all_available |= MSR_FP;
477#endif
478#ifdef CONFIG_ALTIVEC
479 if (cpu_has_feature(CPU_FTR_ALTIVEC))
480 msr_all_available |= MSR_VEC;
481#endif
482#ifdef CONFIG_VSX
483 if (cpu_has_feature(CPU_FTR_VSX))
484 msr_all_available |= MSR_VSX;
485#endif
486#ifdef CONFIG_SPE
487 if (cpu_has_feature(CPU_FTR_SPE))
488 msr_all_available |= MSR_SPE;
489#endif
490
491 return 0;
492}
493early_initcall(init_msr_all_available);
494
495void giveup_all(struct task_struct *tsk)
496{
497 unsigned long usermsr;
498
499 if (!tsk->thread.regs)
500 return;
501
502 usermsr = tsk->thread.regs->msr;
503
504 if ((usermsr & msr_all_available) == 0)
505 return;
506
507 msr_check_and_set(msr_all_available);
Cyril Burb0f16b42016-09-23 16:18:09 +1000508 check_if_tm_restore_required(tsk);
Anton Blanchardc2085052015-10-29 11:44:08 +1100509
Benjamin Herrenschmidt96c79b62017-08-16 16:01:18 +1000510 WARN_ON((usermsr & MSR_VSX) && !((usermsr & MSR_FP) && (usermsr & MSR_VEC)));
511
Anton Blanchardc2085052015-10-29 11:44:08 +1100512#ifdef CONFIG_PPC_FPU
513 if (usermsr & MSR_FP)
514 __giveup_fpu(tsk);
515#endif
516#ifdef CONFIG_ALTIVEC
517 if (usermsr & MSR_VEC)
518 __giveup_altivec(tsk);
519#endif
Anton Blanchardc2085052015-10-29 11:44:08 +1100520#ifdef CONFIG_SPE
521 if (usermsr & MSR_SPE)
522 __giveup_spe(tsk);
523#endif
524
525 msr_check_and_clear(msr_all_available);
526}
527EXPORT_SYMBOL(giveup_all);
528
Cyril Bur70fe3d92016-02-29 17:53:47 +1100529void restore_math(struct pt_regs *regs)
530{
531 unsigned long msr;
532
Cyril Burdc16b552016-09-23 16:18:08 +1000533 if (!msr_tm_active(regs->msr) &&
534 !current->thread.load_fp && !loadvec(current->thread))
Cyril Bur70fe3d92016-02-29 17:53:47 +1100535 return;
536
537 msr = regs->msr;
538 msr_check_and_set(msr_all_available);
539
540 /*
541 * Only reload if the bit is not set in the user MSR, the bit BEING set
542 * indicates that the registers are hot
543 */
544 if ((!(msr & MSR_FP)) && restore_fp(current))
545 msr |= MSR_FP | current->thread.fpexc_mode;
546
547 if ((!(msr & MSR_VEC)) && restore_altivec(current))
548 msr |= MSR_VEC;
549
550 if ((msr & (MSR_FP | MSR_VEC)) == (MSR_FP | MSR_VEC) &&
551 restore_vsx(current)) {
552 msr |= MSR_VSX;
553 }
554
555 msr_check_and_clear(msr_all_available);
556
557 regs->msr = msr;
558}
559
Mathieu Malaterre1cdf0392018-02-25 18:22:23 +0100560static void save_all(struct task_struct *tsk)
Cyril Burde2a20a2016-02-29 17:53:48 +1100561{
562 unsigned long usermsr;
563
564 if (!tsk->thread.regs)
565 return;
566
567 usermsr = tsk->thread.regs->msr;
568
569 if ((usermsr & msr_all_available) == 0)
570 return;
571
572 msr_check_and_set(msr_all_available);
573
Benjamin Herrenschmidt96c79b62017-08-16 16:01:18 +1000574 WARN_ON((usermsr & MSR_VSX) && !((usermsr & MSR_FP) && (usermsr & MSR_VEC)));
Cyril Burde2a20a2016-02-29 17:53:48 +1100575
Benjamin Herrenschmidt96c79b62017-08-16 16:01:18 +1000576 if (usermsr & MSR_FP)
577 save_fpu(tsk);
578
579 if (usermsr & MSR_VEC)
580 save_altivec(tsk);
Cyril Burde2a20a2016-02-29 17:53:48 +1100581
582 if (usermsr & MSR_SPE)
583 __giveup_spe(tsk);
584
585 msr_check_and_clear(msr_all_available);
Ram Paic76662e2018-07-17 06:51:05 -0700586 thread_pkey_regs_save(&tsk->thread);
Cyril Burde2a20a2016-02-29 17:53:48 +1100587}
588
Anton Blanchard579e6332015-10-29 11:44:09 +1100589void flush_all_to_thread(struct task_struct *tsk)
590{
591 if (tsk->thread.regs) {
592 preempt_disable();
593 BUG_ON(tsk != current);
Cyril Burde2a20a2016-02-29 17:53:48 +1100594 save_all(tsk);
Anton Blanchard579e6332015-10-29 11:44:09 +1100595
596#ifdef CONFIG_SPE
597 if (tsk->thread.regs->msr & MSR_SPE)
598 tsk->thread.spefscr = mfspr(SPRN_SPEFSCR);
599#endif
600
601 preempt_enable();
602 }
603}
604EXPORT_SYMBOL(flush_all_to_thread);
605
Dave Kleikamp3bffb652010-02-08 11:51:18 +0000606#ifdef CONFIG_PPC_ADV_DEBUG_REGS
607void do_send_trap(struct pt_regs *regs, unsigned long address,
Eric W. Biederman47355042018-01-16 16:12:38 -0600608 unsigned long error_code, int breakpt)
Dave Kleikamp3bffb652010-02-08 11:51:18 +0000609{
Eric W. Biederman47355042018-01-16 16:12:38 -0600610 current->thread.trap_nr = TRAP_HWBKPT;
Dave Kleikamp3bffb652010-02-08 11:51:18 +0000611 if (notify_die(DIE_DABR_MATCH, "dabr_match", regs, error_code,
612 11, SIGSEGV) == NOTIFY_STOP)
613 return;
614
615 /* Deliver the signal to userspace */
Eric W. Biedermanf71dd7d2018-01-22 14:37:25 -0600616 force_sig_ptrace_errno_trap(breakpt, /* breakpoint or watchpoint id */
617 (void __user *)address);
Dave Kleikamp3bffb652010-02-08 11:51:18 +0000618}
619#else /* !CONFIG_PPC_ADV_DEBUG_REGS */
Michael Neuling9422de32012-12-20 14:06:44 +0000620void do_break (struct pt_regs *regs, unsigned long address,
Luis Machadod6a61bf2008-07-24 02:10:41 +1000621 unsigned long error_code)
622{
623 siginfo_t info;
624
Ananth N Mavinakayanahalli41ab5262012-08-23 21:27:09 +0000625 current->thread.trap_nr = TRAP_HWBKPT;
Luis Machadod6a61bf2008-07-24 02:10:41 +1000626 if (notify_die(DIE_DABR_MATCH, "dabr_match", regs, error_code,
627 11, SIGSEGV) == NOTIFY_STOP)
628 return;
629
Michael Neuling9422de32012-12-20 14:06:44 +0000630 if (debugger_break_match(regs))
Luis Machadod6a61bf2008-07-24 02:10:41 +1000631 return;
632
Michael Neuling9422de32012-12-20 14:06:44 +0000633 /* Clear the breakpoint */
634 hw_breakpoint_disable();
Luis Machadod6a61bf2008-07-24 02:10:41 +1000635
636 /* Deliver the signal to userspace */
Eric W. Biederman3eb0f512018-04-17 15:26:37 -0500637 clear_siginfo(&info);
Luis Machadod6a61bf2008-07-24 02:10:41 +1000638 info.si_signo = SIGTRAP;
639 info.si_errno = 0;
640 info.si_code = TRAP_HWBKPT;
641 info.si_addr = (void __user *)address;
642 force_sig_info(SIGTRAP, &info, current);
643}
Dave Kleikamp3bffb652010-02-08 11:51:18 +0000644#endif /* CONFIG_PPC_ADV_DEBUG_REGS */
Luis Machadod6a61bf2008-07-24 02:10:41 +1000645
Michael Neuling9422de32012-12-20 14:06:44 +0000646static DEFINE_PER_CPU(struct arch_hw_breakpoint, current_brk);
Michael Ellermana2ceff52008-03-28 19:11:48 +1100647
Dave Kleikamp3bffb652010-02-08 11:51:18 +0000648#ifdef CONFIG_PPC_ADV_DEBUG_REGS
649/*
650 * Set the debug registers back to their default "safe" values.
651 */
652static void set_debug_reg_defaults(struct thread_struct *thread)
653{
Bharat Bhushan51ae8d42013-07-04 11:45:46 +0530654 thread->debug.iac1 = thread->debug.iac2 = 0;
Dave Kleikamp3bffb652010-02-08 11:51:18 +0000655#if CONFIG_PPC_ADV_DEBUG_IACS > 2
Bharat Bhushan51ae8d42013-07-04 11:45:46 +0530656 thread->debug.iac3 = thread->debug.iac4 = 0;
Dave Kleikamp3bffb652010-02-08 11:51:18 +0000657#endif
Bharat Bhushan51ae8d42013-07-04 11:45:46 +0530658 thread->debug.dac1 = thread->debug.dac2 = 0;
Dave Kleikamp3bffb652010-02-08 11:51:18 +0000659#if CONFIG_PPC_ADV_DEBUG_DVCS > 0
Bharat Bhushan51ae8d42013-07-04 11:45:46 +0530660 thread->debug.dvc1 = thread->debug.dvc2 = 0;
Dave Kleikamp3bffb652010-02-08 11:51:18 +0000661#endif
Bharat Bhushan51ae8d42013-07-04 11:45:46 +0530662 thread->debug.dbcr0 = 0;
Dave Kleikamp3bffb652010-02-08 11:51:18 +0000663#ifdef CONFIG_BOOKE
664 /*
665 * Force User/Supervisor bits to b11 (user-only MSR[PR]=1)
666 */
Bharat Bhushan51ae8d42013-07-04 11:45:46 +0530667 thread->debug.dbcr1 = DBCR1_IAC1US | DBCR1_IAC2US |
Dave Kleikamp3bffb652010-02-08 11:51:18 +0000668 DBCR1_IAC3US | DBCR1_IAC4US;
669 /*
670 * Force Data Address Compare User/Supervisor bits to be User-only
671 * (0b11 MSR[PR]=1) and set all other bits in DBCR2 register to be 0.
672 */
Bharat Bhushan51ae8d42013-07-04 11:45:46 +0530673 thread->debug.dbcr2 = DBCR2_DAC1US | DBCR2_DAC2US;
Dave Kleikamp3bffb652010-02-08 11:51:18 +0000674#else
Bharat Bhushan51ae8d42013-07-04 11:45:46 +0530675 thread->debug.dbcr1 = 0;
Dave Kleikamp3bffb652010-02-08 11:51:18 +0000676#endif
677}
678
Scott Woodf5f97212013-11-22 15:52:29 -0600679static void prime_debug_regs(struct debug_reg *debug)
Dave Kleikamp3bffb652010-02-08 11:51:18 +0000680{
Scott Wood6cecf762013-05-13 14:14:53 +0000681 /*
682 * We could have inherited MSR_DE from userspace, since
683 * it doesn't get cleared on exception entry. Make sure
684 * MSR_DE is clear before we enable any debug events.
685 */
686 mtmsr(mfmsr() & ~MSR_DE);
687
Scott Woodf5f97212013-11-22 15:52:29 -0600688 mtspr(SPRN_IAC1, debug->iac1);
689 mtspr(SPRN_IAC2, debug->iac2);
Dave Kleikamp3bffb652010-02-08 11:51:18 +0000690#if CONFIG_PPC_ADV_DEBUG_IACS > 2
Scott Woodf5f97212013-11-22 15:52:29 -0600691 mtspr(SPRN_IAC3, debug->iac3);
692 mtspr(SPRN_IAC4, debug->iac4);
Dave Kleikamp3bffb652010-02-08 11:51:18 +0000693#endif
Scott Woodf5f97212013-11-22 15:52:29 -0600694 mtspr(SPRN_DAC1, debug->dac1);
695 mtspr(SPRN_DAC2, debug->dac2);
Dave Kleikamp3bffb652010-02-08 11:51:18 +0000696#if CONFIG_PPC_ADV_DEBUG_DVCS > 0
Scott Woodf5f97212013-11-22 15:52:29 -0600697 mtspr(SPRN_DVC1, debug->dvc1);
698 mtspr(SPRN_DVC2, debug->dvc2);
Dave Kleikamp3bffb652010-02-08 11:51:18 +0000699#endif
Scott Woodf5f97212013-11-22 15:52:29 -0600700 mtspr(SPRN_DBCR0, debug->dbcr0);
701 mtspr(SPRN_DBCR1, debug->dbcr1);
Dave Kleikamp3bffb652010-02-08 11:51:18 +0000702#ifdef CONFIG_BOOKE
Scott Woodf5f97212013-11-22 15:52:29 -0600703 mtspr(SPRN_DBCR2, debug->dbcr2);
Dave Kleikamp3bffb652010-02-08 11:51:18 +0000704#endif
705}
706/*
707 * Unless neither the old or new thread are making use of the
708 * debug registers, set the debug registers from the values
709 * stored in the new thread.
710 */
Scott Woodf5f97212013-11-22 15:52:29 -0600711void switch_booke_debug_regs(struct debug_reg *new_debug)
Dave Kleikamp3bffb652010-02-08 11:51:18 +0000712{
Bharat Bhushan51ae8d42013-07-04 11:45:46 +0530713 if ((current->thread.debug.dbcr0 & DBCR0_IDM)
Scott Woodf5f97212013-11-22 15:52:29 -0600714 || (new_debug->dbcr0 & DBCR0_IDM))
715 prime_debug_regs(new_debug);
Dave Kleikamp3bffb652010-02-08 11:51:18 +0000716}
Bharat Bhushan3743c9b2013-07-04 12:27:44 +0530717EXPORT_SYMBOL_GPL(switch_booke_debug_regs);
Dave Kleikamp3bffb652010-02-08 11:51:18 +0000718#else /* !CONFIG_PPC_ADV_DEBUG_REGS */
K.Prasade0780b72011-02-10 04:44:35 +0000719#ifndef CONFIG_HAVE_HW_BREAKPOINT
Dave Kleikamp3bffb652010-02-08 11:51:18 +0000720static void set_debug_reg_defaults(struct thread_struct *thread)
721{
Michael Neuling9422de32012-12-20 14:06:44 +0000722 thread->hw_brk.address = 0;
723 thread->hw_brk.type = 0;
Nicholas Piggin252988c2018-04-01 15:50:36 +1000724 if (ppc_breakpoint_available())
725 set_breakpoint(&thread->hw_brk);
Dave Kleikamp3bffb652010-02-08 11:51:18 +0000726}
K.Prasade0780b72011-02-10 04:44:35 +0000727#endif /* !CONFIG_HAVE_HW_BREAKPOINT */
Dave Kleikamp3bffb652010-02-08 11:51:18 +0000728#endif /* CONFIG_PPC_ADV_DEBUG_REGS */
729
Dave Kleikamp172ae2e2010-02-08 11:50:57 +0000730#ifdef CONFIG_PPC_ADV_DEBUG_REGS
Michael Neuling9422de32012-12-20 14:06:44 +0000731static inline int __set_dabr(unsigned long dabr, unsigned long dabrx)
732{
Benjamin Herrenschmidtc6c9eac2009-09-08 14:16:58 +0000733 mtspr(SPRN_DAC1, dabr);
Dave Kleikamp221c1852010-03-05 10:43:24 +0000734#ifdef CONFIG_PPC_47x
735 isync();
736#endif
Michael Neuling9422de32012-12-20 14:06:44 +0000737 return 0;
738}
Benjamin Herrenschmidtc6c9eac2009-09-08 14:16:58 +0000739#elif defined(CONFIG_PPC_BOOK3S)
Michael Neuling9422de32012-12-20 14:06:44 +0000740static inline int __set_dabr(unsigned long dabr, unsigned long dabrx)
741{
Michael Ellermancab0af92005-11-03 15:30:49 +1100742 mtspr(SPRN_DABR, dabr);
Michael Neuling82a9f162013-05-16 20:27:31 +0000743 if (cpu_has_feature(CPU_FTR_DABRX))
744 mtspr(SPRN_DABRX, dabrx);
Michael Ellermancab0af92005-11-03 15:30:49 +1100745 return 0;
Paul Mackerras14cf11a2005-09-26 16:04:21 +1000746}
Christophe Leroy4ad86222016-11-29 09:52:15 +0100747#elif defined(CONFIG_PPC_8xx)
748static inline int __set_dabr(unsigned long dabr, unsigned long dabrx)
749{
750 unsigned long addr = dabr & ~HW_BRK_TYPE_DABR;
751 unsigned long lctrl1 = 0x90000000; /* compare type: equal on E & F */
752 unsigned long lctrl2 = 0x8e000002; /* watchpoint 1 on cmp E | F */
753
754 if ((dabr & HW_BRK_TYPE_RDWR) == HW_BRK_TYPE_READ)
755 lctrl1 |= 0xa0000;
756 else if ((dabr & HW_BRK_TYPE_RDWR) == HW_BRK_TYPE_WRITE)
757 lctrl1 |= 0xf0000;
758 else if ((dabr & HW_BRK_TYPE_RDWR) == 0)
759 lctrl2 = 0;
760
761 mtspr(SPRN_LCTRL2, 0);
762 mtspr(SPRN_CMPE, addr);
763 mtspr(SPRN_CMPF, addr + 4);
764 mtspr(SPRN_LCTRL1, lctrl1);
765 mtspr(SPRN_LCTRL2, lctrl2);
766
767 return 0;
768}
Michael Neuling9422de32012-12-20 14:06:44 +0000769#else
770static inline int __set_dabr(unsigned long dabr, unsigned long dabrx)
771{
772 return -EINVAL;
773}
774#endif
775
776static inline int set_dabr(struct arch_hw_breakpoint *brk)
777{
778 unsigned long dabr, dabrx;
779
780 dabr = brk->address | (brk->type & HW_BRK_TYPE_DABR);
781 dabrx = ((brk->type >> 3) & 0x7);
782
783 if (ppc_md.set_dabr)
784 return ppc_md.set_dabr(dabr, dabrx);
785
786 return __set_dabr(dabr, dabrx);
787}
788
Michael Neulingbf99de32012-12-20 14:06:45 +0000789static inline int set_dawr(struct arch_hw_breakpoint *brk)
790{
Michael Neuling05d694e2013-01-24 15:02:58 +0000791 unsigned long dawr, dawrx, mrd;
Michael Neulingbf99de32012-12-20 14:06:45 +0000792
793 dawr = brk->address;
794
795 dawrx = (brk->type & (HW_BRK_TYPE_READ | HW_BRK_TYPE_WRITE)) \
796 << (63 - 58); //* read/write bits */
797 dawrx |= ((brk->type & (HW_BRK_TYPE_TRANSLATE)) >> 2) \
798 << (63 - 59); //* translate */
799 dawrx |= (brk->type & (HW_BRK_TYPE_PRIV_ALL)) \
800 >> 3; //* PRIM bits */
Michael Neuling05d694e2013-01-24 15:02:58 +0000801 /* dawr length is stored in field MDR bits 48:53. Matches range in
802 doublewords (64 bits) baised by -1 eg. 0b000000=1DW and
803 0b111111=64DW.
804 brk->len is in bytes.
805 This aligns up to double word size, shifts and does the bias.
806 */
807 mrd = ((brk->len + 7) >> 3) - 1;
808 dawrx |= (mrd & 0x3f) << (63 - 53);
Michael Neulingbf99de32012-12-20 14:06:45 +0000809
810 if (ppc_md.set_dawr)
811 return ppc_md.set_dawr(dawr, dawrx);
812 mtspr(SPRN_DAWR, dawr);
813 mtspr(SPRN_DAWRX, dawrx);
814 return 0;
815}
816
Paul Gortmaker21f58502014-04-29 15:25:17 -0400817void __set_breakpoint(struct arch_hw_breakpoint *brk)
Michael Neuling9422de32012-12-20 14:06:44 +0000818{
Christoph Lameter69111ba2014-10-21 15:23:25 -0500819 memcpy(this_cpu_ptr(&current_brk), brk, sizeof(*brk));
Michael Neuling9422de32012-12-20 14:06:44 +0000820
Michael Neulingbf99de32012-12-20 14:06:45 +0000821 if (cpu_has_feature(CPU_FTR_DAWR))
Nicholas Piggin252988c2018-04-01 15:50:36 +1000822 // Power8 or later
Paul Gortmaker04c32a52014-04-29 15:25:16 -0400823 set_dawr(brk);
Nicholas Piggin252988c2018-04-01 15:50:36 +1000824 else if (!cpu_has_feature(CPU_FTR_ARCH_207S))
825 // Power7 or earlier
Paul Gortmaker04c32a52014-04-29 15:25:16 -0400826 set_dabr(brk);
Nicholas Piggin252988c2018-04-01 15:50:36 +1000827 else
828 // Shouldn't happen due to higher level checks
829 WARN_ON_ONCE(1);
Michael Neuling9422de32012-12-20 14:06:44 +0000830}
Paul Mackerras14cf11a2005-09-26 16:04:21 +1000831
Paul Gortmaker21f58502014-04-29 15:25:17 -0400832void set_breakpoint(struct arch_hw_breakpoint *brk)
833{
834 preempt_disable();
835 __set_breakpoint(brk);
836 preempt_enable();
837}
838
Michael Neuling404b27d2018-03-27 15:37:17 +1100839/* Check if we have DAWR or DABR hardware */
840bool ppc_breakpoint_available(void)
841{
842 if (cpu_has_feature(CPU_FTR_DAWR))
843 return true; /* POWER8 DAWR */
844 if (cpu_has_feature(CPU_FTR_ARCH_207S))
845 return false; /* POWER9 with DAWR disabled */
846 /* DABR: Everything but POWER8 and POWER9 */
847 return true;
848}
849EXPORT_SYMBOL_GPL(ppc_breakpoint_available);
850
Michael Neuling9422de32012-12-20 14:06:44 +0000851static inline bool hw_brk_match(struct arch_hw_breakpoint *a,
852 struct arch_hw_breakpoint *b)
853{
854 if (a->address != b->address)
855 return false;
856 if (a->type != b->type)
857 return false;
858 if (a->len != b->len)
859 return false;
860 return true;
861}
Paul Mackerrasd31626f2014-01-13 15:56:29 +1100862
Michael Neulingfb096922013-02-13 16:21:37 +0000863#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
Cyril Bur5d176f72016-09-14 18:02:16 +1000864
865static inline bool tm_enabled(struct task_struct *tsk)
866{
867 return tsk && tsk->thread.regs && (tsk->thread.regs->msr & MSR_TM);
868}
869
Paul Mackerrasd31626f2014-01-13 15:56:29 +1100870static void tm_reclaim_thread(struct thread_struct *thr,
871 struct thread_info *ti, uint8_t cause)
872{
Michael Neuling7f821fc2015-11-19 15:44:45 +1100873 /*
874 * Use the current MSR TM suspended bit to track if we have
875 * checkpointed state outstanding.
876 * On signal delivery, we'd normally reclaim the checkpointed
877 * state to obtain stack pointer (see:get_tm_stackpointer()).
878 * This will then directly return to userspace without going
879 * through __switch_to(). However, if the stack frame is bad,
880 * we need to exit this thread which calls __switch_to() which
881 * will again attempt to reclaim the already saved tm state.
882 * Hence we need to check that we've not already reclaimed
883 * this state.
884 * We do this using the current MSR, rather tracking it in
885 * some specific thread_struct bit, as it has the additional
Michael Ellerman027dfac2016-06-01 16:34:37 +1000886 * benefit of checking for a potential TM bad thing exception.
Michael Neuling7f821fc2015-11-19 15:44:45 +1100887 */
888 if (!MSR_TM_SUSPENDED(mfmsr()))
889 return;
890
Cyril Bur91381b92017-11-02 14:09:04 +1100891 giveup_all(container_of(thr, struct task_struct, thread));
892
Cyril Bureb5c3f12017-11-02 14:09:05 +1100893 tm_reclaim(thr, cause);
894
Michael Neulingf48e91e2017-05-08 17:16:26 +1000895 /*
896 * If we are in a transaction and FP is off then we can't have
897 * used FP inside that transaction. Hence the checkpointed
898 * state is the same as the live state. We need to copy the
899 * live state to the checkpointed state so that when the
900 * transaction is restored, the checkpointed state is correct
901 * and the aborted transaction sees the correct state. We use
902 * ckpt_regs.msr here as that's what tm_reclaim will use to
903 * determine if it's going to write the checkpointed state or
904 * not. So either this will write the checkpointed registers,
905 * or reclaim will. Similarly for VMX.
906 */
907 if ((thr->ckpt_regs.msr & MSR_FP) == 0)
908 memcpy(&thr->ckfp_state, &thr->fp_state,
909 sizeof(struct thread_fp_state));
910 if ((thr->ckpt_regs.msr & MSR_VEC) == 0)
911 memcpy(&thr->ckvr_state, &thr->vr_state,
912 sizeof(struct thread_vr_state));
Paul Mackerrasd31626f2014-01-13 15:56:29 +1100913}
914
915void tm_reclaim_current(uint8_t cause)
916{
917 tm_enable();
918 tm_reclaim_thread(&current->thread, current_thread_info(), cause);
919}
920
Michael Neulingfb096922013-02-13 16:21:37 +0000921static inline void tm_reclaim_task(struct task_struct *tsk)
922{
923 /* We have to work out if we're switching from/to a task that's in the
924 * middle of a transaction.
925 *
926 * In switching we need to maintain a 2nd register state as
927 * oldtask->thread.ckpt_regs. We tm_reclaim(oldproc); this saves the
Cyril Bur000ec282016-09-23 16:18:25 +1000928 * checkpointed (tbegin) state in ckpt_regs, ckfp_state and
929 * ckvr_state
Michael Neulingfb096922013-02-13 16:21:37 +0000930 *
931 * We also context switch (save) TFHAR/TEXASR/TFIAR in here.
932 */
933 struct thread_struct *thr = &tsk->thread;
934
935 if (!thr->regs)
936 return;
937
938 if (!MSR_TM_ACTIVE(thr->regs->msr))
939 goto out_and_saveregs;
940
Michael Neuling92fb8692017-10-12 21:17:19 +1100941 WARN_ON(tm_suspend_disabled);
942
Michael Neulingfb096922013-02-13 16:21:37 +0000943 TM_DEBUG("--- tm_reclaim on pid %d (NIP=%lx, "
944 "ccr=%lx, msr=%lx, trap=%lx)\n",
945 tsk->pid, thr->regs->nip,
946 thr->regs->ccr, thr->regs->msr,
947 thr->regs->trap);
948
Paul Mackerrasd31626f2014-01-13 15:56:29 +1100949 tm_reclaim_thread(thr, task_thread_info(tsk), TM_CAUSE_RESCHED);
Michael Neulingfb096922013-02-13 16:21:37 +0000950
951 TM_DEBUG("--- tm_reclaim on pid %d complete\n",
952 tsk->pid);
953
954out_and_saveregs:
955 /* Always save the regs here, even if a transaction's not active.
956 * This context-switches a thread's TM info SPRs. We do it here to
957 * be consistent with the restore path (in recheckpoint) which
958 * cannot happen later in _switch().
959 */
960 tm_save_sprs(thr);
961}
962
Cyril Bureb5c3f12017-11-02 14:09:05 +1100963extern void __tm_recheckpoint(struct thread_struct *thread);
Michael Neulinge6b8fd02014-04-04 20:19:48 +1100964
Cyril Bureb5c3f12017-11-02 14:09:05 +1100965void tm_recheckpoint(struct thread_struct *thread)
Michael Neulinge6b8fd02014-04-04 20:19:48 +1100966{
967 unsigned long flags;
968
Cyril Bur5d176f72016-09-14 18:02:16 +1000969 if (!(thread->regs->msr & MSR_TM))
970 return;
971
Michael Neulinge6b8fd02014-04-04 20:19:48 +1100972 /* We really can't be interrupted here as the TEXASR registers can't
973 * change and later in the trecheckpoint code, we have a userspace R1.
974 * So let's hard disable over this region.
975 */
976 local_irq_save(flags);
977 hard_irq_disable();
978
979 /* The TM SPRs are restored here, so that TEXASR.FS can be set
980 * before the trecheckpoint and no explosion occurs.
981 */
982 tm_restore_sprs(thread);
983
Cyril Bureb5c3f12017-11-02 14:09:05 +1100984 __tm_recheckpoint(thread);
Michael Neulinge6b8fd02014-04-04 20:19:48 +1100985
986 local_irq_restore(flags);
987}
988
Michael Neulingbc2a9402013-02-13 16:21:40 +0000989static inline void tm_recheckpoint_new_task(struct task_struct *new)
Michael Neulingfb096922013-02-13 16:21:37 +0000990{
Michael Neulingfb096922013-02-13 16:21:37 +0000991 if (!cpu_has_feature(CPU_FTR_TM))
992 return;
993
994 /* Recheckpoint the registers of the thread we're about to switch to.
995 *
996 * If the task was using FP, we non-lazily reload both the original and
997 * the speculative FP register states. This is because the kernel
998 * doesn't see if/when a TM rollback occurs, so if we take an FP
Cyril Burdc310662016-09-23 16:18:24 +1000999 * unavailable later, we are unable to determine which set of FP regs
Michael Neulingfb096922013-02-13 16:21:37 +00001000 * need to be restored.
1001 */
Cyril Bur5d176f72016-09-14 18:02:16 +10001002 if (!tm_enabled(new))
Michael Neulingfb096922013-02-13 16:21:37 +00001003 return;
1004
Michael Neulinge6b8fd02014-04-04 20:19:48 +11001005 if (!MSR_TM_ACTIVE(new->thread.regs->msr)){
1006 tm_restore_sprs(&new->thread);
Michael Neulingfb096922013-02-13 16:21:37 +00001007 return;
Michael Neulinge6b8fd02014-04-04 20:19:48 +11001008 }
Michael Neulingfb096922013-02-13 16:21:37 +00001009 /* Recheckpoint to restore original checkpointed register state. */
Cyril Bureb5c3f12017-11-02 14:09:05 +11001010 TM_DEBUG("*** tm_recheckpoint of pid %d (new->msr 0x%lx)\n",
1011 new->pid, new->thread.regs->msr);
Michael Neulingfb096922013-02-13 16:21:37 +00001012
Cyril Bureb5c3f12017-11-02 14:09:05 +11001013 tm_recheckpoint(&new->thread);
Michael Neulingfb096922013-02-13 16:21:37 +00001014
Cyril Burdc310662016-09-23 16:18:24 +10001015 /*
1016 * The checkpointed state has been restored but the live state has
1017 * not, ensure all the math functionality is turned off to trigger
1018 * restore_math() to reload.
1019 */
1020 new->thread.regs->msr &= ~(MSR_FP | MSR_VEC | MSR_VSX);
Michael Neulingfb096922013-02-13 16:21:37 +00001021
1022 TM_DEBUG("*** tm_recheckpoint of pid %d complete "
1023 "(kernel msr 0x%lx)\n",
1024 new->pid, mfmsr());
1025}
1026
Cyril Burdc310662016-09-23 16:18:24 +10001027static inline void __switch_to_tm(struct task_struct *prev,
1028 struct task_struct *new)
Michael Neulingfb096922013-02-13 16:21:37 +00001029{
1030 if (cpu_has_feature(CPU_FTR_TM)) {
Cyril Bur5d176f72016-09-14 18:02:16 +10001031 if (tm_enabled(prev) || tm_enabled(new))
1032 tm_enable();
1033
1034 if (tm_enabled(prev)) {
1035 prev->thread.load_tm++;
1036 tm_reclaim_task(prev);
1037 if (!MSR_TM_ACTIVE(prev->thread.regs->msr) && prev->thread.load_tm == 0)
1038 prev->thread.regs->msr &= ~MSR_TM;
1039 }
1040
Cyril Burdc310662016-09-23 16:18:24 +10001041 tm_recheckpoint_new_task(new);
Michael Neulingfb096922013-02-13 16:21:37 +00001042 }
1043}
Paul Mackerrasd31626f2014-01-13 15:56:29 +11001044
1045/*
1046 * This is called if we are on the way out to userspace and the
1047 * TIF_RESTORE_TM flag is set. It checks if we need to reload
1048 * FP and/or vector state and does so if necessary.
1049 * If userspace is inside a transaction (whether active or
1050 * suspended) and FP/VMX/VSX instructions have ever been enabled
1051 * inside that transaction, then we have to keep them enabled
1052 * and keep the FP/VMX/VSX state loaded while ever the transaction
1053 * continues. The reason is that if we didn't, and subsequently
1054 * got a FP/VMX/VSX unavailable interrupt inside a transaction,
1055 * we don't know whether it's the same transaction, and thus we
1056 * don't know which of the checkpointed state and the transactional
1057 * state to use.
1058 */
1059void restore_tm_state(struct pt_regs *regs)
1060{
1061 unsigned long msr_diff;
1062
Cyril Burdc310662016-09-23 16:18:24 +10001063 /*
1064 * This is the only moment we should clear TIF_RESTORE_TM as
1065 * it is here that ckpt_regs.msr and pt_regs.msr become the same
1066 * again, anything else could lead to an incorrect ckpt_msr being
1067 * saved and therefore incorrect signal contexts.
1068 */
Paul Mackerrasd31626f2014-01-13 15:56:29 +11001069 clear_thread_flag(TIF_RESTORE_TM);
1070 if (!MSR_TM_ACTIVE(regs->msr))
1071 return;
1072
Anshuman Khandual829023d2015-07-06 16:24:10 +05301073 msr_diff = current->thread.ckpt_regs.msr & ~regs->msr;
Paul Mackerrasd31626f2014-01-13 15:56:29 +11001074 msr_diff &= MSR_FP | MSR_VEC | MSR_VSX;
Cyril Bur70fe3d92016-02-29 17:53:47 +11001075
Cyril Burdc16b552016-09-23 16:18:08 +10001076 /* Ensure that restore_math() will restore */
1077 if (msr_diff & MSR_FP)
1078 current->thread.load_fp = 1;
Valentin Rothberg39715bf2016-10-05 07:57:26 +02001079#ifdef CONFIG_ALTIVEC
Cyril Burdc16b552016-09-23 16:18:08 +10001080 if (cpu_has_feature(CPU_FTR_ALTIVEC) && msr_diff & MSR_VEC)
1081 current->thread.load_vec = 1;
1082#endif
Cyril Bur70fe3d92016-02-29 17:53:47 +11001083 restore_math(regs);
1084
Paul Mackerrasd31626f2014-01-13 15:56:29 +11001085 regs->msr |= msr_diff;
1086}
1087
Michael Neulingfb096922013-02-13 16:21:37 +00001088#else
1089#define tm_recheckpoint_new_task(new)
Cyril Burdc310662016-09-23 16:18:24 +10001090#define __switch_to_tm(prev, new)
Michael Neulingfb096922013-02-13 16:21:37 +00001091#endif /* CONFIG_PPC_TRANSACTIONAL_MEM */
Michael Neuling9422de32012-12-20 14:06:44 +00001092
Anton Blanchard152d5232015-10-29 11:43:55 +11001093static inline void save_sprs(struct thread_struct *t)
1094{
1095#ifdef CONFIG_ALTIVEC
Oliver O'Halloran01d7c2a22016-03-08 09:08:47 +11001096 if (cpu_has_feature(CPU_FTR_ALTIVEC))
Anton Blanchard152d5232015-10-29 11:43:55 +11001097 t->vrsave = mfspr(SPRN_VRSAVE);
1098#endif
1099#ifdef CONFIG_PPC_BOOK3S_64
1100 if (cpu_has_feature(CPU_FTR_DSCR))
1101 t->dscr = mfspr(SPRN_DSCR);
1102
1103 if (cpu_has_feature(CPU_FTR_ARCH_207S)) {
1104 t->bescr = mfspr(SPRN_BESCR);
1105 t->ebbhr = mfspr(SPRN_EBBHR);
1106 t->ebbrr = mfspr(SPRN_EBBRR);
1107
1108 t->fscr = mfspr(SPRN_FSCR);
1109
1110 /*
1111 * Note that the TAR is not available for use in the kernel.
1112 * (To provide this, the TAR should be backed up/restored on
1113 * exception entry/exit instead, and be in pt_regs. FIXME,
1114 * this should be in pt_regs anyway (for debug).)
1115 */
1116 t->tar = mfspr(SPRN_TAR);
1117 }
1118#endif
Ram Pai06bb53b2018-01-18 17:50:31 -08001119
1120 thread_pkey_regs_save(t);
Anton Blanchard152d5232015-10-29 11:43:55 +11001121}
1122
1123static inline void restore_sprs(struct thread_struct *old_thread,
1124 struct thread_struct *new_thread)
1125{
1126#ifdef CONFIG_ALTIVEC
1127 if (cpu_has_feature(CPU_FTR_ALTIVEC) &&
1128 old_thread->vrsave != new_thread->vrsave)
1129 mtspr(SPRN_VRSAVE, new_thread->vrsave);
1130#endif
1131#ifdef CONFIG_PPC_BOOK3S_64
1132 if (cpu_has_feature(CPU_FTR_DSCR)) {
1133 u64 dscr = get_paca()->dscr_default;
Michael Neulingb57bd2d2016-06-09 12:31:08 +10001134 if (new_thread->dscr_inherit)
Anton Blanchard152d5232015-10-29 11:43:55 +11001135 dscr = new_thread->dscr;
Anton Blanchard152d5232015-10-29 11:43:55 +11001136
1137 if (old_thread->dscr != dscr)
1138 mtspr(SPRN_DSCR, dscr);
Anton Blanchard152d5232015-10-29 11:43:55 +11001139 }
1140
1141 if (cpu_has_feature(CPU_FTR_ARCH_207S)) {
1142 if (old_thread->bescr != new_thread->bescr)
1143 mtspr(SPRN_BESCR, new_thread->bescr);
1144 if (old_thread->ebbhr != new_thread->ebbhr)
1145 mtspr(SPRN_EBBHR, new_thread->ebbhr);
1146 if (old_thread->ebbrr != new_thread->ebbrr)
1147 mtspr(SPRN_EBBRR, new_thread->ebbrr);
1148
Michael Neulingb57bd2d2016-06-09 12:31:08 +10001149 if (old_thread->fscr != new_thread->fscr)
1150 mtspr(SPRN_FSCR, new_thread->fscr);
1151
Anton Blanchard152d5232015-10-29 11:43:55 +11001152 if (old_thread->tar != new_thread->tar)
1153 mtspr(SPRN_TAR, new_thread->tar);
1154 }
Sukadev Bhattiproluec233ed2017-11-07 18:23:53 -08001155
Alastair D'Silva3449f192018-05-11 16:12:58 +10001156 if (cpu_has_feature(CPU_FTR_P9_TIDR) &&
Sukadev Bhattiproluec233ed2017-11-07 18:23:53 -08001157 old_thread->tidr != new_thread->tidr)
1158 mtspr(SPRN_TIDR, new_thread->tidr);
Anton Blanchard152d5232015-10-29 11:43:55 +11001159#endif
Ram Pai06bb53b2018-01-18 17:50:31 -08001160
1161 thread_pkey_regs_restore(new_thread, old_thread);
Anton Blanchard152d5232015-10-29 11:43:55 +11001162}
1163
Nicholas Piggin07d2a622017-06-09 01:36:09 +10001164#ifdef CONFIG_PPC_BOOK3S_64
1165#define CP_SIZE 128
1166static const u8 dummy_copy_buffer[CP_SIZE] __attribute__((aligned(CP_SIZE)));
1167#endif
1168
Paul Mackerras14cf11a2005-09-26 16:04:21 +10001169struct task_struct *__switch_to(struct task_struct *prev,
1170 struct task_struct *new)
1171{
1172 struct thread_struct *new_thread, *old_thread;
Paul Mackerras14cf11a2005-09-26 16:04:21 +10001173 struct task_struct *last;
Peter Zijlstrad6bf29b2011-05-24 17:11:48 -07001174#ifdef CONFIG_PPC_BOOK3S_64
1175 struct ppc64_tlb_batch *batch;
1176#endif
Paul Mackerras14cf11a2005-09-26 16:04:21 +10001177
Anton Blanchard152d5232015-10-29 11:43:55 +11001178 new_thread = &new->thread;
1179 old_thread = &current->thread;
1180
Michael Neuling7ba5fef2013-10-02 17:15:14 +10001181 WARN_ON(!irqs_disabled());
1182
Michael Ellerman4e003742017-10-19 15:08:43 +11001183#ifdef CONFIG_PPC_BOOK3S_64
Christoph Lameter69111ba2014-10-21 15:23:25 -05001184 batch = this_cpu_ptr(&ppc64_tlb_batch);
Peter Zijlstrad6bf29b2011-05-24 17:11:48 -07001185 if (batch->active) {
1186 current_thread_info()->local_flags |= _TLF_LAZY_MMU;
1187 if (batch->index)
1188 __flush_tlb_pending(batch);
1189 batch->active = 0;
1190 }
Michael Ellerman4e003742017-10-19 15:08:43 +11001191#endif /* CONFIG_PPC_BOOK3S_64 */
Paul Mackerras06d67d52005-10-10 22:29:05 +10001192
Anton Blanchardf3d885c2015-10-29 11:44:10 +11001193#ifdef CONFIG_PPC_ADV_DEBUG_REGS
1194 switch_booke_debug_regs(&new->thread.debug);
1195#else
1196/*
1197 * For PPC_BOOK3S_64, we use the hw-breakpoint interfaces that would
1198 * schedule DABR
1199 */
1200#ifndef CONFIG_HAVE_HW_BREAKPOINT
1201 if (unlikely(!hw_brk_match(this_cpu_ptr(&current_brk), &new->thread.hw_brk)))
1202 __set_breakpoint(&new->thread.hw_brk);
1203#endif /* CONFIG_HAVE_HW_BREAKPOINT */
1204#endif
1205
1206 /*
1207 * We need to save SPRs before treclaim/trecheckpoint as these will
1208 * change a number of them.
1209 */
1210 save_sprs(&prev->thread);
1211
Anton Blanchardf3d885c2015-10-29 11:44:10 +11001212 /* Save FPU, Altivec, VSX and SPE state */
1213 giveup_all(prev);
1214
Cyril Burdc310662016-09-23 16:18:24 +10001215 __switch_to_tm(prev, new);
1216
Nicholas Piggine4c0fc52017-06-09 01:36:06 +10001217 if (!radix_enabled()) {
1218 /*
1219 * We can't take a PMU exception inside _switch() since there
1220 * is a window where the kernel stack SLB and the kernel stack
1221 * are out of sync. Hard disable here.
1222 */
1223 hard_irq_disable();
1224 }
Michael Neulingbc2a9402013-02-13 16:21:40 +00001225
Anton Blanchard20dbe672015-12-10 20:44:39 +11001226 /*
1227 * Call restore_sprs() before calling _switch(). If we move it after
1228 * _switch() then we miss out on calling it for new tasks. The reason
1229 * for this is we manually create a stack frame for new tasks that
1230 * directly returns through ret_from_fork() or
1231 * ret_from_kernel_thread(). See copy_thread() for details.
1232 */
Anton Blanchardf3d885c2015-10-29 11:44:10 +11001233 restore_sprs(old_thread, new_thread);
1234
Anton Blanchard20dbe672015-12-10 20:44:39 +11001235 last = _switch(old_thread, new_thread);
1236
Michael Ellerman4e003742017-10-19 15:08:43 +11001237#ifdef CONFIG_PPC_BOOK3S_64
Peter Zijlstrad6bf29b2011-05-24 17:11:48 -07001238 if (current_thread_info()->local_flags & _TLF_LAZY_MMU) {
1239 current_thread_info()->local_flags &= ~_TLF_LAZY_MMU;
Christoph Lameter69111ba2014-10-21 15:23:25 -05001240 batch = this_cpu_ptr(&ppc64_tlb_batch);
Peter Zijlstrad6bf29b2011-05-24 17:11:48 -07001241 batch->active = 1;
1242 }
Cyril Bur70fe3d92016-02-29 17:53:47 +11001243
Nicholas Piggin07d2a622017-06-09 01:36:09 +10001244 if (current_thread_info()->task->thread.regs) {
Cyril Bur70fe3d92016-02-29 17:53:47 +11001245 restore_math(current_thread_info()->task->thread.regs);
Nicholas Piggin07d2a622017-06-09 01:36:09 +10001246
1247 /*
1248 * The copy-paste buffer can only store into foreign real
1249 * addresses, so unprivileged processes can not see the
1250 * data or use it in any way unless they have foreign real
Sukadev Bhattiprolu9d2a4d72017-11-07 18:23:54 -08001251 * mappings. If the new process has the foreign real address
1252 * mappings, we must issue a cp_abort to clear any state and
1253 * prevent snooping, corruption or a covert channel.
Nicholas Piggin07d2a622017-06-09 01:36:09 +10001254 */
Nicholas Piggin2bf10712018-07-05 18:47:00 +10001255 if (current_thread_info()->task->thread.used_vas)
Sukadev Bhattiprolu9d2a4d72017-11-07 18:23:54 -08001256 asm volatile(PPC_CP_ABORT);
Nicholas Piggin07d2a622017-06-09 01:36:09 +10001257 }
Michael Ellerman4e003742017-10-19 15:08:43 +11001258#endif /* CONFIG_PPC_BOOK3S_64 */
Peter Zijlstrad6bf29b2011-05-24 17:11:48 -07001259
Paul Mackerras14cf11a2005-09-26 16:04:21 +10001260 return last;
1261}
1262
Paul Mackerras06d67d52005-10-10 22:29:05 +10001263static int instructions_to_print = 16;
1264
Paul Mackerras06d67d52005-10-10 22:29:05 +10001265static void show_instructions(struct pt_regs *regs)
1266{
1267 int i;
1268 unsigned long pc = regs->nip - (instructions_to_print * 3 / 4 *
1269 sizeof(int));
1270
1271 printk("Instruction dump:");
1272
1273 for (i = 0; i < instructions_to_print; i++) {
1274 int instr;
1275
1276 if (!(i % 8))
Andrew Donnellan2ffd04d2016-11-04 17:20:40 +11001277 pr_cont("\n");
Paul Mackerras06d67d52005-10-10 22:29:05 +10001278
Scott Wood0de2d822007-09-28 04:38:55 +10001279#if !defined(CONFIG_BOOKE)
1280 /* If executing with the IMMU off, adjust pc rather
1281 * than print XXXXXXXX.
1282 */
1283 if (!(regs->msr & MSR_IR))
1284 pc = (unsigned long)phys_to_virt(pc);
1285#endif
1286
Anton Blanchard00ae36d2006-10-13 12:17:16 +10001287 if (!__kernel_text_address(pc) ||
Anton Blanchard7b051f62014-10-13 20:27:15 +11001288 probe_kernel_address((unsigned int __user *)pc, instr)) {
Andrew Donnellan2ffd04d2016-11-04 17:20:40 +11001289 pr_cont("XXXXXXXX ");
Paul Mackerras06d67d52005-10-10 22:29:05 +10001290 } else {
1291 if (regs->nip == pc)
Andrew Donnellan2ffd04d2016-11-04 17:20:40 +11001292 pr_cont("<%08x> ", instr);
Paul Mackerras06d67d52005-10-10 22:29:05 +10001293 else
Andrew Donnellan2ffd04d2016-11-04 17:20:40 +11001294 pr_cont("%08x ", instr);
Paul Mackerras06d67d52005-10-10 22:29:05 +10001295 }
1296
1297 pc += sizeof(int);
1298 }
1299
Andrew Donnellan2ffd04d2016-11-04 17:20:40 +11001300 pr_cont("\n");
Paul Mackerras06d67d52005-10-10 22:29:05 +10001301}
1302
Michael Neuling801c0b22015-11-20 15:15:32 +11001303struct regbit {
Paul Mackerras06d67d52005-10-10 22:29:05 +10001304 unsigned long bit;
1305 const char *name;
Michael Neuling801c0b22015-11-20 15:15:32 +11001306};
1307
1308static struct regbit msr_bits[] = {
Anton Blanchard3bfd0c9c2011-11-24 19:35:57 +00001309#if defined(CONFIG_PPC64) && !defined(CONFIG_BOOKE)
1310 {MSR_SF, "SF"},
1311 {MSR_HV, "HV"},
1312#endif
1313 {MSR_VEC, "VEC"},
1314 {MSR_VSX, "VSX"},
1315#ifdef CONFIG_BOOKE
1316 {MSR_CE, "CE"},
1317#endif
Paul Mackerras06d67d52005-10-10 22:29:05 +10001318 {MSR_EE, "EE"},
1319 {MSR_PR, "PR"},
1320 {MSR_FP, "FP"},
1321 {MSR_ME, "ME"},
Anton Blanchard3bfd0c9c2011-11-24 19:35:57 +00001322#ifdef CONFIG_BOOKE
Kumar Gala1b983262008-11-19 04:39:53 +00001323 {MSR_DE, "DE"},
Anton Blanchard3bfd0c9c2011-11-24 19:35:57 +00001324#else
1325 {MSR_SE, "SE"},
1326 {MSR_BE, "BE"},
1327#endif
Paul Mackerras06d67d52005-10-10 22:29:05 +10001328 {MSR_IR, "IR"},
1329 {MSR_DR, "DR"},
Anton Blanchard3bfd0c9c2011-11-24 19:35:57 +00001330 {MSR_PMM, "PMM"},
1331#ifndef CONFIG_BOOKE
1332 {MSR_RI, "RI"},
1333 {MSR_LE, "LE"},
1334#endif
Paul Mackerras06d67d52005-10-10 22:29:05 +10001335 {0, NULL}
1336};
1337
Michael Neuling801c0b22015-11-20 15:15:32 +11001338static void print_bits(unsigned long val, struct regbit *bits, const char *sep)
Paul Mackerras06d67d52005-10-10 22:29:05 +10001339{
Michael Neuling801c0b22015-11-20 15:15:32 +11001340 const char *s = "";
Paul Mackerras06d67d52005-10-10 22:29:05 +10001341
Paul Mackerras06d67d52005-10-10 22:29:05 +10001342 for (; bits->bit; ++bits)
1343 if (val & bits->bit) {
Michael Ellermandb5ba5a2016-11-02 22:20:47 +11001344 pr_cont("%s%s", s, bits->name);
Michael Neuling801c0b22015-11-20 15:15:32 +11001345 s = sep;
Paul Mackerras06d67d52005-10-10 22:29:05 +10001346 }
Michael Neuling801c0b22015-11-20 15:15:32 +11001347}
1348
1349#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
1350static struct regbit msr_tm_bits[] = {
1351 {MSR_TS_T, "T"},
1352 {MSR_TS_S, "S"},
1353 {MSR_TM, "E"},
1354 {0, NULL}
1355};
1356
1357static void print_tm_bits(unsigned long val)
1358{
1359/*
1360 * This only prints something if at least one of the TM bit is set.
1361 * Inside the TM[], the output means:
1362 * E: Enabled (bit 32)
1363 * S: Suspended (bit 33)
1364 * T: Transactional (bit 34)
1365 */
1366 if (val & (MSR_TM | MSR_TS_S | MSR_TS_T)) {
Michael Ellermandb5ba5a2016-11-02 22:20:47 +11001367 pr_cont(",TM[");
Michael Neuling801c0b22015-11-20 15:15:32 +11001368 print_bits(val, msr_tm_bits, "");
Michael Ellermandb5ba5a2016-11-02 22:20:47 +11001369 pr_cont("]");
Michael Neuling801c0b22015-11-20 15:15:32 +11001370 }
1371}
1372#else
1373static void print_tm_bits(unsigned long val) {}
1374#endif
1375
1376static void print_msr_bits(unsigned long val)
1377{
Michael Ellermandb5ba5a2016-11-02 22:20:47 +11001378 pr_cont("<");
Michael Neuling801c0b22015-11-20 15:15:32 +11001379 print_bits(val, msr_bits, ",");
1380 print_tm_bits(val);
Michael Ellermandb5ba5a2016-11-02 22:20:47 +11001381 pr_cont(">");
Paul Mackerras06d67d52005-10-10 22:29:05 +10001382}
1383
1384#ifdef CONFIG_PPC64
anton@samba.orgf6f7dde2007-03-20 20:38:19 -05001385#define REG "%016lx"
Paul Mackerras06d67d52005-10-10 22:29:05 +10001386#define REGS_PER_LINE 4
1387#define LAST_VOLATILE 13
1388#else
anton@samba.orgf6f7dde2007-03-20 20:38:19 -05001389#define REG "%08lx"
Paul Mackerras06d67d52005-10-10 22:29:05 +10001390#define REGS_PER_LINE 8
1391#define LAST_VOLATILE 12
1392#endif
1393
Paul Mackerras14cf11a2005-09-26 16:04:21 +10001394void show_regs(struct pt_regs * regs)
1395{
1396 int i, trap;
1397
Tejun Heoa43cb952013-04-30 15:27:17 -07001398 show_regs_print_info(KERN_DEFAULT);
1399
Michael Ellermana6036102017-08-23 23:56:24 +10001400 printk("NIP: "REG" LR: "REG" CTR: "REG"\n",
Paul Mackerras06d67d52005-10-10 22:29:05 +10001401 regs->nip, regs->link, regs->ctr);
Michael Ellerman182dc9c2017-12-18 16:33:36 +11001402 printk("REGS: %px TRAP: %04lx %s (%s)\n",
Serge E. Hallyn96b644b2006-10-02 02:18:13 -07001403 regs, regs->trap, print_tainted(), init_utsname()->release);
Michael Ellermana6036102017-08-23 23:56:24 +10001404 printk("MSR: "REG" ", regs->msr);
Michael Neuling801c0b22015-11-20 15:15:32 +11001405 print_msr_bits(regs->msr);
Michael Ellermanf6fc73f2017-08-23 23:56:23 +10001406 pr_cont(" CR: %08lx XER: %08lx\n", regs->ccr, regs->xer);
Paul Mackerras14cf11a2005-09-26 16:04:21 +10001407 trap = TRAP(regs);
Benjamin Herrenschmidt2271db22018-01-12 13:28:49 +11001408 if ((TRAP(regs) != 0xc00) && cpu_has_feature(CPU_FTR_CFAR))
Michael Ellerman7dae8652016-11-03 20:45:26 +11001409 pr_cont("CFAR: "REG" ", regs->orig_gpr3);
Anton Blanchardc5400642013-11-15 15:41:19 +11001410 if (trap == 0x200 || trap == 0x300 || trap == 0x600)
Kumar Galaba28c9a2011-10-06 02:53:38 +00001411#if defined(CONFIG_4xx) || defined(CONFIG_BOOKE)
Michael Ellerman7dae8652016-11-03 20:45:26 +11001412 pr_cont("DEAR: "REG" ESR: "REG" ", regs->dar, regs->dsisr);
Kumar Gala14170782007-07-26 00:46:15 -05001413#else
Michael Ellerman7dae8652016-11-03 20:45:26 +11001414 pr_cont("DAR: "REG" DSISR: %08lx ", regs->dar, regs->dsisr);
Anton Blanchard9db8bcf2013-11-15 15:48:38 +11001415#endif
1416#ifdef CONFIG_PPC64
Nicholas Piggin3130a7b2018-05-10 11:04:24 +10001417 pr_cont("IRQMASK: %lx ", regs->softe);
Anton Blanchard9db8bcf2013-11-15 15:48:38 +11001418#endif
1419#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
Anton Blanchard6d888d12013-11-18 13:19:17 +11001420 if (MSR_TM_ACTIVE(regs->msr))
Michael Ellerman7dae8652016-11-03 20:45:26 +11001421 pr_cont("\nPACATMSCRATCH: %016llx ", get_paca()->tm_scratch);
Kumar Gala14170782007-07-26 00:46:15 -05001422#endif
Paul Mackerras14cf11a2005-09-26 16:04:21 +10001423
1424 for (i = 0; i < 32; i++) {
Paul Mackerras06d67d52005-10-10 22:29:05 +10001425 if ((i % REGS_PER_LINE) == 0)
Michael Ellerman7dae8652016-11-03 20:45:26 +11001426 pr_cont("\nGPR%02d: ", i);
1427 pr_cont(REG " ", regs->gpr[i]);
Paul Mackerras06d67d52005-10-10 22:29:05 +10001428 if (i == LAST_VOLATILE && !FULL_REGS(regs))
Paul Mackerras14cf11a2005-09-26 16:04:21 +10001429 break;
1430 }
Michael Ellerman7dae8652016-11-03 20:45:26 +11001431 pr_cont("\n");
Paul Mackerras14cf11a2005-09-26 16:04:21 +10001432#ifdef CONFIG_KALLSYMS
1433 /*
1434 * Lookup NIP late so we have the best change of getting the
1435 * above info out without failing
1436 */
Benjamin Herrenschmidt058c78f2008-07-07 13:44:31 +10001437 printk("NIP ["REG"] %pS\n", regs->nip, (void *)regs->nip);
1438 printk("LR ["REG"] %pS\n", regs->link, (void *)regs->link);
Paul Mackerras14cf11a2005-09-26 16:04:21 +10001439#endif
1440 show_stack(current, (unsigned long *) regs->gpr[1]);
Paul Mackerras06d67d52005-10-10 22:29:05 +10001441 if (!user_mode(regs))
1442 show_instructions(regs);
Paul Mackerras14cf11a2005-09-26 16:04:21 +10001443}
1444
Paul Mackerras14cf11a2005-09-26 16:04:21 +10001445void flush_thread(void)
1446{
K.Prasade0780b72011-02-10 04:44:35 +00001447#ifdef CONFIG_HAVE_HW_BREAKPOINT
K.Prasad5aae8a52010-06-15 11:35:19 +05301448 flush_ptrace_hw_breakpoint(current);
K.Prasade0780b72011-02-10 04:44:35 +00001449#else /* CONFIG_HAVE_HW_BREAKPOINT */
Dave Kleikamp3bffb652010-02-08 11:51:18 +00001450 set_debug_reg_defaults(&current->thread);
K.Prasade0780b72011-02-10 04:44:35 +00001451#endif /* CONFIG_HAVE_HW_BREAKPOINT */
Paul Mackerras14cf11a2005-09-26 16:04:21 +10001452}
1453
Sukadev Bhattiprolu9d2a4d72017-11-07 18:23:54 -08001454int set_thread_uses_vas(void)
1455{
1456#ifdef CONFIG_PPC_BOOK3S_64
1457 if (!cpu_has_feature(CPU_FTR_ARCH_300))
1458 return -EINVAL;
1459
1460 current->thread.used_vas = 1;
1461
1462 /*
1463 * Even a process that has no foreign real address mapping can use
1464 * an unpaired COPY instruction (to no real effect). Issue CP_ABORT
1465 * to clear any pending COPY and prevent a covert channel.
1466 *
1467 * __switch_to() will issue CP_ABORT on future context switches.
1468 */
1469 asm volatile(PPC_CP_ABORT);
1470
1471#endif /* CONFIG_PPC_BOOK3S_64 */
1472 return 0;
1473}
1474
Sukadev Bhattiproluec233ed2017-11-07 18:23:53 -08001475#ifdef CONFIG_PPC64
Alastair D'Silva71cc64a2018-05-11 16:12:59 +10001476/**
1477 * Assign a TIDR (thread ID) for task @t and set it in the thread
Sukadev Bhattiproluec233ed2017-11-07 18:23:53 -08001478 * structure. For now, we only support setting TIDR for 'current' task.
Alastair D'Silva71cc64a2018-05-11 16:12:59 +10001479 *
1480 * Since the TID value is a truncated form of it PID, it is possible
1481 * (but unlikely) for 2 threads to have the same TID. In the unlikely event
1482 * that 2 threads share the same TID and are waiting, one of the following
1483 * cases will happen:
1484 *
1485 * 1. The correct thread is running, the wrong thread is not
1486 * In this situation, the correct thread is woken and proceeds to pass it's
1487 * condition check.
1488 *
1489 * 2. Neither threads are running
1490 * In this situation, neither thread will be woken. When scheduled, the waiting
1491 * threads will execute either a wait, which will return immediately, followed
1492 * by a condition check, which will pass for the correct thread and fail
1493 * for the wrong thread, or they will execute the condition check immediately.
1494 *
1495 * 3. The wrong thread is running, the correct thread is not
1496 * The wrong thread will be woken, but will fail it's condition check and
1497 * re-execute wait. The correct thread, when scheduled, will execute either
1498 * it's condition check (which will pass), or wait, which returns immediately
1499 * when called the first time after the thread is scheduled, followed by it's
1500 * condition check (which will pass).
1501 *
1502 * 4. Both threads are running
1503 * Both threads will be woken. The wrong thread will fail it's condition check
1504 * and execute another wait, while the correct thread will pass it's condition
1505 * check.
1506 *
1507 * @t: the task to set the thread ID for
Sukadev Bhattiproluec233ed2017-11-07 18:23:53 -08001508 */
1509int set_thread_tidr(struct task_struct *t)
1510{
Alastair D'Silva3449f192018-05-11 16:12:58 +10001511 if (!cpu_has_feature(CPU_FTR_P9_TIDR))
Sukadev Bhattiproluec233ed2017-11-07 18:23:53 -08001512 return -EINVAL;
1513
1514 if (t != current)
1515 return -EINVAL;
1516
Vaibhav Jain7e4d4232017-11-24 14:03:38 +05301517 if (t->thread.tidr)
1518 return 0;
1519
Alastair D'Silva71cc64a2018-05-11 16:12:59 +10001520 t->thread.tidr = (u16)task_pid_nr(t);
Sukadev Bhattiproluec233ed2017-11-07 18:23:53 -08001521 mtspr(SPRN_TIDR, t->thread.tidr);
1522
1523 return 0;
1524}
Christophe Lombardb1db5512018-01-11 09:55:25 +01001525EXPORT_SYMBOL_GPL(set_thread_tidr);
Sukadev Bhattiproluec233ed2017-11-07 18:23:53 -08001526
1527#endif /* CONFIG_PPC64 */
1528
Paul Mackerras14cf11a2005-09-26 16:04:21 +10001529void
1530release_thread(struct task_struct *t)
1531{
1532}
1533
1534/*
Suresh Siddha55ccf3f2012-05-16 15:03:51 -07001535 * this gets called so that we can store coprocessor state into memory and
1536 * copy the current task into the new thread.
Paul Mackerras14cf11a2005-09-26 16:04:21 +10001537 */
Suresh Siddha55ccf3f2012-05-16 15:03:51 -07001538int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src)
Paul Mackerras14cf11a2005-09-26 16:04:21 +10001539{
Anton Blanchard579e6332015-10-29 11:44:09 +11001540 flush_all_to_thread(src);
Michael Neuling621b5062014-03-03 14:21:40 +11001541 /*
1542 * Flush TM state out so we can copy it. __switch_to_tm() does this
1543 * flush but it removes the checkpointed state from the current CPU and
1544 * transitions the CPU out of TM mode. Hence we need to call
1545 * tm_recheckpoint_new_task() (on the same task) to restore the
1546 * checkpointed state back and the TM mode.
Cyril Bur5d176f72016-09-14 18:02:16 +10001547 *
1548 * Can't pass dst because it isn't ready. Doesn't matter, passing
1549 * dst is only important for __switch_to()
Michael Neuling621b5062014-03-03 14:21:40 +11001550 */
Cyril Burdc310662016-09-23 16:18:24 +10001551 __switch_to_tm(src, src);
Michael Ellerman330a1eb2013-06-28 18:15:16 +10001552
Suresh Siddha55ccf3f2012-05-16 15:03:51 -07001553 *dst = *src;
Michael Ellerman330a1eb2013-06-28 18:15:16 +10001554
1555 clear_task_ebb(dst);
1556
Suresh Siddha55ccf3f2012-05-16 15:03:51 -07001557 return 0;
Paul Mackerras14cf11a2005-09-26 16:04:21 +10001558}
1559
Michael Ellermancec15482014-07-10 12:29:21 +10001560static void setup_ksp_vsid(struct task_struct *p, unsigned long sp)
1561{
Michael Ellerman4e003742017-10-19 15:08:43 +11001562#ifdef CONFIG_PPC_BOOK3S_64
Michael Ellermancec15482014-07-10 12:29:21 +10001563 unsigned long sp_vsid;
1564 unsigned long llp = mmu_psize_defs[mmu_linear_psize].sllp;
1565
Aneesh Kumar K.Vcaca2852016-04-29 23:26:07 +10001566 if (radix_enabled())
1567 return;
1568
Michael Ellermancec15482014-07-10 12:29:21 +10001569 if (mmu_has_feature(MMU_FTR_1T_SEGMENT))
1570 sp_vsid = get_kernel_vsid(sp, MMU_SEGSIZE_1T)
1571 << SLB_VSID_SHIFT_1T;
1572 else
1573 sp_vsid = get_kernel_vsid(sp, MMU_SEGSIZE_256M)
1574 << SLB_VSID_SHIFT;
1575 sp_vsid |= SLB_VSID_KERNEL | llp;
1576 p->thread.ksp_vsid = sp_vsid;
1577#endif
1578}
1579
Paul Mackerras14cf11a2005-09-26 16:04:21 +10001580/*
1581 * Copy a thread..
1582 */
Alexey Kardashevskiyefcac652011-03-02 15:18:48 +00001583
Alex Dowad6eca8932015-03-13 20:14:46 +02001584/*
1585 * Copy architecture-specific thread state
1586 */
Alexey Dobriyan6f2c55b2009-04-02 16:56:59 -07001587int copy_thread(unsigned long clone_flags, unsigned long usp,
Alex Dowad6eca8932015-03-13 20:14:46 +02001588 unsigned long kthread_arg, struct task_struct *p)
Paul Mackerras14cf11a2005-09-26 16:04:21 +10001589{
1590 struct pt_regs *childregs, *kregs;
1591 extern void ret_from_fork(void);
Al Viro58254e12012-09-12 18:32:42 -04001592 extern void ret_from_kernel_thread(void);
1593 void (*f)(void);
Al Viro0cec6fd2006-01-12 01:06:02 -08001594 unsigned long sp = (unsigned long)task_stack_page(p) + THREAD_SIZE;
Michael Ellerman5d31a962016-03-24 22:04:04 +11001595 struct thread_info *ti = task_thread_info(p);
1596
1597 klp_init_thread_info(ti);
Paul Mackerras14cf11a2005-09-26 16:04:21 +10001598
Paul Mackerras14cf11a2005-09-26 16:04:21 +10001599 /* Copy registers */
1600 sp -= sizeof(struct pt_regs);
1601 childregs = (struct pt_regs *) sp;
Al Viroab758192012-10-21 22:33:39 -04001602 if (unlikely(p->flags & PF_KTHREAD)) {
Alex Dowad6eca8932015-03-13 20:14:46 +02001603 /* kernel thread */
Al Viro58254e12012-09-12 18:32:42 -04001604 memset(childregs, 0, sizeof(struct pt_regs));
Paul Mackerras14cf11a2005-09-26 16:04:21 +10001605 childregs->gpr[1] = sp + sizeof(struct pt_regs);
Anton Blanchard7cedd602014-02-04 16:08:51 +11001606 /* function */
1607 if (usp)
1608 childregs->gpr[14] = ppc_function_entry((void *)usp);
Al Viro58254e12012-09-12 18:32:42 -04001609#ifdef CONFIG_PPC64
Al Virob5e2fc12006-01-12 01:06:01 -08001610 clear_tsk_thread_flag(p, TIF_32BIT);
Madhavan Srinivasanc2e480b2017-12-20 09:25:42 +05301611 childregs->softe = IRQS_ENABLED;
Paul Mackerras06d67d52005-10-10 22:29:05 +10001612#endif
Alex Dowad6eca8932015-03-13 20:14:46 +02001613 childregs->gpr[15] = kthread_arg;
Paul Mackerras14cf11a2005-09-26 16:04:21 +10001614 p->thread.regs = NULL; /* no user register state */
Al Viro138d1ce2012-10-11 08:41:43 -04001615 ti->flags |= _TIF_RESTOREALL;
Al Viro58254e12012-09-12 18:32:42 -04001616 f = ret_from_kernel_thread;
Paul Mackerras14cf11a2005-09-26 16:04:21 +10001617 } else {
Alex Dowad6eca8932015-03-13 20:14:46 +02001618 /* user thread */
Al Viroafa86fc2012-10-22 22:51:14 -04001619 struct pt_regs *regs = current_pt_regs();
Al Viro58254e12012-09-12 18:32:42 -04001620 CHECK_FULL_REGS(regs);
1621 *childregs = *regs;
Al Viroea516b12012-10-21 22:28:43 -04001622 if (usp)
1623 childregs->gpr[1] = usp;
Paul Mackerras14cf11a2005-09-26 16:04:21 +10001624 p->thread.regs = childregs;
Al Viro58254e12012-09-12 18:32:42 -04001625 childregs->gpr[3] = 0; /* Result from fork() */
Paul Mackerras06d67d52005-10-10 22:29:05 +10001626 if (clone_flags & CLONE_SETTLS) {
1627#ifdef CONFIG_PPC64
Denis Kirjanov9904b002010-07-29 22:04:39 +00001628 if (!is_32bit_task())
Paul Mackerras06d67d52005-10-10 22:29:05 +10001629 childregs->gpr[13] = childregs->gpr[6];
1630 else
1631#endif
1632 childregs->gpr[2] = childregs->gpr[6];
1633 }
Al Viro58254e12012-09-12 18:32:42 -04001634
1635 f = ret_from_fork;
Paul Mackerras14cf11a2005-09-26 16:04:21 +10001636 }
Cyril Burd272f662016-02-29 17:53:46 +11001637 childregs->msr &= ~(MSR_FP|MSR_VEC|MSR_VSX);
Paul Mackerras14cf11a2005-09-26 16:04:21 +10001638 sp -= STACK_FRAME_OVERHEAD;
Paul Mackerras14cf11a2005-09-26 16:04:21 +10001639
1640 /*
1641 * The way this works is that at some point in the future
1642 * some task will call _switch to switch to the new task.
1643 * That will pop off the stack frame created below and start
1644 * the new task running at ret_from_fork. The new task will
1645 * do some house keeping and then return from the fork or clone
1646 * system call, using the stack frame created above.
1647 */
Li Zhongaf945cf2013-05-06 22:44:41 +00001648 ((unsigned long *)sp)[0] = 0;
Paul Mackerras14cf11a2005-09-26 16:04:21 +10001649 sp -= sizeof(struct pt_regs);
1650 kregs = (struct pt_regs *) sp;
1651 sp -= STACK_FRAME_OVERHEAD;
1652 p->thread.ksp = sp;
Benjamin Herrenschmidtcbc95652013-09-24 15:17:21 +10001653#ifdef CONFIG_PPC32
Kumar Gala85218822008-04-28 16:21:22 +10001654 p->thread.ksp_limit = (unsigned long)task_stack_page(p) +
1655 _ALIGN_UP(sizeof(struct thread_info), 16);
Benjamin Herrenschmidtcbc95652013-09-24 15:17:21 +10001656#endif
Oleg Nesterov28d170ab2013-04-21 06:47:59 +00001657#ifdef CONFIG_HAVE_HW_BREAKPOINT
1658 p->thread.ptrace_bps[0] = NULL;
1659#endif
1660
Paul Mackerras18461962013-09-10 20:21:10 +10001661 p->thread.fp_save_area = NULL;
1662#ifdef CONFIG_ALTIVEC
1663 p->thread.vr_save_area = NULL;
1664#endif
1665
Michael Ellermancec15482014-07-10 12:29:21 +10001666 setup_ksp_vsid(p, sp);
Paul Mackerras06d67d52005-10-10 22:29:05 +10001667
Alexey Kardashevskiyefcac652011-03-02 15:18:48 +00001668#ifdef CONFIG_PPC64
1669 if (cpu_has_feature(CPU_FTR_DSCR)) {
Anton Blanchard1021cb22012-09-03 16:49:47 +00001670 p->thread.dscr_inherit = current->thread.dscr_inherit;
Anton Blancharddb1231dc2015-12-09 20:11:47 +11001671 p->thread.dscr = mfspr(SPRN_DSCR);
Alexey Kardashevskiyefcac652011-03-02 15:18:48 +00001672 }
Haren Myneni92779242012-12-06 21:49:56 +00001673 if (cpu_has_feature(CPU_FTR_HAS_PPR))
1674 p->thread.ppr = INIT_PPR;
Sukadev Bhattiproluec233ed2017-11-07 18:23:53 -08001675
1676 p->thread.tidr = 0;
Alexey Kardashevskiyefcac652011-03-02 15:18:48 +00001677#endif
Anton Blanchard7cedd602014-02-04 16:08:51 +11001678 kregs->nip = ppc_function_entry(f);
Paul Mackerras14cf11a2005-09-26 16:04:21 +10001679 return 0;
1680}
1681
1682/*
1683 * Set up a thread for executing a new program
1684 */
Paul Mackerras06d67d52005-10-10 22:29:05 +10001685void start_thread(struct pt_regs *regs, unsigned long start, unsigned long sp)
Paul Mackerras14cf11a2005-09-26 16:04:21 +10001686{
Michael Ellerman90eac722005-10-21 16:01:33 +10001687#ifdef CONFIG_PPC64
1688 unsigned long load_addr = regs->gpr[2]; /* saved by ELF_PLAT_INIT */
1689#endif
1690
Paul Mackerras06d67d52005-10-10 22:29:05 +10001691 /*
1692 * If we exec out of a kernel thread then thread.regs will not be
1693 * set. Do it now.
1694 */
1695 if (!current->thread.regs) {
Al Viro0cec6fd2006-01-12 01:06:02 -08001696 struct pt_regs *regs = task_stack_page(current) + THREAD_SIZE;
1697 current->thread.regs = regs - 1;
Paul Mackerras06d67d52005-10-10 22:29:05 +10001698 }
1699
Cyril Bur8e96a872016-06-17 14:58:34 +10001700#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
1701 /*
1702 * Clear any transactional state, we're exec()ing. The cause is
1703 * not important as there will never be a recheckpoint so it's not
1704 * user visible.
1705 */
1706 if (MSR_TM_SUSPENDED(mfmsr()))
1707 tm_reclaim_current(0);
1708#endif
1709
Paul Mackerras14cf11a2005-09-26 16:04:21 +10001710 memset(regs->gpr, 0, sizeof(regs->gpr));
1711 regs->ctr = 0;
1712 regs->link = 0;
1713 regs->xer = 0;
1714 regs->ccr = 0;
Paul Mackerras14cf11a2005-09-26 16:04:21 +10001715 regs->gpr[1] = sp;
Paul Mackerras06d67d52005-10-10 22:29:05 +10001716
Roland McGrath474f8192007-09-24 16:52:44 -07001717 /*
1718 * We have just cleared all the nonvolatile GPRs, so make
1719 * FULL_REGS(regs) return true. This is necessary to allow
1720 * ptrace to examine the thread immediately after exec.
1721 */
1722 regs->trap &= ~1UL;
1723
Paul Mackerras06d67d52005-10-10 22:29:05 +10001724#ifdef CONFIG_PPC32
1725 regs->mq = 0;
1726 regs->nip = start;
Paul Mackerras14cf11a2005-09-26 16:04:21 +10001727 regs->msr = MSR_USER;
Paul Mackerras06d67d52005-10-10 22:29:05 +10001728#else
Denis Kirjanov9904b002010-07-29 22:04:39 +00001729 if (!is_32bit_task()) {
Rusty Russell94af3ab2013-11-20 22:15:02 +11001730 unsigned long entry;
Paul Mackerras06d67d52005-10-10 22:29:05 +10001731
Rusty Russell94af3ab2013-11-20 22:15:02 +11001732 if (is_elf2_task()) {
1733 /* Look ma, no function descriptors! */
1734 entry = start;
Paul Mackerras06d67d52005-10-10 22:29:05 +10001735
Rusty Russell94af3ab2013-11-20 22:15:02 +11001736 /*
1737 * Ulrich says:
1738 * The latest iteration of the ABI requires that when
1739 * calling a function (at its global entry point),
1740 * the caller must ensure r12 holds the entry point
1741 * address (so that the function can quickly
1742 * establish addressability).
1743 */
1744 regs->gpr[12] = start;
1745 /* Make sure that's restored on entry to userspace. */
1746 set_thread_flag(TIF_RESTOREALL);
1747 } else {
1748 unsigned long toc;
1749
1750 /* start is a relocated pointer to the function
1751 * descriptor for the elf _start routine. The first
1752 * entry in the function descriptor is the entry
1753 * address of _start and the second entry is the TOC
1754 * value we need to use.
1755 */
1756 __get_user(entry, (unsigned long __user *)start);
1757 __get_user(toc, (unsigned long __user *)start+1);
1758
1759 /* Check whether the e_entry function descriptor entries
1760 * need to be relocated before we can use them.
1761 */
1762 if (load_addr != 0) {
1763 entry += load_addr;
1764 toc += load_addr;
1765 }
1766 regs->gpr[2] = toc;
Paul Mackerras06d67d52005-10-10 22:29:05 +10001767 }
1768 regs->nip = entry;
Paul Mackerras06d67d52005-10-10 22:29:05 +10001769 regs->msr = MSR_USER64;
Stephen Rothwelld4bf9a72005-10-13 13:40:54 +10001770 } else {
1771 regs->nip = start;
1772 regs->gpr[2] = 0;
1773 regs->msr = MSR_USER32;
Paul Mackerras06d67d52005-10-10 22:29:05 +10001774 }
1775#endif
Michael Neulingce48b212008-06-25 14:07:18 +10001776#ifdef CONFIG_VSX
1777 current->thread.used_vsr = 0;
1778#endif
Breno Leitao11958922017-06-02 18:43:30 -03001779 current->thread.load_fp = 0;
Paul Mackerrasde79f7b2013-09-10 20:20:42 +10001780 memset(&current->thread.fp_state, 0, sizeof(current->thread.fp_state));
Paul Mackerras18461962013-09-10 20:21:10 +10001781 current->thread.fp_save_area = NULL;
Paul Mackerras14cf11a2005-09-26 16:04:21 +10001782#ifdef CONFIG_ALTIVEC
Paul Mackerrasde79f7b2013-09-10 20:20:42 +10001783 memset(&current->thread.vr_state, 0, sizeof(current->thread.vr_state));
1784 current->thread.vr_state.vscr.u[3] = 0x00010000; /* Java mode disabled */
Paul Mackerras18461962013-09-10 20:21:10 +10001785 current->thread.vr_save_area = NULL;
Paul Mackerras14cf11a2005-09-26 16:04:21 +10001786 current->thread.vrsave = 0;
1787 current->thread.used_vr = 0;
Breno Leitao11958922017-06-02 18:43:30 -03001788 current->thread.load_vec = 0;
Paul Mackerras14cf11a2005-09-26 16:04:21 +10001789#endif /* CONFIG_ALTIVEC */
1790#ifdef CONFIG_SPE
1791 memset(current->thread.evr, 0, sizeof(current->thread.evr));
1792 current->thread.acc = 0;
1793 current->thread.spefscr = 0;
1794 current->thread.used_spe = 0;
1795#endif /* CONFIG_SPE */
Michael Neulingbc2a9402013-02-13 16:21:40 +00001796#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
Michael Neulingbc2a9402013-02-13 16:21:40 +00001797 current->thread.tm_tfhar = 0;
1798 current->thread.tm_texasr = 0;
1799 current->thread.tm_tfiar = 0;
Breno Leitao7f22ced2017-06-05 11:40:59 -03001800 current->thread.load_tm = 0;
Michael Neulingbc2a9402013-02-13 16:21:40 +00001801#endif /* CONFIG_PPC_TRANSACTIONAL_MEM */
Ram Pai06bb53b2018-01-18 17:50:31 -08001802
1803 thread_pkey_regs_init(&current->thread);
Paul Mackerras14cf11a2005-09-26 16:04:21 +10001804}
Anton Blancharde1802b02014-08-20 08:00:02 +10001805EXPORT_SYMBOL(start_thread);
Paul Mackerras14cf11a2005-09-26 16:04:21 +10001806
1807#define PR_FP_ALL_EXCEPT (PR_FP_EXC_DIV | PR_FP_EXC_OVF | PR_FP_EXC_UND \
1808 | PR_FP_EXC_RES | PR_FP_EXC_INV)
1809
1810int set_fpexc_mode(struct task_struct *tsk, unsigned int val)
1811{
1812 struct pt_regs *regs = tsk->thread.regs;
1813
1814 /* This is a bit hairy. If we are an SPE enabled processor
1815 * (have embedded fp) we store the IEEE exception enable flags in
1816 * fpexc_mode. fpexc_mode is also used for setting FP exception
1817 * mode (asyn, precise, disabled) for 'Classic' FP. */
1818 if (val & PR_FP_EXC_SW_ENABLE) {
1819#ifdef CONFIG_SPE
Kumar Gala5e14d212007-09-13 01:44:20 -05001820 if (cpu_has_feature(CPU_FTR_SPE)) {
Joseph Myers640e9222013-12-10 23:07:45 +00001821 /*
1822 * When the sticky exception bits are set
1823 * directly by userspace, it must call prctl
1824 * with PR_GET_FPEXC (with PR_FP_EXC_SW_ENABLE
1825 * in the existing prctl settings) or
1826 * PR_SET_FPEXC (with PR_FP_EXC_SW_ENABLE in
1827 * the bits being set). <fenv.h> functions
1828 * saving and restoring the whole
1829 * floating-point environment need to do so
1830 * anyway to restore the prctl settings from
1831 * the saved environment.
1832 */
1833 tsk->thread.spefscr_last = mfspr(SPRN_SPEFSCR);
Kumar Gala5e14d212007-09-13 01:44:20 -05001834 tsk->thread.fpexc_mode = val &
1835 (PR_FP_EXC_SW_ENABLE | PR_FP_ALL_EXCEPT);
1836 return 0;
1837 } else {
1838 return -EINVAL;
1839 }
Paul Mackerras14cf11a2005-09-26 16:04:21 +10001840#else
1841 return -EINVAL;
1842#endif
Paul Mackerras14cf11a2005-09-26 16:04:21 +10001843 }
Paul Mackerras06d67d52005-10-10 22:29:05 +10001844
1845 /* on a CONFIG_SPE this does not hurt us. The bits that
1846 * __pack_fe01 use do not overlap with bits used for
1847 * PR_FP_EXC_SW_ENABLE. Additionally, the MSR[FE0,FE1] bits
1848 * on CONFIG_SPE implementations are reserved so writing to
1849 * them does not change anything */
1850 if (val > PR_FP_EXC_PRECISE)
1851 return -EINVAL;
1852 tsk->thread.fpexc_mode = __pack_fe01(val);
1853 if (regs != NULL && (regs->msr & MSR_FP) != 0)
1854 regs->msr = (regs->msr & ~(MSR_FE0|MSR_FE1))
1855 | tsk->thread.fpexc_mode;
Paul Mackerras14cf11a2005-09-26 16:04:21 +10001856 return 0;
1857}
1858
1859int get_fpexc_mode(struct task_struct *tsk, unsigned long adr)
1860{
1861 unsigned int val;
1862
1863 if (tsk->thread.fpexc_mode & PR_FP_EXC_SW_ENABLE)
1864#ifdef CONFIG_SPE
Joseph Myers640e9222013-12-10 23:07:45 +00001865 if (cpu_has_feature(CPU_FTR_SPE)) {
1866 /*
1867 * When the sticky exception bits are set
1868 * directly by userspace, it must call prctl
1869 * with PR_GET_FPEXC (with PR_FP_EXC_SW_ENABLE
1870 * in the existing prctl settings) or
1871 * PR_SET_FPEXC (with PR_FP_EXC_SW_ENABLE in
1872 * the bits being set). <fenv.h> functions
1873 * saving and restoring the whole
1874 * floating-point environment need to do so
1875 * anyway to restore the prctl settings from
1876 * the saved environment.
1877 */
1878 tsk->thread.spefscr_last = mfspr(SPRN_SPEFSCR);
Kumar Gala5e14d212007-09-13 01:44:20 -05001879 val = tsk->thread.fpexc_mode;
Joseph Myers640e9222013-12-10 23:07:45 +00001880 } else
Kumar Gala5e14d212007-09-13 01:44:20 -05001881 return -EINVAL;
Paul Mackerras14cf11a2005-09-26 16:04:21 +10001882#else
1883 return -EINVAL;
1884#endif
1885 else
1886 val = __unpack_fe01(tsk->thread.fpexc_mode);
1887 return put_user(val, (unsigned int __user *) adr);
1888}
1889
Paul Mackerrasfab5db92006-06-07 16:14:40 +10001890int set_endian(struct task_struct *tsk, unsigned int val)
1891{
1892 struct pt_regs *regs = tsk->thread.regs;
1893
1894 if ((val == PR_ENDIAN_LITTLE && !cpu_has_feature(CPU_FTR_REAL_LE)) ||
1895 (val == PR_ENDIAN_PPC_LITTLE && !cpu_has_feature(CPU_FTR_PPC_LE)))
1896 return -EINVAL;
1897
1898 if (regs == NULL)
1899 return -EINVAL;
1900
1901 if (val == PR_ENDIAN_BIG)
1902 regs->msr &= ~MSR_LE;
1903 else if (val == PR_ENDIAN_LITTLE || val == PR_ENDIAN_PPC_LITTLE)
1904 regs->msr |= MSR_LE;
1905 else
1906 return -EINVAL;
1907
1908 return 0;
1909}
1910
1911int get_endian(struct task_struct *tsk, unsigned long adr)
1912{
1913 struct pt_regs *regs = tsk->thread.regs;
1914 unsigned int val;
1915
1916 if (!cpu_has_feature(CPU_FTR_PPC_LE) &&
1917 !cpu_has_feature(CPU_FTR_REAL_LE))
1918 return -EINVAL;
1919
1920 if (regs == NULL)
1921 return -EINVAL;
1922
1923 if (regs->msr & MSR_LE) {
1924 if (cpu_has_feature(CPU_FTR_REAL_LE))
1925 val = PR_ENDIAN_LITTLE;
1926 else
1927 val = PR_ENDIAN_PPC_LITTLE;
1928 } else
1929 val = PR_ENDIAN_BIG;
1930
1931 return put_user(val, (unsigned int __user *)adr);
1932}
1933
Paul Mackerrase9370ae2006-06-07 16:15:39 +10001934int set_unalign_ctl(struct task_struct *tsk, unsigned int val)
1935{
1936 tsk->thread.align_ctl = val;
1937 return 0;
1938}
1939
1940int get_unalign_ctl(struct task_struct *tsk, unsigned long adr)
1941{
1942 return put_user(tsk->thread.align_ctl, (unsigned int __user *)adr);
1943}
1944
Paul Mackerrasbb72c482007-02-19 11:42:42 +11001945static inline int valid_irq_stack(unsigned long sp, struct task_struct *p,
1946 unsigned long nbytes)
1947{
1948 unsigned long stack_page;
1949 unsigned long cpu = task_cpu(p);
1950
1951 /*
1952 * Avoid crashing if the stack has overflowed and corrupted
1953 * task_cpu(p), which is in the thread_info struct.
1954 */
1955 if (cpu < NR_CPUS && cpu_possible(cpu)) {
1956 stack_page = (unsigned long) hardirq_ctx[cpu];
1957 if (sp >= stack_page + sizeof(struct thread_struct)
1958 && sp <= stack_page + THREAD_SIZE - nbytes)
1959 return 1;
1960
1961 stack_page = (unsigned long) softirq_ctx[cpu];
1962 if (sp >= stack_page + sizeof(struct thread_struct)
1963 && sp <= stack_page + THREAD_SIZE - nbytes)
1964 return 1;
1965 }
1966 return 0;
1967}
1968
Anton Blanchard2f251942006-03-27 11:46:18 +11001969int validate_sp(unsigned long sp, struct task_struct *p,
Paul Mackerras14cf11a2005-09-26 16:04:21 +10001970 unsigned long nbytes)
1971{
Al Viro0cec6fd2006-01-12 01:06:02 -08001972 unsigned long stack_page = (unsigned long)task_stack_page(p);
Paul Mackerras14cf11a2005-09-26 16:04:21 +10001973
1974 if (sp >= stack_page + sizeof(struct thread_struct)
1975 && sp <= stack_page + THREAD_SIZE - nbytes)
1976 return 1;
1977
Paul Mackerrasbb72c482007-02-19 11:42:42 +11001978 return valid_irq_stack(sp, p, nbytes);
Paul Mackerras14cf11a2005-09-26 16:04:21 +10001979}
1980
Anton Blanchard2f251942006-03-27 11:46:18 +11001981EXPORT_SYMBOL(validate_sp);
1982
Paul Mackerras06d67d52005-10-10 22:29:05 +10001983unsigned long get_wchan(struct task_struct *p)
1984{
1985 unsigned long ip, sp;
1986 int count = 0;
1987
1988 if (!p || p == current || p->state == TASK_RUNNING)
1989 return 0;
1990
1991 sp = p->thread.ksp;
Benjamin Herrenschmidtec2b36b2008-04-17 14:34:59 +10001992 if (!validate_sp(sp, p, STACK_FRAME_OVERHEAD))
Paul Mackerras06d67d52005-10-10 22:29:05 +10001993 return 0;
1994
1995 do {
1996 sp = *(unsigned long *)sp;
Kautuk Consul4ca360f2016-04-19 15:48:21 +05301997 if (!validate_sp(sp, p, STACK_FRAME_OVERHEAD) ||
1998 p->state == TASK_RUNNING)
Paul Mackerras06d67d52005-10-10 22:29:05 +10001999 return 0;
2000 if (count > 0) {
Benjamin Herrenschmidtec2b36b2008-04-17 14:34:59 +10002001 ip = ((unsigned long *)sp)[STACK_FRAME_LR_SAVE];
Paul Mackerras06d67d52005-10-10 22:29:05 +10002002 if (!in_sched_functions(ip))
2003 return ip;
2004 }
2005 } while (count++ < 16);
2006 return 0;
2007}
Paul Mackerras06d67d52005-10-10 22:29:05 +10002008
Johannes Bergc4d04be2008-11-20 03:24:07 +00002009static int kstack_depth_to_print = CONFIG_PRINT_STACK_DEPTH;
Paul Mackerras14cf11a2005-09-26 16:04:21 +10002010
2011void show_stack(struct task_struct *tsk, unsigned long *stack)
2012{
Paul Mackerras06d67d52005-10-10 22:29:05 +10002013 unsigned long sp, ip, lr, newsp;
Paul Mackerras14cf11a2005-09-26 16:04:21 +10002014 int count = 0;
Paul Mackerras06d67d52005-10-10 22:29:05 +10002015 int firstframe = 1;
Steven Rostedt6794c782009-02-09 21:10:27 -08002016#ifdef CONFIG_FUNCTION_GRAPH_TRACER
2017 int curr_frame = current->curr_ret_stack;
2018 extern void return_to_handler(void);
Steven Rostedt9135c3c2009-09-15 08:20:15 -07002019 unsigned long rth = (unsigned long)return_to_handler;
Steven Rostedt6794c782009-02-09 21:10:27 -08002020#endif
Paul Mackerras14cf11a2005-09-26 16:04:21 +10002021
2022 sp = (unsigned long) stack;
2023 if (tsk == NULL)
2024 tsk = current;
2025 if (sp == 0) {
2026 if (tsk == current)
Anton Blanchardacf620e2014-10-13 19:41:39 +11002027 sp = current_stack_pointer();
Paul Mackerras14cf11a2005-09-26 16:04:21 +10002028 else
2029 sp = tsk->thread.ksp;
2030 }
2031
Paul Mackerras06d67d52005-10-10 22:29:05 +10002032 lr = 0;
2033 printk("Call Trace:\n");
Paul Mackerras14cf11a2005-09-26 16:04:21 +10002034 do {
Benjamin Herrenschmidtec2b36b2008-04-17 14:34:59 +10002035 if (!validate_sp(sp, tsk, STACK_FRAME_OVERHEAD))
Paul Mackerras06d67d52005-10-10 22:29:05 +10002036 return;
2037
2038 stack = (unsigned long *) sp;
2039 newsp = stack[0];
Benjamin Herrenschmidtec2b36b2008-04-17 14:34:59 +10002040 ip = stack[STACK_FRAME_LR_SAVE];
Paul Mackerras06d67d52005-10-10 22:29:05 +10002041 if (!firstframe || ip != lr) {
Benjamin Herrenschmidt058c78f2008-07-07 13:44:31 +10002042 printk("["REG"] ["REG"] %pS", sp, ip, (void *)ip);
Steven Rostedt6794c782009-02-09 21:10:27 -08002043#ifdef CONFIG_FUNCTION_GRAPH_TRACER
Anton Blanchard7d56c652014-09-17 17:07:03 +10002044 if ((ip == rth) && curr_frame >= 0) {
Michael Ellerman9a1f4902016-11-02 22:20:46 +11002045 pr_cont(" (%pS)",
Steven Rostedt6794c782009-02-09 21:10:27 -08002046 (void *)current->ret_stack[curr_frame].ret);
2047 curr_frame--;
2048 }
2049#endif
Paul Mackerras06d67d52005-10-10 22:29:05 +10002050 if (firstframe)
Michael Ellerman9a1f4902016-11-02 22:20:46 +11002051 pr_cont(" (unreliable)");
2052 pr_cont("\n");
Paul Mackerras14cf11a2005-09-26 16:04:21 +10002053 }
Paul Mackerras06d67d52005-10-10 22:29:05 +10002054 firstframe = 0;
2055
2056 /*
2057 * See if this is an exception frame.
2058 * We look for the "regshere" marker in the current frame.
2059 */
Benjamin Herrenschmidtec2b36b2008-04-17 14:34:59 +10002060 if (validate_sp(sp, tsk, STACK_INT_FRAME_SIZE)
2061 && stack[STACK_FRAME_MARKER] == STACK_FRAME_REGS_MARKER) {
Paul Mackerras06d67d52005-10-10 22:29:05 +10002062 struct pt_regs *regs = (struct pt_regs *)
2063 (sp + STACK_FRAME_OVERHEAD);
Paul Mackerras06d67d52005-10-10 22:29:05 +10002064 lr = regs->link;
Paul Mackerras9be9be22014-06-12 16:53:08 +10002065 printk("--- interrupt: %lx at %pS\n LR = %pS\n",
Benjamin Herrenschmidt058c78f2008-07-07 13:44:31 +10002066 regs->trap, (void *)regs->nip, (void *)lr);
Paul Mackerras06d67d52005-10-10 22:29:05 +10002067 firstframe = 1;
2068 }
2069
2070 sp = newsp;
2071 } while (count++ < kstack_depth_to_print);
Paul Mackerras14cf11a2005-09-26 16:04:21 +10002072}
Paul Mackerras06d67d52005-10-10 22:29:05 +10002073
Anton Blanchardcb2c9b22006-02-13 14:48:35 +11002074#ifdef CONFIG_PPC64
Benjamin Herrenschmidtfe1952f2012-03-01 12:45:27 +11002075/* Called with hard IRQs off */
Michael Ellerman0e377392013-06-13 21:04:56 +10002076void notrace __ppc64_runlatch_on(void)
Anton Blanchardcb2c9b22006-02-13 14:48:35 +11002077{
Benjamin Herrenschmidtfe1952f2012-03-01 12:45:27 +11002078 struct thread_info *ti = current_thread_info();
Anton Blanchardcb2c9b22006-02-13 14:48:35 +11002079
Nicholas Piggind1d0d5f2017-08-12 02:39:07 +10002080 if (cpu_has_feature(CPU_FTR_ARCH_206)) {
2081 /*
2082 * Least significant bit (RUN) is the only writable bit of
2083 * the CTRL register, so we can avoid mfspr. 2.06 is not the
2084 * earliest ISA where this is the case, but it's convenient.
2085 */
2086 mtspr(SPRN_CTRLT, CTRL_RUNLATCH);
2087 } else {
2088 unsigned long ctrl;
2089
2090 /*
2091 * Some architectures (e.g., Cell) have writable fields other
2092 * than RUN, so do the read-modify-write.
2093 */
2094 ctrl = mfspr(SPRN_CTRLF);
2095 ctrl |= CTRL_RUNLATCH;
2096 mtspr(SPRN_CTRLT, ctrl);
2097 }
Anton Blanchardcb2c9b22006-02-13 14:48:35 +11002098
Benjamin Herrenschmidtfae2e0f2012-04-11 10:42:15 +10002099 ti->local_flags |= _TLF_RUNLATCH;
Anton Blanchardcb2c9b22006-02-13 14:48:35 +11002100}
2101
Benjamin Herrenschmidtfe1952f2012-03-01 12:45:27 +11002102/* Called with hard IRQs off */
Michael Ellerman0e377392013-06-13 21:04:56 +10002103void notrace __ppc64_runlatch_off(void)
Anton Blanchardcb2c9b22006-02-13 14:48:35 +11002104{
Benjamin Herrenschmidtfe1952f2012-03-01 12:45:27 +11002105 struct thread_info *ti = current_thread_info();
Anton Blanchardcb2c9b22006-02-13 14:48:35 +11002106
Benjamin Herrenschmidtfae2e0f2012-04-11 10:42:15 +10002107 ti->local_flags &= ~_TLF_RUNLATCH;
Anton Blanchardcb2c9b22006-02-13 14:48:35 +11002108
Nicholas Piggind1d0d5f2017-08-12 02:39:07 +10002109 if (cpu_has_feature(CPU_FTR_ARCH_206)) {
2110 mtspr(SPRN_CTRLT, 0);
2111 } else {
2112 unsigned long ctrl;
2113
2114 ctrl = mfspr(SPRN_CTRLF);
2115 ctrl &= ~CTRL_RUNLATCH;
2116 mtspr(SPRN_CTRLT, ctrl);
2117 }
Anton Blanchardcb2c9b22006-02-13 14:48:35 +11002118}
Benjamin Herrenschmidtfe1952f2012-03-01 12:45:27 +11002119#endif /* CONFIG_PPC64 */
Benjamin Herrenschmidtf6a61682008-04-18 16:56:17 +10002120
Anton Blanchardd8390882009-02-22 01:50:03 +00002121unsigned long arch_align_stack(unsigned long sp)
2122{
2123 if (!(current->personality & ADDR_NO_RANDOMIZE) && randomize_va_space)
2124 sp -= get_random_int() & ~PAGE_MASK;
2125 return sp & ~0xf;
2126}
Anton Blanchard912f9ee2009-02-22 01:50:04 +00002127
2128static inline unsigned long brk_rnd(void)
2129{
2130 unsigned long rnd = 0;
2131
2132 /* 8MB for 32bit, 1GB for 64bit */
2133 if (is_32bit_task())
Daniel Cashman5ef11c32016-02-26 15:19:37 -08002134 rnd = (get_random_long() % (1UL<<(23-PAGE_SHIFT)));
Anton Blanchard912f9ee2009-02-22 01:50:04 +00002135 else
Daniel Cashman5ef11c32016-02-26 15:19:37 -08002136 rnd = (get_random_long() % (1UL<<(30-PAGE_SHIFT)));
Anton Blanchard912f9ee2009-02-22 01:50:04 +00002137
2138 return rnd << PAGE_SHIFT;
2139}
2140
2141unsigned long arch_randomize_brk(struct mm_struct *mm)
2142{
Anton Blanchard8bbde7a2009-09-21 16:52:35 +00002143 unsigned long base = mm->brk;
2144 unsigned long ret;
2145
Michael Ellerman4e003742017-10-19 15:08:43 +11002146#ifdef CONFIG_PPC_BOOK3S_64
Anton Blanchard8bbde7a2009-09-21 16:52:35 +00002147 /*
2148 * If we are using 1TB segments and we are allowed to randomise
2149 * the heap, we can put it above 1TB so it is backed by a 1TB
2150 * segment. Otherwise the heap will be in the bottom 1TB
2151 * which always uses 256MB segments and this may result in a
Aneesh Kumar K.Vcaca2852016-04-29 23:26:07 +10002152 * performance penalty. We don't need to worry about radix. For
2153 * radix, mmu_highuser_ssize remains unchanged from 256MB.
Anton Blanchard8bbde7a2009-09-21 16:52:35 +00002154 */
2155 if (!is_32bit_task() && (mmu_highuser_ssize == MMU_SEGSIZE_1T))
2156 base = max_t(unsigned long, mm->brk, 1UL << SID_SHIFT_1T);
2157#endif
2158
2159 ret = PAGE_ALIGN(base + brk_rnd());
Anton Blanchard912f9ee2009-02-22 01:50:04 +00002160
2161 if (ret < mm->brk)
2162 return mm->brk;
2163
2164 return ret;
2165}
Anton Blanchard501cb162009-02-22 01:50:07 +00002166