blob: dc55f68a6ee36dfd730e757d1ff172e43a3227be [file] [log] [blame]
Thomas Gleixner457c8992019-05-19 13:08:55 +01001// SPDX-License-Identifier: GPL-2.0-only
Linus Torvalds1da177e2005-04-16 15:20:36 -07002/*
3 * linux/kernel/fork.c
4 *
5 * Copyright (C) 1991, 1992 Linus Torvalds
6 */
7
8/*
9 * 'fork.c' contains the help-routines for the 'fork' system call
10 * (see also entry.S and others).
11 * Fork is rather simple, once you get the hang of it, but the memory
12 * management can be a bitch. See 'mm/memory.c': 'copy_page_range()'
13 */
14
Christian Braunerb3e583822019-03-27 13:04:15 +010015#include <linux/anon_inodes.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070016#include <linux/slab.h>
Ingo Molnar4eb5aaa2017-02-08 18:51:29 +010017#include <linux/sched/autogroup.h>
Ingo Molnar6e84f312017-02-08 18:51:29 +010018#include <linux/sched/mm.h>
Ingo Molnarf7ccbae2017-02-08 18:51:30 +010019#include <linux/sched/coredump.h>
Ingo Molnar8703e8a2017-02-08 18:51:30 +010020#include <linux/sched/user.h>
Ingo Molnar6a3827d2017-02-08 18:51:31 +010021#include <linux/sched/numa_balancing.h>
Ingo Molnar03441a32017-02-08 18:51:35 +010022#include <linux/sched/stat.h>
Ingo Molnar29930022017-02-08 18:51:36 +010023#include <linux/sched/task.h>
Ingo Molnar68db0cf2017-02-08 18:51:37 +010024#include <linux/sched/task_stack.h>
Ingo Molnar32ef5512017-02-05 11:48:36 +010025#include <linux/sched/cputime.h>
Christian Braunerb3e583822019-03-27 13:04:15 +010026#include <linux/seq_file.h>
Ingo Molnar037741a2017-02-03 10:08:30 +010027#include <linux/rtmutex.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070028#include <linux/init.h>
29#include <linux/unistd.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070030#include <linux/module.h>
31#include <linux/vmalloc.h>
32#include <linux/completion.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070033#include <linux/personality.h>
34#include <linux/mempolicy.h>
35#include <linux/sem.h>
36#include <linux/file.h>
Al Viro9f3acc32008-04-24 07:44:08 -040037#include <linux/fdtable.h>
Jens Axboeda9cbc82008-06-30 20:42:08 +020038#include <linux/iocontext.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070039#include <linux/key.h>
40#include <linux/binfmts.h>
41#include <linux/mman.h>
Andrea Arcangelicddb8a52008-07-28 15:46:29 -070042#include <linux/mmu_notifier.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070043#include <linux/fs.h>
Davidlohr Bueso615d6e82014-04-07 15:37:25 -070044#include <linux/mm.h>
45#include <linux/vmacache.h>
Serge E. Hallynab516012006-10-02 02:18:06 -070046#include <linux/nsproxy.h>
Randy.Dunlapc59ede72006-01-11 12:17:46 -080047#include <linux/capability.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070048#include <linux/cpu.h>
Paul Menageb4f48b62007-10-18 23:39:33 -070049#include <linux/cgroup.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070050#include <linux/security.h>
Mel Gormana1e78772008-07-23 21:27:23 -070051#include <linux/hugetlb.h>
Will Drewrye2cfabdf2012-04-12 16:47:57 -050052#include <linux/seccomp.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070053#include <linux/swap.h>
54#include <linux/syscalls.h>
55#include <linux/jiffies.h>
56#include <linux/futex.h>
Linus Torvalds8141c7f2008-11-15 10:20:36 -080057#include <linux/compat.h>
Eric Dumazet207205a2011-03-22 16:30:44 -070058#include <linux/kthread.h>
Andrew Morton7c3ab7382006-12-10 02:19:19 -080059#include <linux/task_io_accounting_ops.h>
Dipankar Sarmaab2af1f2005-09-09 13:04:13 -070060#include <linux/rcupdate.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070061#include <linux/ptrace.h>
62#include <linux/mount.h>
63#include <linux/audit.h>
Pavel Emelianov78fb7462008-02-07 00:13:51 -080064#include <linux/memcontrol.h>
Frederic Weisbeckerf201ae22008-11-23 06:22:56 +010065#include <linux/ftrace.h>
Mike Galbraith5e2bf012012-05-10 13:01:45 -070066#include <linux/proc_fs.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070067#include <linux/profile.h>
68#include <linux/rmap.h>
Hugh Dickinsf8af4da2009-09-21 17:01:57 -070069#include <linux/ksm.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070070#include <linux/acct.h>
Pavel Emelyanov893e26e2017-02-22 15:42:27 -080071#include <linux/userfaultfd_k.h>
Jay Lan8f0ab512006-09-30 23:28:59 -070072#include <linux/tsacct_kern.h>
Matt Helsley9f460802005-11-07 00:59:16 -080073#include <linux/cn_proc.h>
Rafael J. Wysockiba96a0c2007-05-23 13:57:25 -070074#include <linux/freezer.h>
Shailabh Nagarca74e922006-07-14 00:24:36 -070075#include <linux/delayacct.h>
Shailabh Nagarad4ecbc2006-07-14 00:24:44 -070076#include <linux/taskstats_kern.h>
Arjan van de Ven0a4254052006-09-26 10:52:38 +020077#include <linux/random.h>
Miloslav Trmac522ed772007-07-15 23:40:56 -070078#include <linux/tty.h>
Jens Axboefd0928d2008-01-24 08:52:45 +010079#include <linux/blkdev.h>
Al Viro5ad4e532009-03-29 19:50:06 -040080#include <linux/fs_struct.h>
Eric Sandeen7c9f8862008-04-22 16:38:23 -050081#include <linux/magic.h>
Ingo Molnarcdd6c482009-09-21 12:02:48 +020082#include <linux/perf_event.h>
Stanislaw Gruszka42c4ab42009-07-29 12:15:26 +020083#include <linux/posix-timers.h>
Avi Kivity8e7cac72009-11-29 16:34:48 +020084#include <linux/user-return-notifier.h>
Ying Han3d5992d2010-10-26 14:21:23 -070085#include <linux/oom.h>
Andrea Arcangeliba761492011-01-13 15:46:58 -080086#include <linux/khugepaged.h>
Oleg Nesterovd80e7312012-02-24 20:07:11 +010087#include <linux/signalfd.h>
Srikar Dronamraju0326f5a2012-03-13 23:30:11 +053088#include <linux/uprobes.h>
Kent Overstreeta27bb332013-05-07 16:19:08 -070089#include <linux/aio.h>
Gideon Israel Dsouza52f5684c2014-04-07 15:39:20 -070090#include <linux/compiler.h>
Heinrich Schuchardt16db3d32015-04-16 12:47:50 -070091#include <linux/sysctl.h>
Dmitry Vyukov5c9a8752016-03-22 14:27:30 -070092#include <linux/kcov.h>
Josh Poimboeufd83a7cb2017-02-13 19:42:40 -060093#include <linux/livepatch.h>
Mark Rutland48ac3c12017-07-14 12:23:09 +010094#include <linux/thread_info.h>
Alexander Popovafaef012018-08-17 01:16:58 +030095#include <linux/stackleak.h>
Daniel Axtenseafb1492019-11-30 17:54:57 -080096#include <linux/kasan.h>
Sami Tolvanend08b9f02020-04-27 09:00:07 -070097#include <linux/scs.h>
Jens Axboe0f212202020-09-13 13:09:39 -060098#include <linux/io_uring.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070099
Linus Torvalds1da177e2005-04-16 15:20:36 -0700100#include <asm/pgalloc.h>
Linus Torvalds7c0f6ba2016-12-24 11:46:01 -0800101#include <linux/uaccess.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700102#include <asm/mmu_context.h>
103#include <asm/cacheflush.h>
104#include <asm/tlbflush.h>
105
Steven Rostedtad8d75f2009-04-14 19:39:12 -0400106#include <trace/events/sched.h>
107
KAMEZAWA Hiroyuki43d2b112012-01-10 15:08:09 -0800108#define CREATE_TRACE_POINTS
109#include <trace/events/task.h>
110
Linus Torvalds1da177e2005-04-16 15:20:36 -0700111/*
Heinrich Schuchardtac1b3982015-04-16 12:47:47 -0700112 * Minimum number of threads to boot the kernel
113 */
114#define MIN_THREADS 20
115
116/*
117 * Maximum number of threads
118 */
119#define MAX_THREADS FUTEX_TID_MASK
120
121/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700122 * Protected counters by write_lock_irq(&tasklist_lock)
123 */
124unsigned long total_forks; /* Handle normal Linux uptimes. */
Daniel Rebelo de Oliveirafb0a6852011-07-26 16:08:39 -0700125int nr_threads; /* The idle threads do not count.. */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700126
Kefeng Wang8856ae42019-05-31 22:30:12 -0700127static int max_threads; /* tunable limit on nr_threads */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700128
Sai Praneeth Prakhya8495f7e2019-09-25 16:47:27 -0700129#define NAMED_ARRAY_INDEX(x) [x] = __stringify(x)
130
131static const char * const resident_page_types[] = {
132 NAMED_ARRAY_INDEX(MM_FILEPAGES),
133 NAMED_ARRAY_INDEX(MM_ANONPAGES),
134 NAMED_ARRAY_INDEX(MM_SWAPENTS),
135 NAMED_ARRAY_INDEX(MM_SHMEMPAGES),
136};
137
Linus Torvalds1da177e2005-04-16 15:20:36 -0700138DEFINE_PER_CPU(unsigned long, process_counts) = 0;
139
Christoph Hellwigc59923a2006-07-10 04:45:40 -0700140__cacheline_aligned DEFINE_RWLOCK(tasklist_lock); /* outer */
Paul E. McKenneydb1466b2010-03-03 07:46:56 -0800141
142#ifdef CONFIG_PROVE_RCU
143int lockdep_tasklist_lock_is_held(void)
144{
145 return lockdep_is_held(&tasklist_lock);
146}
147EXPORT_SYMBOL_GPL(lockdep_tasklist_lock_is_held);
148#endif /* #ifdef CONFIG_PROVE_RCU */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700149
150int nr_processes(void)
151{
152 int cpu;
153 int total = 0;
154
Ian Campbell1d510752009-11-03 10:11:14 +0000155 for_each_possible_cpu(cpu)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700156 total += per_cpu(process_counts, cpu);
157
158 return total;
159}
160
Akinobu Mitaf19b9f72012-07-30 14:42:33 -0700161void __weak arch_release_task_struct(struct task_struct *tsk)
162{
163}
164
Thomas Gleixnerf5e10282012-05-05 15:05:48 +0000165#ifndef CONFIG_ARCH_TASK_STRUCT_ALLOCATOR
Christoph Lametere18b8902006-12-06 20:33:20 -0800166static struct kmem_cache *task_struct_cachep;
Thomas Gleixner41101802012-05-05 15:05:41 +0000167
168static inline struct task_struct *alloc_task_struct_node(int node)
169{
170 return kmem_cache_alloc_node(task_struct_cachep, GFP_KERNEL, node);
171}
172
Thomas Gleixner41101802012-05-05 15:05:41 +0000173static inline void free_task_struct(struct task_struct *tsk)
174{
Thomas Gleixner41101802012-05-05 15:05:41 +0000175 kmem_cache_free(task_struct_cachep, tsk);
176}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700177#endif
178
Linus Torvaldsb235bee2016-06-24 15:09:37 -0700179#ifndef CONFIG_ARCH_THREAD_STACK_ALLOCATOR
Thomas Gleixner41101802012-05-05 15:05:41 +0000180
Thomas Gleixner0d15d742012-05-05 15:05:41 +0000181/*
182 * Allocate pages if THREAD_SIZE is >= PAGE_SIZE, otherwise use a
183 * kmemcache based allocator.
184 */
Andy Lutomirskiba14a192016-08-11 02:35:21 -0700185# if THREAD_SIZE >= PAGE_SIZE || defined(CONFIG_VMAP_STACK)
Andy Lutomirskiac496bf2016-09-15 22:45:49 -0700186
187#ifdef CONFIG_VMAP_STACK
188/*
189 * vmalloc() is a bit slow, and calling vfree() enough times will force a TLB
190 * flush. Try to minimize the number of calls by caching stacks.
191 */
192#define NR_CACHED_STACKS 2
193static DEFINE_PER_CPU(struct vm_struct *, cached_stacks[NR_CACHED_STACKS]);
Hoeun Ryu19659c52017-05-08 15:56:11 -0700194
195static int free_vm_stack_cache(unsigned int cpu)
196{
197 struct vm_struct **cached_vm_stacks = per_cpu_ptr(cached_stacks, cpu);
198 int i;
199
200 for (i = 0; i < NR_CACHED_STACKS; i++) {
201 struct vm_struct *vm_stack = cached_vm_stacks[i];
202
203 if (!vm_stack)
204 continue;
205
206 vfree(vm_stack->addr);
207 cached_vm_stacks[i] = NULL;
208 }
209
210 return 0;
211}
Andy Lutomirskiac496bf2016-09-15 22:45:49 -0700212#endif
213
Andy Lutomirskiba14a192016-08-11 02:35:21 -0700214static unsigned long *alloc_thread_stack_node(struct task_struct *tsk, int node)
FUJITA Tomonorib69c49b2008-07-25 01:45:40 -0700215{
Andy Lutomirskiba14a192016-08-11 02:35:21 -0700216#ifdef CONFIG_VMAP_STACK
Andy Lutomirskiac496bf2016-09-15 22:45:49 -0700217 void *stack;
218 int i;
219
Andy Lutomirskiac496bf2016-09-15 22:45:49 -0700220 for (i = 0; i < NR_CACHED_STACKS; i++) {
Christoph Lameter112166f2017-07-12 14:33:11 -0700221 struct vm_struct *s;
222
223 s = this_cpu_xchg(cached_stacks[i], NULL);
Andy Lutomirskiac496bf2016-09-15 22:45:49 -0700224
225 if (!s)
226 continue;
Andy Lutomirskiac496bf2016-09-15 22:45:49 -0700227
Daniel Axtenseafb1492019-11-30 17:54:57 -0800228 /* Clear the KASAN shadow of the stack. */
229 kasan_unpoison_shadow(s->addr, THREAD_SIZE);
230
Konstantin Khlebnikovca182552017-10-13 15:58:22 -0700231 /* Clear stale pointers from reused stack. */
232 memset(s->addr, 0, THREAD_SIZE);
Kees Cooke01e8062018-04-20 14:55:31 -0700233
Andy Lutomirskiac496bf2016-09-15 22:45:49 -0700234 tsk->stack_vm_area = s;
Shakeel Buttba4a4572019-01-08 15:22:57 -0800235 tsk->stack = s->addr;
Andy Lutomirskiac496bf2016-09-15 22:45:49 -0700236 return s->addr;
237 }
Andy Lutomirskiac496bf2016-09-15 22:45:49 -0700238
Roman Gushchin9b6f7e12018-10-26 15:03:19 -0700239 /*
240 * Allocated stacks are cached and later reused by new threads,
241 * so memcg accounting is performed manually on assigning/releasing
242 * stacks to tasks. Drop __GFP_ACCOUNT.
243 */
Mark Rutland48ac3c12017-07-14 12:23:09 +0100244 stack = __vmalloc_node_range(THREAD_SIZE, THREAD_ALIGN,
Andy Lutomirskiac496bf2016-09-15 22:45:49 -0700245 VMALLOC_START, VMALLOC_END,
Roman Gushchin9b6f7e12018-10-26 15:03:19 -0700246 THREADINFO_GFP & ~__GFP_ACCOUNT,
Andy Lutomirskiac496bf2016-09-15 22:45:49 -0700247 PAGE_KERNEL,
248 0, node, __builtin_return_address(0));
Andy Lutomirskiba14a192016-08-11 02:35:21 -0700249
250 /*
251 * We can't call find_vm_area() in interrupt context, and
252 * free_thread_stack() can be called in interrupt context,
253 * so cache the vm_struct.
254 */
Rik van Riel5eed6f12018-12-21 14:30:54 -0800255 if (stack) {
Andy Lutomirskiba14a192016-08-11 02:35:21 -0700256 tsk->stack_vm_area = find_vm_area(stack);
Rik van Riel5eed6f12018-12-21 14:30:54 -0800257 tsk->stack = stack;
258 }
Andy Lutomirskiba14a192016-08-11 02:35:21 -0700259 return stack;
260#else
Vladimir Davydov49491482016-07-26 15:24:24 -0700261 struct page *page = alloc_pages_node(node, THREADINFO_GFP,
262 THREAD_SIZE_ORDER);
Eric Dumazetb6a84012011-03-22 16:30:42 -0700263
Andrea Arcangeli1bf45802019-06-28 12:07:14 -0700264 if (likely(page)) {
Andrey Konovalov8dcc1d32020-08-06 23:24:57 -0700265 tsk->stack = kasan_reset_tag(page_address(page));
Andrea Arcangeli1bf45802019-06-28 12:07:14 -0700266 return tsk->stack;
267 }
268 return NULL;
Andy Lutomirskiba14a192016-08-11 02:35:21 -0700269#endif
FUJITA Tomonorib69c49b2008-07-25 01:45:40 -0700270}
271
Andy Lutomirskiba14a192016-08-11 02:35:21 -0700272static inline void free_thread_stack(struct task_struct *tsk)
FUJITA Tomonorib69c49b2008-07-25 01:45:40 -0700273{
Andy Lutomirskiac496bf2016-09-15 22:45:49 -0700274#ifdef CONFIG_VMAP_STACK
Roman Gushchin9b6f7e12018-10-26 15:03:19 -0700275 struct vm_struct *vm = task_stack_vm_area(tsk);
276
277 if (vm) {
Andy Lutomirskiac496bf2016-09-15 22:45:49 -0700278 int i;
279
Shakeel Butt991e7672020-08-06 23:21:37 -0700280 for (i = 0; i < THREAD_SIZE / PAGE_SIZE; i++)
Roman Gushchinf4b00ea2020-04-01 21:06:46 -0700281 memcg_kmem_uncharge_page(vm->pages[i], 0);
Roman Gushchin9b6f7e12018-10-26 15:03:19 -0700282
Andy Lutomirskiac496bf2016-09-15 22:45:49 -0700283 for (i = 0; i < NR_CACHED_STACKS; i++) {
Christoph Lameter112166f2017-07-12 14:33:11 -0700284 if (this_cpu_cmpxchg(cached_stacks[i],
285 NULL, tsk->stack_vm_area) != NULL)
Andy Lutomirskiac496bf2016-09-15 22:45:49 -0700286 continue;
287
Andy Lutomirskiac496bf2016-09-15 22:45:49 -0700288 return;
289 }
Andy Lutomirskiac496bf2016-09-15 22:45:49 -0700290
Andrey Ryabinin0f110a92016-12-12 16:44:14 -0800291 vfree_atomic(tsk->stack);
Andy Lutomirskiac496bf2016-09-15 22:45:49 -0700292 return;
293 }
294#endif
295
296 __free_pages(virt_to_page(tsk->stack), THREAD_SIZE_ORDER);
FUJITA Tomonorib69c49b2008-07-25 01:45:40 -0700297}
Thomas Gleixner0d15d742012-05-05 15:05:41 +0000298# else
Linus Torvaldsb235bee2016-06-24 15:09:37 -0700299static struct kmem_cache *thread_stack_cache;
Thomas Gleixner0d15d742012-05-05 15:05:41 +0000300
Michael Ellerman9521d392016-06-25 21:53:30 +1000301static unsigned long *alloc_thread_stack_node(struct task_struct *tsk,
Thomas Gleixner0d15d742012-05-05 15:05:41 +0000302 int node)
303{
Rik van Riel5eed6f12018-12-21 14:30:54 -0800304 unsigned long *stack;
305 stack = kmem_cache_alloc_node(thread_stack_cache, THREADINFO_GFP, node);
Andrey Konovalov8dcc1d32020-08-06 23:24:57 -0700306 stack = kasan_reset_tag(stack);
Rik van Riel5eed6f12018-12-21 14:30:54 -0800307 tsk->stack = stack;
308 return stack;
Thomas Gleixner0d15d742012-05-05 15:05:41 +0000309}
310
Andy Lutomirskiba14a192016-08-11 02:35:21 -0700311static void free_thread_stack(struct task_struct *tsk)
Thomas Gleixner0d15d742012-05-05 15:05:41 +0000312{
Andy Lutomirskiba14a192016-08-11 02:35:21 -0700313 kmem_cache_free(thread_stack_cache, tsk->stack);
Thomas Gleixner0d15d742012-05-05 15:05:41 +0000314}
315
Linus Torvaldsb235bee2016-06-24 15:09:37 -0700316void thread_stack_cache_init(void)
Thomas Gleixner0d15d742012-05-05 15:05:41 +0000317{
David Windsorf9d299462017-06-10 22:50:41 -0400318 thread_stack_cache = kmem_cache_create_usercopy("thread_stack",
319 THREAD_SIZE, THREAD_SIZE, 0, 0,
320 THREAD_SIZE, NULL);
Linus Torvaldsb235bee2016-06-24 15:09:37 -0700321 BUG_ON(thread_stack_cache == NULL);
Thomas Gleixner0d15d742012-05-05 15:05:41 +0000322}
323# endif
FUJITA Tomonorib69c49b2008-07-25 01:45:40 -0700324#endif
325
Linus Torvalds1da177e2005-04-16 15:20:36 -0700326/* SLAB cache for signal_struct structures (tsk->signal) */
Christoph Lametere18b8902006-12-06 20:33:20 -0800327static struct kmem_cache *signal_cachep;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700328
329/* SLAB cache for sighand_struct structures (tsk->sighand) */
Christoph Lametere18b8902006-12-06 20:33:20 -0800330struct kmem_cache *sighand_cachep;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700331
332/* SLAB cache for files_struct structures (tsk->files) */
Christoph Lametere18b8902006-12-06 20:33:20 -0800333struct kmem_cache *files_cachep;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700334
335/* SLAB cache for fs_struct structures (tsk->fs) */
Christoph Lametere18b8902006-12-06 20:33:20 -0800336struct kmem_cache *fs_cachep;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700337
338/* SLAB cache for vm_area_struct structures */
Linus Torvalds3928d4f2018-07-21 13:48:51 -0700339static struct kmem_cache *vm_area_cachep;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700340
341/* SLAB cache for mm_struct structures (tsk->mm) */
Christoph Lametere18b8902006-12-06 20:33:20 -0800342static struct kmem_cache *mm_cachep;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700343
Linus Torvalds490fc052018-07-21 15:24:03 -0700344struct vm_area_struct *vm_area_alloc(struct mm_struct *mm)
Linus Torvalds3928d4f2018-07-21 13:48:51 -0700345{
Andrew Mortona6704682018-08-21 21:53:06 -0700346 struct vm_area_struct *vma;
Linus Torvalds490fc052018-07-21 15:24:03 -0700347
Andrew Mortona6704682018-08-21 21:53:06 -0700348 vma = kmem_cache_alloc(vm_area_cachep, GFP_KERNEL);
Kirill A. Shutemov027232d2018-07-26 16:37:25 -0700349 if (vma)
350 vma_init(vma, mm);
Linus Torvalds490fc052018-07-21 15:24:03 -0700351 return vma;
Linus Torvalds3928d4f2018-07-21 13:48:51 -0700352}
353
354struct vm_area_struct *vm_area_dup(struct vm_area_struct *orig)
355{
Linus Torvalds95faf692018-07-21 14:48:45 -0700356 struct vm_area_struct *new = kmem_cache_alloc(vm_area_cachep, GFP_KERNEL);
357
358 if (new) {
Qian Caicda099b2020-02-19 11:00:54 -0800359 ASSERT_EXCLUSIVE_WRITER(orig->vm_flags);
360 ASSERT_EXCLUSIVE_WRITER(orig->vm_file);
361 /*
362 * orig->shared.rb may be modified concurrently, but the clone
363 * will be reinitialized.
364 */
365 *new = data_race(*orig);
Linus Torvalds95faf692018-07-21 14:48:45 -0700366 INIT_LIST_HEAD(&new->anon_vma_chain);
Li Xinhaie39a4b32020-04-06 20:03:39 -0700367 new->vm_next = new->vm_prev = NULL;
Linus Torvalds95faf692018-07-21 14:48:45 -0700368 }
369 return new;
Linus Torvalds3928d4f2018-07-21 13:48:51 -0700370}
371
372void vm_area_free(struct vm_area_struct *vma)
373{
374 kmem_cache_free(vm_area_cachep, vma);
375}
376
Andy Lutomirskiba14a192016-08-11 02:35:21 -0700377static void account_kernel_stack(struct task_struct *tsk, int account)
KOSAKI Motohiroc6a7f572009-09-21 17:01:32 -0700378{
Andy Lutomirskiba14a192016-08-11 02:35:21 -0700379 void *stack = task_stack_page(tsk);
380 struct vm_struct *vm = task_stack_vm_area(tsk);
KOSAKI Motohiroc6a7f572009-09-21 17:01:32 -0700381
Andy Lutomirskiefdc9492016-07-28 15:48:17 -0700382
Shakeel Butt991e7672020-08-06 23:21:37 -0700383 /* All stack pages are in the same node. */
384 if (vm)
385 mod_lruvec_page_state(vm->pages[0], NR_KERNEL_STACK_KB,
386 account * (THREAD_SIZE / 1024));
387 else
388 mod_lruvec_slab_state(stack, NR_KERNEL_STACK_KB,
389 account * (THREAD_SIZE / 1024));
KOSAKI Motohiroc6a7f572009-09-21 17:01:32 -0700390}
391
Roman Gushchin9b6f7e12018-10-26 15:03:19 -0700392static int memcg_charge_kernel_stack(struct task_struct *tsk)
393{
394#ifdef CONFIG_VMAP_STACK
395 struct vm_struct *vm = task_stack_vm_area(tsk);
396 int ret;
397
Shakeel Butt991e7672020-08-06 23:21:37 -0700398 BUILD_BUG_ON(IS_ENABLED(CONFIG_VMAP_STACK) && PAGE_SIZE % 1024 != 0);
399
Roman Gushchin9b6f7e12018-10-26 15:03:19 -0700400 if (vm) {
401 int i;
402
Shakeel Butt991e7672020-08-06 23:21:37 -0700403 BUG_ON(vm->nr_pages != THREAD_SIZE / PAGE_SIZE);
404
Roman Gushchin9b6f7e12018-10-26 15:03:19 -0700405 for (i = 0; i < THREAD_SIZE / PAGE_SIZE; i++) {
406 /*
Roman Gushchinf4b00ea2020-04-01 21:06:46 -0700407 * If memcg_kmem_charge_page() fails, page->mem_cgroup
Shakeel Butt991e7672020-08-06 23:21:37 -0700408 * pointer is NULL, and memcg_kmem_uncharge_page() in
409 * free_thread_stack() will ignore this page.
Roman Gushchin9b6f7e12018-10-26 15:03:19 -0700410 */
Roman Gushchinf4b00ea2020-04-01 21:06:46 -0700411 ret = memcg_kmem_charge_page(vm->pages[i], GFP_KERNEL,
412 0);
Roman Gushchin9b6f7e12018-10-26 15:03:19 -0700413 if (ret)
414 return ret;
Roman Gushchin9b6f7e12018-10-26 15:03:19 -0700415 }
416 }
417#endif
418 return 0;
419}
420
Andy Lutomirski68f24b082016-09-15 22:45:48 -0700421static void release_task_stack(struct task_struct *tsk)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700422{
Andy Lutomirski405c0752016-10-31 08:11:43 -0700423 if (WARN_ON(tsk->state != TASK_DEAD))
424 return; /* Better to leak the stack than to free prematurely */
425
Andy Lutomirskiba14a192016-08-11 02:35:21 -0700426 account_kernel_stack(tsk, -1);
Andy Lutomirskiba14a192016-08-11 02:35:21 -0700427 free_thread_stack(tsk);
Andy Lutomirski68f24b082016-09-15 22:45:48 -0700428 tsk->stack = NULL;
429#ifdef CONFIG_VMAP_STACK
430 tsk->stack_vm_area = NULL;
431#endif
432}
433
434#ifdef CONFIG_THREAD_INFO_IN_TASK
435void put_task_stack(struct task_struct *tsk)
436{
Elena Reshetovaf0b89d32019-01-18 14:27:30 +0200437 if (refcount_dec_and_test(&tsk->stack_refcount))
Andy Lutomirski68f24b082016-09-15 22:45:48 -0700438 release_task_stack(tsk);
439}
440#endif
441
442void free_task(struct task_struct *tsk)
443{
Sami Tolvanend08b9f02020-04-27 09:00:07 -0700444 scs_release(tsk);
445
Andy Lutomirski68f24b082016-09-15 22:45:48 -0700446#ifndef CONFIG_THREAD_INFO_IN_TASK
447 /*
448 * The task is finally done with both the stack and thread_info,
449 * so free both.
450 */
451 release_task_stack(tsk);
452#else
453 /*
454 * If the task had a separate stack allocation, it should be gone
455 * by now.
456 */
Elena Reshetovaf0b89d32019-01-18 14:27:30 +0200457 WARN_ON_ONCE(refcount_read(&tsk->stack_refcount) != 0);
Andy Lutomirski68f24b082016-09-15 22:45:48 -0700458#endif
Ingo Molnar23f78d4a2006-06-27 02:54:53 -0700459 rt_mutex_debug_task_free(tsk);
Frederic Weisbeckerfb526072008-11-25 21:07:04 +0100460 ftrace_graph_exit_task(tsk);
Akinobu Mitaf19b9f72012-07-30 14:42:33 -0700461 arch_release_task_struct(tsk);
Oleg Nesterov1da5c462016-11-29 18:50:57 +0100462 if (tsk->flags & PF_KTHREAD)
463 free_kthread_struct(tsk);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700464 free_task_struct(tsk);
465}
466EXPORT_SYMBOL(free_task);
467
Linus Torvalds1da177e2005-04-16 15:20:36 -0700468#ifdef CONFIG_MMU
Emese Revfy0766f782016-06-20 20:42:34 +0200469static __latent_entropy int dup_mmap(struct mm_struct *mm,
470 struct mm_struct *oldmm)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700471{
Linus Torvalds297c5ee2010-08-20 16:24:55 -0700472 struct vm_area_struct *mpnt, *tmp, *prev, **pprev;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700473 struct rb_node **rb_link, *rb_parent;
474 int retval;
475 unsigned long charge;
Pavel Emelyanov893e26e2017-02-22 15:42:27 -0800476 LIST_HEAD(uf);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700477
Oleg Nesterov32cdba12012-11-14 19:03:42 +0100478 uprobe_start_dup_mmap();
Michel Lespinassed8ed45c2020-06-08 21:33:25 -0700479 if (mmap_write_lock_killable(oldmm)) {
Michal Hocko7c051262016-05-23 16:25:48 -0700480 retval = -EINTR;
481 goto fail_uprobe_end;
482 }
Ralf Baechleec8c0442006-12-12 17:14:57 +0000483 flush_cache_dup_mm(oldmm);
Oleg Nesterovf8ac4ec2012-08-08 17:11:42 +0200484 uprobe_dup_mmap(oldmm, mm);
Ingo Molnarad339452006-07-03 00:25:15 -0700485 /*
486 * Not linked in yet - no deadlock potential:
487 */
Michel Lespinasseaaa2cc52020-06-08 21:33:33 -0700488 mmap_write_lock_nested(mm, SINGLE_DEPTH_NESTING);
Hugh Dickins7ee78232005-10-29 18:16:08 -0700489
Konstantin Khlebnikov90f31d02015-04-16 12:47:56 -0700490 /* No ordering required: file already has been exposed. */
491 RCU_INIT_POINTER(mm->exe_file, get_mm_exe_file(oldmm));
492
Vladimir Davydov4f7d4612014-08-08 14:22:01 -0700493 mm->total_vm = oldmm->total_vm;
Konstantin Khlebnikov84638332016-01-14 15:22:07 -0800494 mm->data_vm = oldmm->data_vm;
Vladimir Davydov4f7d4612014-08-08 14:22:01 -0700495 mm->exec_vm = oldmm->exec_vm;
496 mm->stack_vm = oldmm->stack_vm;
497
Linus Torvalds1da177e2005-04-16 15:20:36 -0700498 rb_link = &mm->mm_rb.rb_node;
499 rb_parent = NULL;
500 pprev = &mm->mmap;
Hugh Dickinsf8af4da2009-09-21 17:01:57 -0700501 retval = ksm_fork(mm, oldmm);
502 if (retval)
503 goto out;
Andrea Arcangeliba761492011-01-13 15:46:58 -0800504 retval = khugepaged_fork(mm, oldmm);
505 if (retval)
506 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700507
Linus Torvalds297c5ee2010-08-20 16:24:55 -0700508 prev = NULL;
Hugh Dickinsfd3e42f2005-10-29 18:16:06 -0700509 for (mpnt = oldmm->mmap; mpnt; mpnt = mpnt->vm_next) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700510 struct file *file;
511
512 if (mpnt->vm_flags & VM_DONTCOPY) {
Konstantin Khlebnikov84638332016-01-14 15:22:07 -0800513 vm_stat_account(mm, mpnt->vm_flags, -vma_pages(mpnt));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700514 continue;
515 }
516 charge = 0;
Tetsuo Handa655c79b2018-06-14 15:26:34 -0700517 /*
518 * Don't duplicate many vmas if we've been oom-killed (for
519 * example)
520 */
521 if (fatal_signal_pending(current)) {
522 retval = -EINTR;
523 goto out;
524 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700525 if (mpnt->vm_flags & VM_ACCOUNT) {
Huang Shijieb2412b72012-07-30 14:42:30 -0700526 unsigned long len = vma_pages(mpnt);
527
Al Viro191c5422012-02-13 03:58:52 +0000528 if (security_vm_enough_memory_mm(oldmm, len)) /* sic */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700529 goto fail_nomem;
530 charge = len;
531 }
Linus Torvalds3928d4f2018-07-21 13:48:51 -0700532 tmp = vm_area_dup(mpnt);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700533 if (!tmp)
534 goto fail_nomem;
Oleg Nesterovef0855d2013-09-11 14:20:14 -0700535 retval = vma_dup_policy(mpnt, tmp);
536 if (retval)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700537 goto fail_nomem_policy;
Andrea Arcangelia247c3a2010-09-22 13:05:12 -0700538 tmp->vm_mm = mm;
Pavel Emelyanov893e26e2017-02-22 15:42:27 -0800539 retval = dup_userfaultfd(tmp, &uf);
540 if (retval)
541 goto fail_nomem_anon_vma_fork;
Rik van Rield2cd9ed2017-09-06 16:25:15 -0700542 if (tmp->vm_flags & VM_WIPEONFORK) {
Li Xinhai93949bb2020-04-06 20:03:33 -0700543 /*
544 * VM_WIPEONFORK gets a clean slate in the child.
545 * Don't prepare anon_vma until fault since we don't
546 * copy page for current vma.
547 */
Rik van Rield2cd9ed2017-09-06 16:25:15 -0700548 tmp->anon_vma = NULL;
Rik van Rield2cd9ed2017-09-06 16:25:15 -0700549 } else if (anon_vma_fork(tmp, mpnt))
Rik van Riel5beb4932010-03-05 13:42:07 -0800550 goto fail_nomem_anon_vma_fork;
Pavel Emelyanov893e26e2017-02-22 15:42:27 -0800551 tmp->vm_flags &= ~(VM_LOCKED | VM_LOCKONFAULT);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700552 file = tmp->vm_file;
553 if (file) {
Al Viro496ad9a2013-01-23 17:07:38 -0500554 struct inode *inode = file_inode(file);
Hugh Dickinsb88ed202008-12-10 20:48:52 +0000555 struct address_space *mapping = file->f_mapping;
556
Linus Torvalds1da177e2005-04-16 15:20:36 -0700557 get_file(file);
558 if (tmp->vm_flags & VM_DENYWRITE)
Miaohe Lin73eb7f92020-10-15 20:10:08 -0700559 put_write_access(inode);
Davidlohr Bueso83cde9e2014-12-12 16:54:21 -0800560 i_mmap_lock_write(mapping);
Hugh Dickinsb88ed202008-12-10 20:48:52 +0000561 if (tmp->vm_flags & VM_SHARED)
Miaohe Lincf508b52020-10-13 16:54:10 -0700562 mapping_allow_writable(mapping);
Hugh Dickinsb88ed202008-12-10 20:48:52 +0000563 flush_dcache_mmap_lock(mapping);
564 /* insert tmp into the share list, just after mpnt */
Kirill A. Shutemov27ba0642015-02-10 14:09:59 -0800565 vma_interval_tree_insert_after(tmp, mpnt,
566 &mapping->i_mmap);
Hugh Dickinsb88ed202008-12-10 20:48:52 +0000567 flush_dcache_mmap_unlock(mapping);
Davidlohr Bueso83cde9e2014-12-12 16:54:21 -0800568 i_mmap_unlock_write(mapping);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700569 }
570
571 /*
Mel Gormana1e78772008-07-23 21:27:23 -0700572 * Clear hugetlb-related page reserves for children. This only
573 * affects MAP_PRIVATE mappings. Faults generated by the child
574 * are not guaranteed to succeed, even if read-only
575 */
576 if (is_vm_hugetlb_page(tmp))
577 reset_vma_resv_huge_pages(tmp);
578
579 /*
Hugh Dickins7ee78232005-10-29 18:16:08 -0700580 * Link in the new vma and copy the page table entries.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700581 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700582 *pprev = tmp;
583 pprev = &tmp->vm_next;
Linus Torvalds297c5ee2010-08-20 16:24:55 -0700584 tmp->vm_prev = prev;
585 prev = tmp;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700586
587 __vma_link_rb(mm, tmp, rb_link, rb_parent);
588 rb_link = &tmp->vm_rb.rb_right;
589 rb_parent = &tmp->vm_rb;
590
591 mm->map_count++;
Rik van Rield2cd9ed2017-09-06 16:25:15 -0700592 if (!(tmp->vm_flags & VM_WIPEONFORK))
Peter Xuc78f4632020-10-13 16:54:21 -0700593 retval = copy_page_range(tmp, mpnt);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700594
595 if (tmp->vm_ops && tmp->vm_ops->open)
596 tmp->vm_ops->open(tmp);
597
598 if (retval)
599 goto out;
600 }
Jeremy Fitzhardinged6dd61c2007-05-02 19:27:14 +0200601 /* a new mm has just been created */
Nadav Amit1ed0cc52018-09-04 15:45:41 -0700602 retval = arch_dup_mmap(oldmm, mm);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700603out:
Michel Lespinassed8ed45c2020-06-08 21:33:25 -0700604 mmap_write_unlock(mm);
Hugh Dickinsfd3e42f2005-10-29 18:16:06 -0700605 flush_tlb_mm(oldmm);
Michel Lespinassed8ed45c2020-06-08 21:33:25 -0700606 mmap_write_unlock(oldmm);
Pavel Emelyanov893e26e2017-02-22 15:42:27 -0800607 dup_userfaultfd_complete(&uf);
Michal Hocko7c051262016-05-23 16:25:48 -0700608fail_uprobe_end:
Oleg Nesterov32cdba12012-11-14 19:03:42 +0100609 uprobe_end_dup_mmap();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700610 return retval;
Rik van Riel5beb4932010-03-05 13:42:07 -0800611fail_nomem_anon_vma_fork:
Oleg Nesterovef0855d2013-09-11 14:20:14 -0700612 mpol_put(vma_policy(tmp));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700613fail_nomem_policy:
Linus Torvalds3928d4f2018-07-21 13:48:51 -0700614 vm_area_free(tmp);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700615fail_nomem:
616 retval = -ENOMEM;
617 vm_unacct_memory(charge);
618 goto out;
619}
620
Daniel Rebelo de Oliveirafb0a6852011-07-26 16:08:39 -0700621static inline int mm_alloc_pgd(struct mm_struct *mm)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700622{
623 mm->pgd = pgd_alloc(mm);
624 if (unlikely(!mm->pgd))
625 return -ENOMEM;
626 return 0;
627}
628
Daniel Rebelo de Oliveirafb0a6852011-07-26 16:08:39 -0700629static inline void mm_free_pgd(struct mm_struct *mm)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700630{
Benjamin Herrenschmidt5e541972008-02-04 22:29:14 -0800631 pgd_free(mm, mm->pgd);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700632}
633#else
Konstantin Khlebnikov90f31d02015-04-16 12:47:56 -0700634static int dup_mmap(struct mm_struct *mm, struct mm_struct *oldmm)
635{
Michel Lespinassed8ed45c2020-06-08 21:33:25 -0700636 mmap_write_lock(oldmm);
Konstantin Khlebnikov90f31d02015-04-16 12:47:56 -0700637 RCU_INIT_POINTER(mm->exe_file, get_mm_exe_file(oldmm));
Michel Lespinassed8ed45c2020-06-08 21:33:25 -0700638 mmap_write_unlock(oldmm);
Konstantin Khlebnikov90f31d02015-04-16 12:47:56 -0700639 return 0;
640}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700641#define mm_alloc_pgd(mm) (0)
642#define mm_free_pgd(mm)
643#endif /* CONFIG_MMU */
644
Andrew Mortond70f2a12018-01-31 16:15:51 -0800645static void check_mm(struct mm_struct *mm)
646{
647 int i;
648
Sai Praneeth Prakhya8495f7e2019-09-25 16:47:27 -0700649 BUILD_BUG_ON_MSG(ARRAY_SIZE(resident_page_types) != NR_MM_COUNTERS,
650 "Please make sure 'struct resident_page_types[]' is updated as well");
651
Andrew Mortond70f2a12018-01-31 16:15:51 -0800652 for (i = 0; i < NR_MM_COUNTERS; i++) {
653 long x = atomic_long_read(&mm->rss_stat.count[i]);
654
655 if (unlikely(x))
Sai Praneeth Prakhya8495f7e2019-09-25 16:47:27 -0700656 pr_alert("BUG: Bad rss-counter state mm:%p type:%s val:%ld\n",
657 mm, resident_page_types[i], x);
Andrew Mortond70f2a12018-01-31 16:15:51 -0800658 }
659
660 if (mm_pgtables_bytes(mm))
661 pr_alert("BUG: non-zero pgtables_bytes on freeing mm: %ld\n",
662 mm_pgtables_bytes(mm));
663
664#if defined(CONFIG_TRANSPARENT_HUGEPAGE) && !USE_SPLIT_PMD_PTLOCKS
665 VM_BUG_ON_MM(mm->pmd_huge_pte, mm);
666#endif
667}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700668
Christoph Lametere94b1762006-12-06 20:33:17 -0800669#define allocate_mm() (kmem_cache_alloc(mm_cachep, GFP_KERNEL))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700670#define free_mm(mm) (kmem_cache_free(mm_cachep, (mm)))
671
Andrew Mortond70f2a12018-01-31 16:15:51 -0800672/*
673 * Called when the last reference to the mm
674 * is dropped: either by a lazy thread or by
675 * mmput. Free the page directory and the mm.
676 */
Andrew Mortond34bc482018-02-21 14:45:17 -0800677void __mmdrop(struct mm_struct *mm)
Andrew Mortond70f2a12018-01-31 16:15:51 -0800678{
679 BUG_ON(mm == &init_mm);
Mark Rutland3eda69c2018-04-05 16:25:12 -0700680 WARN_ON_ONCE(mm == current->mm);
681 WARN_ON_ONCE(mm == current->active_mm);
Andrew Mortond70f2a12018-01-31 16:15:51 -0800682 mm_free_pgd(mm);
683 destroy_context(mm);
Jason Gunthorpe984cfe42019-12-18 13:40:35 -0400684 mmu_notifier_subscriptions_destroy(mm);
Andrew Mortond70f2a12018-01-31 16:15:51 -0800685 check_mm(mm);
686 put_user_ns(mm->user_ns);
687 free_mm(mm);
688}
Andrew Mortond34bc482018-02-21 14:45:17 -0800689EXPORT_SYMBOL_GPL(__mmdrop);
Andrew Mortond70f2a12018-01-31 16:15:51 -0800690
691static void mmdrop_async_fn(struct work_struct *work)
692{
693 struct mm_struct *mm;
694
695 mm = container_of(work, struct mm_struct, async_put_work);
696 __mmdrop(mm);
697}
698
699static void mmdrop_async(struct mm_struct *mm)
700{
701 if (unlikely(atomic_dec_and_test(&mm->mm_count))) {
702 INIT_WORK(&mm->async_put_work, mmdrop_async_fn);
703 schedule_work(&mm->async_put_work);
704 }
705}
706
Linus Torvalds1da177e2005-04-16 15:20:36 -0700707static inline void free_signal_struct(struct signal_struct *sig)
708{
709 taskstats_tgid_free(sig);
710 sched_autogroup_exit(sig);
711 /*
712 * __mmdrop is not safe to call from softirq context on x86 due to
713 * pgd_dtor so postpone it to the async context
714 */
715 if (sig->oom_mm)
716 mmdrop_async(sig->oom_mm);
717 kmem_cache_free(signal_cachep, sig);
718}
719
720static inline void put_signal_struct(struct signal_struct *sig)
721{
Elena Reshetova60d4de32019-01-18 14:27:27 +0200722 if (refcount_dec_and_test(&sig->sigcnt))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700723 free_signal_struct(sig);
724}
725
726void __put_task_struct(struct task_struct *tsk)
727{
728 WARN_ON(!tsk->exit_state);
Elena Reshetovaec1d2812019-01-18 14:27:29 +0200729 WARN_ON(refcount_read(&tsk->usage));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700730 WARN_ON(tsk == current);
731
Jens Axboe0f212202020-09-13 13:09:39 -0600732 io_uring_free(tsk);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700733 cgroup_free(tsk);
Jann Horn16d51a52019-07-16 17:20:45 +0200734 task_numa_free(tsk, true);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700735 security_task_free(tsk);
736 exit_creds(tsk);
737 delayacct_tsk_free(tsk);
738 put_signal_struct(tsk->signal);
739
740 if (!profile_handoff_task(tsk))
741 free_task(tsk);
742}
743EXPORT_SYMBOL_GPL(__put_task_struct);
744
745void __init __weak arch_task_cache_init(void) { }
746
747/*
748 * set_max_threads
749 */
750static void set_max_threads(unsigned int max_threads_suggested)
751{
752 u64 threads;
Arun KSca79b0c2018-12-28 00:34:29 -0800753 unsigned long nr_pages = totalram_pages();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700754
755 /*
756 * The number of threads shall be limited such that the thread
757 * structures may only consume a small part of the available memory.
758 */
Arun KS3d6357d2018-12-28 00:34:20 -0800759 if (fls64(nr_pages) + fls64(PAGE_SIZE) > 64)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700760 threads = MAX_THREADS;
761 else
Arun KS3d6357d2018-12-28 00:34:20 -0800762 threads = div64_u64((u64) nr_pages * (u64) PAGE_SIZE,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700763 (u64) THREAD_SIZE * 8UL);
764
765 if (threads > max_threads_suggested)
766 threads = max_threads_suggested;
767
768 max_threads = clamp_t(u64, threads, MIN_THREADS, MAX_THREADS);
769}
770
771#ifdef CONFIG_ARCH_WANTS_DYNAMIC_TASK_STRUCT
772/* Initialized by the architecture: */
773int arch_task_struct_size __read_mostly;
774#endif
775
Christoph Hellwig4189ff22019-08-12 08:55:24 +0200776#ifndef CONFIG_ARCH_TASK_STRUCT_ALLOCATOR
Kees Cook59054292017-08-16 13:00:58 -0700777static void task_struct_whitelist(unsigned long *offset, unsigned long *size)
778{
779 /* Fetch thread_struct whitelist for the architecture. */
780 arch_thread_struct_whitelist(offset, size);
781
782 /*
783 * Handle zero-sized whitelist or empty thread_struct, otherwise
784 * adjust offset to position of thread_struct in task_struct.
785 */
786 if (unlikely(*size == 0))
787 *offset = 0;
788 else
789 *offset += offsetof(struct task_struct, thread);
790}
Christoph Hellwig4189ff22019-08-12 08:55:24 +0200791#endif /* CONFIG_ARCH_TASK_STRUCT_ALLOCATOR */
Kees Cook59054292017-08-16 13:00:58 -0700792
Linus Torvalds1da177e2005-04-16 15:20:36 -0700793void __init fork_init(void)
794{
795 int i;
796#ifndef CONFIG_ARCH_TASK_STRUCT_ALLOCATOR
797#ifndef ARCH_MIN_TASKALIGN
798#define ARCH_MIN_TASKALIGN 0
799#endif
800 int align = max_t(int, L1_CACHE_BYTES, ARCH_MIN_TASKALIGN);
Kees Cook59054292017-08-16 13:00:58 -0700801 unsigned long useroffset, usersize;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700802
803 /* create a slab on which task_structs can be allocated */
Kees Cook59054292017-08-16 13:00:58 -0700804 task_struct_whitelist(&useroffset, &usersize);
805 task_struct_cachep = kmem_cache_create_usercopy("task_struct",
Linus Torvalds1da177e2005-04-16 15:20:36 -0700806 arch_task_struct_size, align,
Kees Cook59054292017-08-16 13:00:58 -0700807 SLAB_PANIC|SLAB_ACCOUNT,
808 useroffset, usersize, NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700809#endif
810
811 /* do the arch specific task caches init */
812 arch_task_cache_init();
813
814 set_max_threads(MAX_THREADS);
815
816 init_task.signal->rlim[RLIMIT_NPROC].rlim_cur = max_threads/2;
817 init_task.signal->rlim[RLIMIT_NPROC].rlim_max = max_threads/2;
818 init_task.signal->rlim[RLIMIT_SIGPENDING] =
819 init_task.signal->rlim[RLIMIT_NPROC];
820
821 for (i = 0; i < UCOUNT_COUNTS; i++) {
822 init_user_ns.ucount_max[i] = max_threads/2;
823 }
824
825#ifdef CONFIG_VMAP_STACK
826 cpuhp_setup_state(CPUHP_BP_PREPARE_DYN, "fork:vm_stack_cache",
827 NULL, free_vm_stack_cache);
828#endif
829
Sami Tolvanend08b9f02020-04-27 09:00:07 -0700830 scs_init();
831
Linus Torvalds1da177e2005-04-16 15:20:36 -0700832 lockdep_init_task(&init_task);
Nadav Amitaad42dd2019-04-26 16:22:44 -0700833 uprobes_init();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700834}
835
836int __weak arch_dup_task_struct(struct task_struct *dst,
837 struct task_struct *src)
838{
839 *dst = *src;
840 return 0;
841}
842
843void set_task_stack_end_magic(struct task_struct *tsk)
844{
845 unsigned long *stackend;
846
847 stackend = end_of_stack(tsk);
848 *stackend = STACK_END_MAGIC; /* for overflow detection */
849}
850
851static struct task_struct *dup_task_struct(struct task_struct *orig, int node)
852{
853 struct task_struct *tsk;
854 unsigned long *stack;
YueHaibing0f4991e2018-12-28 00:40:00 -0800855 struct vm_struct *stack_vm_area __maybe_unused;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700856 int err;
857
858 if (node == NUMA_NO_NODE)
859 node = tsk_fork_get_node(orig);
860 tsk = alloc_task_struct_node(node);
861 if (!tsk)
862 return NULL;
863
864 stack = alloc_thread_stack_node(tsk, node);
865 if (!stack)
866 goto free_tsk;
867
Roman Gushchin9b6f7e12018-10-26 15:03:19 -0700868 if (memcg_charge_kernel_stack(tsk))
869 goto free_stack;
870
Linus Torvalds1da177e2005-04-16 15:20:36 -0700871 stack_vm_area = task_stack_vm_area(tsk);
872
873 err = arch_dup_task_struct(tsk, orig);
874
875 /*
876 * arch_dup_task_struct() clobbers the stack-related fields. Make
877 * sure they're properly initialized before using any stack-related
878 * functions again.
879 */
880 tsk->stack = stack;
881#ifdef CONFIG_VMAP_STACK
882 tsk->stack_vm_area = stack_vm_area;
883#endif
884#ifdef CONFIG_THREAD_INFO_IN_TASK
Elena Reshetovaf0b89d32019-01-18 14:27:30 +0200885 refcount_set(&tsk->stack_refcount, 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700886#endif
887
888 if (err)
889 goto free_stack;
890
Sami Tolvanend08b9f02020-04-27 09:00:07 -0700891 err = scs_prepare(tsk, node);
892 if (err)
893 goto free_stack;
894
Linus Torvalds1da177e2005-04-16 15:20:36 -0700895#ifdef CONFIG_SECCOMP
896 /*
897 * We must handle setting up seccomp filters once we're under
898 * the sighand lock in case orig has changed between now and
899 * then. Until then, filter must be NULL to avoid messing up
900 * the usage counts on the error path calling free_task.
901 */
902 tsk->seccomp.filter = NULL;
903#endif
904
905 setup_thread_stack(tsk, orig);
906 clear_user_return_notifier(tsk);
907 clear_tsk_need_resched(tsk);
908 set_task_stack_end_magic(tsk);
909
Linus Torvalds050e9ba2018-06-14 12:21:18 +0900910#ifdef CONFIG_STACKPROTECTOR
Linus Torvalds1da177e2005-04-16 15:20:36 -0700911 tsk->stack_canary = get_random_canary();
912#endif
Sebastian Andrzej Siewior3bd370622019-04-23 16:26:36 +0200913 if (orig->cpus_ptr == &orig->cpus_mask)
914 tsk->cpus_ptr = &tsk->cpus_mask;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700915
916 /*
Eric W. Biederman0ff7b2c2019-09-14 07:33:58 -0500917 * One for the user space visible state that goes away when reaped.
918 * One for the scheduler.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700919 */
Eric W. Biederman0ff7b2c2019-09-14 07:33:58 -0500920 refcount_set(&tsk->rcu_users, 2);
921 /* One for the rcu users */
922 refcount_set(&tsk->usage, 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700923#ifdef CONFIG_BLK_DEV_IO_TRACE
924 tsk->btrace_seq = 0;
925#endif
926 tsk->splice_pipe = NULL;
927 tsk->task_frag.page = NULL;
928 tsk->wake_q.next = NULL;
929
930 account_kernel_stack(tsk, 1);
931
932 kcov_task_init(tsk);
933
934#ifdef CONFIG_FAULT_INJECTION
935 tsk->fail_nth = 0;
936#endif
937
Josef Bacik2c323012018-07-31 12:39:04 -0400938#ifdef CONFIG_BLK_CGROUP
939 tsk->throttle_queue = NULL;
940 tsk->use_memdelay = 0;
941#endif
942
Shakeel Buttd46eb14b2018-08-17 15:46:39 -0700943#ifdef CONFIG_MEMCG
944 tsk->active_memcg = NULL;
945#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700946 return tsk;
947
948free_stack:
949 free_thread_stack(tsk);
950free_tsk:
951 free_task_struct(tsk);
952 return NULL;
953}
954
Linus Torvalds1da177e2005-04-16 15:20:36 -0700955__cacheline_aligned_in_smp DEFINE_SPINLOCK(mmlist_lock);
956
Hidehiro Kawai4cb0e112009-01-06 14:42:47 -0800957static unsigned long default_dump_filter = MMF_DUMP_FILTER_DEFAULT;
958
959static int __init coredump_filter_setup(char *s)
960{
961 default_dump_filter =
962 (simple_strtoul(s, NULL, 0) << MMF_DUMP_FILTER_SHIFT) &
963 MMF_DUMP_FILTER_MASK;
964 return 1;
965}
966
967__setup("coredump_filter=", coredump_filter_setup);
968
Linus Torvalds1da177e2005-04-16 15:20:36 -0700969#include <linux/init_task.h>
970
Alexey Dobriyan858f0992009-09-23 15:57:32 -0700971static void mm_init_aio(struct mm_struct *mm)
972{
973#ifdef CONFIG_AIO
974 spin_lock_init(&mm->ioctx_lock);
Benjamin LaHaisedb446a02013-07-30 12:54:40 -0400975 mm->ioctx_table = NULL;
Alexey Dobriyan858f0992009-09-23 15:57:32 -0700976#endif
977}
978
Andrea Arcangelic3f3ce02019-05-14 15:40:46 -0700979static __always_inline void mm_clear_owner(struct mm_struct *mm,
980 struct task_struct *p)
981{
982#ifdef CONFIG_MEMCG
983 if (mm->owner == p)
984 WRITE_ONCE(mm->owner, NULL);
985#endif
986}
987
Vladimir Davydov33144e82014-08-08 14:22:03 -0700988static void mm_init_owner(struct mm_struct *mm, struct task_struct *p)
989{
990#ifdef CONFIG_MEMCG
991 mm->owner = p;
992#endif
993}
994
Eric Biggers355627f2017-08-31 16:15:26 -0700995static void mm_init_uprobes_state(struct mm_struct *mm)
996{
997#ifdef CONFIG_UPROBES
998 mm->uprobes_state.xol_area = NULL;
999#endif
1000}
1001
Eric W. Biedermanbfedb582016-10-13 21:23:16 -05001002static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p,
1003 struct user_namespace *user_ns)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001004{
Vladimir Davydov41f727f2014-08-08 14:21:56 -07001005 mm->mmap = NULL;
1006 mm->mm_rb = RB_ROOT;
1007 mm->vmacache_seqnum = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001008 atomic_set(&mm->mm_users, 1);
1009 atomic_set(&mm->mm_count, 1);
Jason Gunthorpe53794652020-12-14 19:05:44 -08001010 seqcount_init(&mm->write_protect_seq);
Michel Lespinassed8ed45c2020-06-08 21:33:25 -07001011 mmap_init_lock(mm);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001012 INIT_LIST_HEAD(&mm->mmlist);
Oleg Nesterov999d9fc2008-07-25 01:47:41 -07001013 mm->core_state = NULL;
Kirill A. Shutemovaf5b0f62017-11-15 17:35:40 -08001014 mm_pgtables_bytes_init(mm);
Vladimir Davydov41f727f2014-08-08 14:21:56 -07001015 mm->map_count = 0;
1016 mm->locked_vm = 0;
Peter Xu008cfe42020-09-25 18:25:57 -04001017 atomic_set(&mm->has_pinned, 0);
Davidlohr Bueso70f8a3c2019-02-06 09:59:15 -08001018 atomic64_set(&mm->pinned_vm, 0);
KAMEZAWA Hiroyukid559db02010-03-05 13:41:39 -08001019 memset(&mm->rss_stat, 0, sizeof(mm->rss_stat));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001020 spin_lock_init(&mm->page_table_lock);
Yang Shi88aa7cc2018-06-07 17:05:28 -07001021 spin_lock_init(&mm->arg_lock);
Vladimir Davydov41f727f2014-08-08 14:21:56 -07001022 mm_init_cpumask(mm);
Alexey Dobriyan858f0992009-09-23 15:57:32 -07001023 mm_init_aio(mm);
Balbir Singhcf475ad22008-04-29 01:00:16 -07001024 mm_init_owner(mm, p);
Eric Biggers2b7e8662017-08-25 15:55:43 -07001025 RCU_INIT_POINTER(mm->exe_file, NULL);
Jason Gunthorpe984cfe42019-12-18 13:40:35 -04001026 mmu_notifier_subscriptions_init(mm);
Nadav Amit16af97d2017-08-10 15:23:56 -07001027 init_tlb_flush_pending(mm);
Vladimir Davydov41f727f2014-08-08 14:21:56 -07001028#if defined(CONFIG_TRANSPARENT_HUGEPAGE) && !USE_SPLIT_PMD_PTLOCKS
1029 mm->pmd_huge_pte = NULL;
1030#endif
Eric Biggers355627f2017-08-31 16:15:26 -07001031 mm_init_uprobes_state(mm);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001032
Alex Thorltona0715cc2014-04-07 15:37:10 -07001033 if (current->mm) {
1034 mm->flags = current->mm->flags & MMF_INIT_MASK;
1035 mm->def_flags = current->mm->def_flags & VM_INIT_DEF_MASK;
1036 } else {
1037 mm->flags = default_dump_filter;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001038 mm->def_flags = 0;
Alex Thorltona0715cc2014-04-07 15:37:10 -07001039 }
1040
Vladimir Davydov41f727f2014-08-08 14:21:56 -07001041 if (mm_alloc_pgd(mm))
1042 goto fail_nopgd;
Pavel Emelianov78fb7462008-02-07 00:13:51 -08001043
Vladimir Davydov41f727f2014-08-08 14:21:56 -07001044 if (init_new_context(p, mm))
1045 goto fail_nocontext;
1046
Eric W. Biedermanbfedb582016-10-13 21:23:16 -05001047 mm->user_ns = get_user_ns(user_ns);
Vladimir Davydov41f727f2014-08-08 14:21:56 -07001048 return mm;
1049
1050fail_nocontext:
1051 mm_free_pgd(mm);
1052fail_nopgd:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001053 free_mm(mm);
1054 return NULL;
1055}
1056
1057/*
1058 * Allocate and initialize an mm_struct.
1059 */
Daniel Rebelo de Oliveirafb0a6852011-07-26 16:08:39 -07001060struct mm_struct *mm_alloc(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001061{
Daniel Rebelo de Oliveirafb0a6852011-07-26 16:08:39 -07001062 struct mm_struct *mm;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001063
1064 mm = allocate_mm();
KOSAKI Motohirode03c722011-05-24 17:12:15 -07001065 if (!mm)
1066 return NULL;
1067
1068 memset(mm, 0, sizeof(*mm));
Eric W. Biedermanbfedb582016-10-13 21:23:16 -05001069 return mm_init(mm, current, current_user_ns());
Linus Torvalds1da177e2005-04-16 15:20:36 -07001070}
1071
Michal Hockoec8d7c12016-05-20 16:57:21 -07001072static inline void __mmput(struct mm_struct *mm)
1073{
1074 VM_BUG_ON(atomic_read(&mm->mm_users));
1075
1076 uprobe_clear_state(mm);
1077 exit_aio(mm);
1078 ksm_exit(mm);
1079 khugepaged_exit(mm); /* must run before exit_mmap */
1080 exit_mmap(mm);
Aaron Lu6fcb52a2016-10-07 17:00:08 -07001081 mm_put_huge_zero_page(mm);
Michal Hockoec8d7c12016-05-20 16:57:21 -07001082 set_mm_exe_file(mm, NULL);
1083 if (!list_empty(&mm->mmlist)) {
1084 spin_lock(&mmlist_lock);
1085 list_del(&mm->mmlist);
1086 spin_unlock(&mmlist_lock);
1087 }
1088 if (mm->binfmt)
1089 module_put(mm->binfmt->module);
1090 mmdrop(mm);
1091}
1092
Linus Torvalds1da177e2005-04-16 15:20:36 -07001093/*
1094 * Decrement the use count and release all resources for an mm.
1095 */
1096void mmput(struct mm_struct *mm)
1097{
Andrew Morton0ae26f12006-06-23 02:05:15 -07001098 might_sleep();
1099
Michal Hockoec8d7c12016-05-20 16:57:21 -07001100 if (atomic_dec_and_test(&mm->mm_users))
1101 __mmput(mm);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001102}
1103EXPORT_SYMBOL_GPL(mmput);
1104
Sherry Yanga1b22892017-10-03 16:15:00 -07001105#ifdef CONFIG_MMU
1106static void mmput_async_fn(struct work_struct *work)
1107{
1108 struct mm_struct *mm = container_of(work, struct mm_struct,
1109 async_put_work);
1110
1111 __mmput(mm);
1112}
1113
1114void mmput_async(struct mm_struct *mm)
1115{
1116 if (atomic_dec_and_test(&mm->mm_users)) {
1117 INIT_WORK(&mm->async_put_work, mmput_async_fn);
1118 schedule_work(&mm->async_put_work);
1119 }
1120}
1121#endif
1122
Konstantin Khlebnikov90f31d02015-04-16 12:47:56 -07001123/**
1124 * set_mm_exe_file - change a reference to the mm's executable file
1125 *
1126 * This changes mm's executable file (shown as symlink /proc/[pid]/exe).
1127 *
Davidlohr Bueso6e399cd2015-04-16 12:47:59 -07001128 * Main users are mmput() and sys_execve(). Callers prevent concurrent
1129 * invocations: in mmput() nobody alive left, in execve task is single
1130 * threaded. sys_prctl(PR_SET_MM_MAP/EXE_FILE) also needs to set the
1131 * mm->exe_file, but does so without using set_mm_exe_file() in order
1132 * to do avoid the need for any locks.
Konstantin Khlebnikov90f31d02015-04-16 12:47:56 -07001133 */
Jiri Slaby38646012011-05-26 16:25:46 -07001134void set_mm_exe_file(struct mm_struct *mm, struct file *new_exe_file)
1135{
Davidlohr Bueso6e399cd2015-04-16 12:47:59 -07001136 struct file *old_exe_file;
1137
1138 /*
1139 * It is safe to dereference the exe_file without RCU as
1140 * this function is only called if nobody else can access
1141 * this mm -- see comment above for justification.
1142 */
1143 old_exe_file = rcu_dereference_raw(mm->exe_file);
Konstantin Khlebnikov90f31d02015-04-16 12:47:56 -07001144
Jiri Slaby38646012011-05-26 16:25:46 -07001145 if (new_exe_file)
1146 get_file(new_exe_file);
Konstantin Khlebnikov90f31d02015-04-16 12:47:56 -07001147 rcu_assign_pointer(mm->exe_file, new_exe_file);
1148 if (old_exe_file)
1149 fput(old_exe_file);
Jiri Slaby38646012011-05-26 16:25:46 -07001150}
1151
Konstantin Khlebnikov90f31d02015-04-16 12:47:56 -07001152/**
1153 * get_mm_exe_file - acquire a reference to the mm's executable file
1154 *
1155 * Returns %NULL if mm has no associated executable file.
1156 * User must release file via fput().
1157 */
Jiri Slaby38646012011-05-26 16:25:46 -07001158struct file *get_mm_exe_file(struct mm_struct *mm)
1159{
1160 struct file *exe_file;
1161
Konstantin Khlebnikov90f31d02015-04-16 12:47:56 -07001162 rcu_read_lock();
1163 exe_file = rcu_dereference(mm->exe_file);
1164 if (exe_file && !get_file_rcu(exe_file))
1165 exe_file = NULL;
1166 rcu_read_unlock();
Jiri Slaby38646012011-05-26 16:25:46 -07001167 return exe_file;
1168}
Davidlohr Bueso11163342015-04-16 12:49:12 -07001169EXPORT_SYMBOL(get_mm_exe_file);
Jiri Slaby38646012011-05-26 16:25:46 -07001170
Linus Torvalds1da177e2005-04-16 15:20:36 -07001171/**
Mateusz Guzikcd81a9172016-08-23 16:20:38 +02001172 * get_task_exe_file - acquire a reference to the task's executable file
1173 *
1174 * Returns %NULL if task's mm (if any) has no associated executable file or
1175 * this is a kernel thread with borrowed mm (see the comment above get_task_mm).
1176 * User must release file via fput().
1177 */
1178struct file *get_task_exe_file(struct task_struct *task)
1179{
1180 struct file *exe_file = NULL;
1181 struct mm_struct *mm;
1182
1183 task_lock(task);
1184 mm = task->mm;
1185 if (mm) {
1186 if (!(task->flags & PF_KTHREAD))
1187 exe_file = get_mm_exe_file(mm);
1188 }
1189 task_unlock(task);
1190 return exe_file;
1191}
1192EXPORT_SYMBOL(get_task_exe_file);
1193
1194/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07001195 * get_task_mm - acquire a reference to the task's mm
1196 *
Oleg Nesterov246bb0b2008-07-25 01:47:38 -07001197 * Returns %NULL if the task has no mm. Checks PF_KTHREAD (meaning
Linus Torvalds1da177e2005-04-16 15:20:36 -07001198 * this kernel workthread has transiently adopted a user mm with use_mm,
1199 * to do its AIO) is not set and if so returns a reference to it, after
1200 * bumping up the use count. User must release the mm via mmput()
1201 * after use. Typically used by /proc and ptrace.
1202 */
1203struct mm_struct *get_task_mm(struct task_struct *task)
1204{
1205 struct mm_struct *mm;
1206
1207 task_lock(task);
1208 mm = task->mm;
1209 if (mm) {
Oleg Nesterov246bb0b2008-07-25 01:47:38 -07001210 if (task->flags & PF_KTHREAD)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001211 mm = NULL;
1212 else
Vegard Nossum3fce3712017-02-27 14:30:10 -08001213 mmget(mm);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001214 }
1215 task_unlock(task);
1216 return mm;
1217}
1218EXPORT_SYMBOL_GPL(get_task_mm);
1219
Christopher Yeoh8cdb8782012-02-02 11:34:09 +10301220struct mm_struct *mm_access(struct task_struct *task, unsigned int mode)
1221{
1222 struct mm_struct *mm;
1223 int err;
1224
Bernd Edlinger3e74fab2020-03-20 21:26:04 +01001225 err = mutex_lock_killable(&task->signal->exec_update_mutex);
Christopher Yeoh8cdb8782012-02-02 11:34:09 +10301226 if (err)
1227 return ERR_PTR(err);
1228
1229 mm = get_task_mm(task);
1230 if (mm && mm != current->mm &&
1231 !ptrace_may_access(task, mode)) {
1232 mmput(mm);
1233 mm = ERR_PTR(-EACCES);
1234 }
Bernd Edlinger3e74fab2020-03-20 21:26:04 +01001235 mutex_unlock(&task->signal->exec_update_mutex);
Christopher Yeoh8cdb8782012-02-02 11:34:09 +10301236
1237 return mm;
1238}
1239
Oleg Nesterov57b59c42012-03-05 14:59:13 -08001240static void complete_vfork_done(struct task_struct *tsk)
Oleg Nesterovc415c3b2012-03-05 14:59:13 -08001241{
Oleg Nesterovd68b46f2012-03-05 14:59:13 -08001242 struct completion *vfork;
Oleg Nesterovc415c3b2012-03-05 14:59:13 -08001243
Oleg Nesterovd68b46f2012-03-05 14:59:13 -08001244 task_lock(tsk);
1245 vfork = tsk->vfork_done;
1246 if (likely(vfork)) {
1247 tsk->vfork_done = NULL;
1248 complete(vfork);
1249 }
1250 task_unlock(tsk);
1251}
1252
1253static int wait_for_vfork_done(struct task_struct *child,
1254 struct completion *vfork)
1255{
1256 int killed;
1257
1258 freezer_do_not_count();
Roman Gushchin76f969e2019-04-19 10:03:04 -07001259 cgroup_enter_frozen();
Oleg Nesterovd68b46f2012-03-05 14:59:13 -08001260 killed = wait_for_completion_killable(vfork);
Roman Gushchin76f969e2019-04-19 10:03:04 -07001261 cgroup_leave_frozen(false);
Oleg Nesterovd68b46f2012-03-05 14:59:13 -08001262 freezer_count();
1263
1264 if (killed) {
1265 task_lock(child);
1266 child->vfork_done = NULL;
1267 task_unlock(child);
1268 }
1269
1270 put_task_struct(child);
1271 return killed;
Oleg Nesterovc415c3b2012-03-05 14:59:13 -08001272}
1273
Linus Torvalds1da177e2005-04-16 15:20:36 -07001274/* Please note the differences between mmput and mm_release.
1275 * mmput is called whenever we stop holding onto a mm_struct,
1276 * error success whatever.
1277 *
1278 * mm_release is called after a mm_struct has been removed
1279 * from the current process.
1280 *
1281 * This difference is important for error handling, when we
1282 * only half set up a mm_struct for a new process and need to restore
1283 * the old one. Because we mmput the new mm_struct before
1284 * restoring the old one. . .
1285 * Eric Biederman 10 January 1998
1286 */
Thomas Gleixner4610ba72019-11-06 22:55:38 +01001287static void mm_release(struct task_struct *tsk, struct mm_struct *mm)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001288{
Srikar Dronamraju0326f5a2012-03-13 23:30:11 +05301289 uprobe_free_utask(tsk);
1290
Linus Torvalds1da177e2005-04-16 15:20:36 -07001291 /* Get rid of any cached register state */
1292 deactivate_mm(tsk, mm);
1293
Roland McGrathfec1d012006-12-06 20:36:34 -08001294 /*
Michal Hocko735f2772016-09-01 16:15:13 -07001295 * Signal userspace if we're not exiting with a core dump
1296 * because we want to leave the value intact for debugging
1297 * purposes.
Roland McGrathfec1d012006-12-06 20:36:34 -08001298 */
Eric Dumazet9c8a8222009-08-06 15:09:28 -07001299 if (tsk->clear_child_tid) {
Michal Hocko735f2772016-09-01 16:15:13 -07001300 if (!(tsk->signal->flags & SIGNAL_GROUP_COREDUMP) &&
Eric Dumazet9c8a8222009-08-06 15:09:28 -07001301 atomic_read(&mm->mm_users) > 1) {
1302 /*
1303 * We don't check the error code - if userspace has
1304 * not set up a proper pointer then tough luck.
1305 */
1306 put_user(0, tsk->clear_child_tid);
Dominik Brodowski2de0db92018-03-11 11:34:26 +01001307 do_futex(tsk->clear_child_tid, FUTEX_WAKE,
1308 1, NULL, NULL, 0, 0);
Eric Dumazet9c8a8222009-08-06 15:09:28 -07001309 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001310 tsk->clear_child_tid = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001311 }
Konstantin Khlebnikovf7505d642012-05-31 16:26:21 -07001312
1313 /*
1314 * All done, finally we can wake up parent and return this mm to him.
1315 * Also kthread_stop() uses this completion for synchronization.
1316 */
1317 if (tsk->vfork_done)
1318 complete_vfork_done(tsk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001319}
1320
Thomas Gleixner4610ba72019-11-06 22:55:38 +01001321void exit_mm_release(struct task_struct *tsk, struct mm_struct *mm)
1322{
Thomas Gleixner150d7152019-11-06 22:55:39 +01001323 futex_exit_release(tsk);
Thomas Gleixner4610ba72019-11-06 22:55:38 +01001324 mm_release(tsk, mm);
1325}
1326
1327void exec_mm_release(struct task_struct *tsk, struct mm_struct *mm)
1328{
Thomas Gleixner150d7152019-11-06 22:55:39 +01001329 futex_exec_release(tsk);
Thomas Gleixner4610ba72019-11-06 22:55:38 +01001330 mm_release(tsk, mm);
1331}
1332
Nadav Amit13585fa2019-04-25 17:11:25 -07001333/**
1334 * dup_mm() - duplicates an existing mm structure
1335 * @tsk: the task_struct with which the new mm will be associated.
1336 * @oldmm: the mm to duplicate.
1337 *
1338 * Allocates a new mm structure and duplicates the provided @oldmm structure
1339 * content into it.
1340 *
1341 * Return: the duplicated mm or NULL on failure.
JANAK DESAIa0a7ec32006-02-07 12:59:01 -08001342 */
Nadav Amit13585fa2019-04-25 17:11:25 -07001343static struct mm_struct *dup_mm(struct task_struct *tsk,
1344 struct mm_struct *oldmm)
JANAK DESAIa0a7ec32006-02-07 12:59:01 -08001345{
Nadav Amit13585fa2019-04-25 17:11:25 -07001346 struct mm_struct *mm;
JANAK DESAIa0a7ec32006-02-07 12:59:01 -08001347 int err;
1348
JANAK DESAIa0a7ec32006-02-07 12:59:01 -08001349 mm = allocate_mm();
1350 if (!mm)
1351 goto fail_nomem;
1352
1353 memcpy(mm, oldmm, sizeof(*mm));
1354
Eric W. Biedermanbfedb582016-10-13 21:23:16 -05001355 if (!mm_init(mm, tsk, mm->user_ns))
JANAK DESAIa0a7ec32006-02-07 12:59:01 -08001356 goto fail_nomem;
1357
JANAK DESAIa0a7ec32006-02-07 12:59:01 -08001358 err = dup_mmap(mm, oldmm);
1359 if (err)
1360 goto free_pt;
1361
1362 mm->hiwater_rss = get_mm_rss(mm);
1363 mm->hiwater_vm = mm->total_vm;
1364
Hiroshi Shimamoto801460d2009-09-23 15:57:41 -07001365 if (mm->binfmt && !try_module_get(mm->binfmt->module))
1366 goto free_pt;
1367
JANAK DESAIa0a7ec32006-02-07 12:59:01 -08001368 return mm;
1369
1370free_pt:
Hiroshi Shimamoto801460d2009-09-23 15:57:41 -07001371 /* don't put binfmt in mmput, we haven't got module yet */
1372 mm->binfmt = NULL;
Andrea Arcangelic3f3ce02019-05-14 15:40:46 -07001373 mm_init_owner(mm, NULL);
JANAK DESAIa0a7ec32006-02-07 12:59:01 -08001374 mmput(mm);
1375
1376fail_nomem:
1377 return NULL;
JANAK DESAIa0a7ec32006-02-07 12:59:01 -08001378}
1379
Daniel Rebelo de Oliveirafb0a6852011-07-26 16:08:39 -07001380static int copy_mm(unsigned long clone_flags, struct task_struct *tsk)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001381{
Daniel Rebelo de Oliveirafb0a6852011-07-26 16:08:39 -07001382 struct mm_struct *mm, *oldmm;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001383 int retval;
1384
1385 tsk->min_flt = tsk->maj_flt = 0;
1386 tsk->nvcsw = tsk->nivcsw = 0;
Mandeep Singh Baines17406b82009-02-06 15:37:47 -08001387#ifdef CONFIG_DETECT_HUNG_TASK
1388 tsk->last_switch_count = tsk->nvcsw + tsk->nivcsw;
Dmitry Vyukova2e51442018-08-21 21:55:52 -07001389 tsk->last_switch_time = 0;
Mandeep Singh Baines17406b82009-02-06 15:37:47 -08001390#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07001391
1392 tsk->mm = NULL;
1393 tsk->active_mm = NULL;
1394
1395 /*
1396 * Are we cloning a kernel thread?
1397 *
1398 * We need to steal a active VM for that..
1399 */
1400 oldmm = current->mm;
1401 if (!oldmm)
1402 return 0;
1403
Davidlohr Bueso615d6e82014-04-07 15:37:25 -07001404 /* initialize the new vmacache entries */
1405 vmacache_flush(tsk);
1406
Linus Torvalds1da177e2005-04-16 15:20:36 -07001407 if (clone_flags & CLONE_VM) {
Vegard Nossum3fce3712017-02-27 14:30:10 -08001408 mmget(oldmm);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001409 mm = oldmm;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001410 goto good_mm;
1411 }
1412
1413 retval = -ENOMEM;
Nadav Amit13585fa2019-04-25 17:11:25 -07001414 mm = dup_mm(tsk, current->mm);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001415 if (!mm)
1416 goto fail_nomem;
1417
Linus Torvalds1da177e2005-04-16 15:20:36 -07001418good_mm:
1419 tsk->mm = mm;
1420 tsk->active_mm = mm;
1421 return 0;
1422
Linus Torvalds1da177e2005-04-16 15:20:36 -07001423fail_nomem:
1424 return retval;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001425}
1426
Alexey Dobriyana39bc512007-10-18 23:41:10 -07001427static int copy_fs(unsigned long clone_flags, struct task_struct *tsk)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001428{
Al Viro498052b2009-03-30 07:20:30 -04001429 struct fs_struct *fs = current->fs;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001430 if (clone_flags & CLONE_FS) {
Al Viro498052b2009-03-30 07:20:30 -04001431 /* tsk->fs is already what we want */
Nick Piggin2a4419b2010-08-18 04:37:33 +10001432 spin_lock(&fs->lock);
Al Viro498052b2009-03-30 07:20:30 -04001433 if (fs->in_exec) {
Nick Piggin2a4419b2010-08-18 04:37:33 +10001434 spin_unlock(&fs->lock);
Al Viro498052b2009-03-30 07:20:30 -04001435 return -EAGAIN;
1436 }
1437 fs->users++;
Nick Piggin2a4419b2010-08-18 04:37:33 +10001438 spin_unlock(&fs->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001439 return 0;
1440 }
Al Viro498052b2009-03-30 07:20:30 -04001441 tsk->fs = copy_fs_struct(fs);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001442 if (!tsk->fs)
1443 return -ENOMEM;
1444 return 0;
1445}
1446
Daniel Rebelo de Oliveirafb0a6852011-07-26 16:08:39 -07001447static int copy_files(unsigned long clone_flags, struct task_struct *tsk)
JANAK DESAIa016f332006-02-07 12:59:02 -08001448{
1449 struct files_struct *oldf, *newf;
1450 int error = 0;
1451
1452 /*
1453 * A background process may not have any files ...
1454 */
1455 oldf = current->files;
1456 if (!oldf)
1457 goto out;
1458
1459 if (clone_flags & CLONE_FILES) {
1460 atomic_inc(&oldf->count);
1461 goto out;
1462 }
1463
Christian Brauner60997c32020-06-03 21:48:55 +02001464 newf = dup_fd(oldf, NR_OPEN_MAX, &error);
JANAK DESAIa016f332006-02-07 12:59:02 -08001465 if (!newf)
1466 goto out;
1467
1468 tsk->files = newf;
1469 error = 0;
1470out:
1471 return error;
1472}
1473
Jens Axboefadad8782008-01-24 08:54:47 +01001474static int copy_io(unsigned long clone_flags, struct task_struct *tsk)
Jens Axboefd0928d2008-01-24 08:52:45 +01001475{
1476#ifdef CONFIG_BLOCK
1477 struct io_context *ioc = current->io_context;
Tejun Heo6e736be2011-12-14 00:33:38 +01001478 struct io_context *new_ioc;
Jens Axboefd0928d2008-01-24 08:52:45 +01001479
1480 if (!ioc)
1481 return 0;
Jens Axboefadad8782008-01-24 08:54:47 +01001482 /*
1483 * Share io context with parent, if CLONE_IO is set
1484 */
1485 if (clone_flags & CLONE_IO) {
Tejun Heo3d487492012-03-05 13:15:25 -08001486 ioc_task_link(ioc);
1487 tsk->io_context = ioc;
Jens Axboefadad8782008-01-24 08:54:47 +01001488 } else if (ioprio_valid(ioc->ioprio)) {
Tejun Heo6e736be2011-12-14 00:33:38 +01001489 new_ioc = get_task_io_context(tsk, GFP_KERNEL, NUMA_NO_NODE);
1490 if (unlikely(!new_ioc))
Jens Axboefd0928d2008-01-24 08:52:45 +01001491 return -ENOMEM;
1492
Tejun Heo6e736be2011-12-14 00:33:38 +01001493 new_ioc->ioprio = ioc->ioprio;
Tejun Heo11a31222012-02-07 07:51:30 +01001494 put_io_context(new_ioc);
Jens Axboefd0928d2008-01-24 08:52:45 +01001495 }
1496#endif
1497 return 0;
1498}
1499
Alexey Dobriyana39bc512007-10-18 23:41:10 -07001500static int copy_sighand(unsigned long clone_flags, struct task_struct *tsk)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001501{
1502 struct sighand_struct *sig;
1503
Zhaolei60348802009-01-06 14:40:46 -08001504 if (clone_flags & CLONE_SIGHAND) {
Elena Reshetovad036bda2019-01-18 14:27:26 +02001505 refcount_inc(&current->sighand->count);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001506 return 0;
1507 }
1508 sig = kmem_cache_alloc(sighand_cachep, GFP_KERNEL);
Madhuparna Bhowmik0c282b02020-01-27 23:28:21 +05301509 RCU_INIT_POINTER(tsk->sighand, sig);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001510 if (!sig)
1511 return -ENOMEM;
Peter Zijlstra9d7fb042015-06-30 11:30:54 +02001512
Elena Reshetovad036bda2019-01-18 14:27:26 +02001513 refcount_set(&sig->count, 1);
Jann Horn06e62a42018-08-21 22:00:58 -07001514 spin_lock_irq(&current->sighand->siglock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001515 memcpy(sig->action, current->sighand->action, sizeof(sig->action));
Jann Horn06e62a42018-08-21 22:00:58 -07001516 spin_unlock_irq(&current->sighand->siglock);
Christian Braunerb612e5d2019-10-14 12:45:37 +02001517
1518 /* Reset all signal handler not set to SIG_IGN to SIG_DFL. */
1519 if (clone_flags & CLONE_CLEAR_SIGHAND)
1520 flush_signal_handlers(tsk, 0);
1521
Linus Torvalds1da177e2005-04-16 15:20:36 -07001522 return 0;
1523}
1524
Oleg Nesterova7e53282006-03-28 16:11:27 -08001525void __cleanup_sighand(struct sighand_struct *sighand)
Oleg Nesterovc81addc2006-03-28 16:11:17 -08001526{
Elena Reshetovad036bda2019-01-18 14:27:26 +02001527 if (refcount_dec_and_test(&sighand->count)) {
Oleg Nesterovd80e7312012-02-24 20:07:11 +01001528 signalfd_cleanup(sighand);
Oleg Nesterov392809b2014-09-28 23:44:18 +02001529 /*
Paul E. McKenney5f0d5a32017-01-18 02:53:44 -08001530 * sighand_cachep is SLAB_TYPESAFE_BY_RCU so we can free it
Oleg Nesterov392809b2014-09-28 23:44:18 +02001531 * without an RCU grace period, see __lock_task_sighand().
1532 */
Oleg Nesterovc81addc2006-03-28 16:11:17 -08001533 kmem_cache_free(sighand_cachep, sighand);
Oleg Nesterovd80e7312012-02-24 20:07:11 +01001534 }
Oleg Nesterovc81addc2006-03-28 16:11:17 -08001535}
1536
Frank Mayharf06febc2008-09-12 09:54:39 -07001537/*
1538 * Initialize POSIX timer handling for a thread group.
1539 */
1540static void posix_cpu_timers_init_group(struct signal_struct *sig)
1541{
Thomas Gleixner2b699422019-08-21 21:09:04 +02001542 struct posix_cputimers *pct = &sig->posix_cputimers;
Jiri Slaby78d7d402010-03-05 13:42:54 -08001543 unsigned long cpu_limit;
1544
Jason Low316c1608d2015-04-28 13:00:20 -07001545 cpu_limit = READ_ONCE(sig->rlim[RLIMIT_CPU].rlim_cur);
Thomas Gleixner3a245c02019-08-21 21:09:06 +02001546 posix_cputimers_group_init(pct, cpu_limit);
Frank Mayharf06febc2008-09-12 09:54:39 -07001547}
1548
Alexey Dobriyana39bc512007-10-18 23:41:10 -07001549static int copy_signal(unsigned long clone_flags, struct task_struct *tsk)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001550{
1551 struct signal_struct *sig;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001552
Oleg Nesterov4ab6c082009-08-26 14:29:24 -07001553 if (clone_flags & CLONE_THREAD)
Peter Zijlstra490dea42008-11-24 17:06:57 +01001554 return 0;
Oleg Nesterov6279a7512009-03-27 01:06:07 +01001555
Veaceslav Falicoa56704e2010-03-10 15:23:01 -08001556 sig = kmem_cache_zalloc(signal_cachep, GFP_KERNEL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001557 tsk->signal = sig;
1558 if (!sig)
1559 return -ENOMEM;
1560
Oleg Nesterovb3ac0222010-05-26 14:43:24 -07001561 sig->nr_threads = 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001562 atomic_set(&sig->live, 1);
Elena Reshetova60d4de32019-01-18 14:27:27 +02001563 refcount_set(&sig->sigcnt, 1);
Oleg Nesterov0c740d02014-01-21 15:49:56 -08001564
1565 /* list_add(thread_node, thread_head) without INIT_LIST_HEAD() */
1566 sig->thread_head = (struct list_head)LIST_HEAD_INIT(tsk->thread_node);
1567 tsk->thread_node = (struct list_head)LIST_HEAD_INIT(sig->thread_head);
1568
Linus Torvalds1da177e2005-04-16 15:20:36 -07001569 init_waitqueue_head(&sig->wait_chldexit);
Oleg Nesterovdb51aec2008-04-30 00:52:52 -07001570 sig->curr_target = tsk;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001571 init_sigpending(&sig->shared_pending);
Eric W. Biedermanc3ad2c32018-07-23 15:20:37 -05001572 INIT_HLIST_HEAD(&sig->multiprocess);
Rik van Riele78c3492014-08-16 13:40:10 -04001573 seqlock_init(&sig->stats_lock);
Peter Zijlstra9d7fb042015-06-30 11:30:54 +02001574 prev_cputime_init(&sig->prev_cputime);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001575
Nicolas Pitrebaa73d92016-11-11 00:10:10 -05001576#ifdef CONFIG_POSIX_TIMERS
Nicolas Pitreb18b6a92017-01-21 00:09:08 -05001577 INIT_LIST_HEAD(&sig->posix_timers);
Thomas Gleixnerc9cb2e32007-02-16 01:27:49 -08001578 hrtimer_init(&sig->real_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001579 sig->real_timer.function = it_real_fn;
Nicolas Pitrebaa73d92016-11-11 00:10:10 -05001580#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07001581
Linus Torvalds1da177e2005-04-16 15:20:36 -07001582 task_lock(current->group_leader);
1583 memcpy(sig->rlim, current->signal->rlim, sizeof sig->rlim);
1584 task_unlock(current->group_leader);
1585
Oleg Nesterov6279a7512009-03-27 01:06:07 +01001586 posix_cpu_timers_init_group(sig);
1587
Miloslav Trmac522ed772007-07-15 23:40:56 -07001588 tty_audit_fork(sig);
Mike Galbraith5091faa2010-11-30 14:18:03 +01001589 sched_autogroup_fork(sig);
Miloslav Trmac522ed772007-07-15 23:40:56 -07001590
David Rientjesa63d83f2010-08-09 17:19:46 -07001591 sig->oom_score_adj = current->signal->oom_score_adj;
Mandeep Singh Bainesdabb16f632011-01-13 15:46:05 -08001592 sig->oom_score_adj_min = current->signal->oom_score_adj_min;
KOSAKI Motohiro28b83c52009-09-21 17:03:13 -07001593
KOSAKI Motohiro9b1bf122010-10-27 15:34:08 -07001594 mutex_init(&sig->cred_guard_mutex);
Eric W. Biedermaneea96732020-03-25 10:03:36 -05001595 mutex_init(&sig->exec_update_mutex);
KOSAKI Motohiro9b1bf122010-10-27 15:34:08 -07001596
Linus Torvalds1da177e2005-04-16 15:20:36 -07001597 return 0;
1598}
1599
Kees Cookdbd952122014-06-27 15:18:48 -07001600static void copy_seccomp(struct task_struct *p)
1601{
1602#ifdef CONFIG_SECCOMP
1603 /*
1604 * Must be called with sighand->lock held, which is common to
1605 * all threads in the group. Holding cred_guard_mutex is not
1606 * needed because this new task is not yet running and cannot
1607 * be racing exec.
1608 */
Guenter Roeck69f6a342014-08-10 20:50:30 -07001609 assert_spin_locked(&current->sighand->siglock);
Kees Cookdbd952122014-06-27 15:18:48 -07001610
1611 /* Ref-count the new filter user, and assign it. */
1612 get_seccomp_filter(current);
1613 p->seccomp = current->seccomp;
1614
1615 /*
1616 * Explicitly enable no_new_privs here in case it got set
1617 * between the task_struct being duplicated and holding the
1618 * sighand lock. The seccomp state and nnp must be in sync.
1619 */
1620 if (task_no_new_privs(current))
1621 task_set_no_new_privs(p);
1622
1623 /*
1624 * If the parent gained a seccomp mode after copying thread
1625 * flags and between before we held the sighand lock, we have
1626 * to manually enable the seccomp thread flag here.
1627 */
1628 if (p->seccomp.mode != SECCOMP_MODE_DISABLED)
1629 set_tsk_thread_flag(p, TIF_SECCOMP);
1630#endif
1631}
1632
Heiko Carstens17da2bd2009-01-14 14:14:10 +01001633SYSCALL_DEFINE1(set_tid_address, int __user *, tidptr)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001634{
1635 current->clear_child_tid = tidptr;
1636
Pavel Emelyanovb4888932007-10-18 23:40:14 -07001637 return task_pid_vnr(current);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001638}
1639
Alexey Dobriyana39bc512007-10-18 23:41:10 -07001640static void rt_mutex_init_task(struct task_struct *p)
Ingo Molnar23f78d4a2006-06-27 02:54:53 -07001641{
Thomas Gleixner1d615482009-11-17 14:54:03 +01001642 raw_spin_lock_init(&p->pi_lock);
Zilvinas Valinskase29e1752007-03-16 13:38:34 -08001643#ifdef CONFIG_RT_MUTEXES
Davidlohr Buesoa23ba902017-09-08 16:15:01 -07001644 p->pi_waiters = RB_ROOT_CACHED;
Xunlei Pange96a77052017-03-23 15:56:08 +01001645 p->pi_top_task = NULL;
Ingo Molnar23f78d4a2006-06-27 02:54:53 -07001646 p->pi_blocked_on = NULL;
Ingo Molnar23f78d4a2006-06-27 02:54:53 -07001647#endif
1648}
1649
Eric W. Biederman2c470472017-09-26 13:06:43 -05001650static inline void init_task_pid_links(struct task_struct *task)
1651{
1652 enum pid_type type;
1653
1654 for (type = PIDTYPE_PID; type < PIDTYPE_MAX; ++type) {
1655 INIT_HLIST_NODE(&task->pid_links[type]);
1656 }
1657}
1658
Oleg Nesterov81907732013-07-03 15:08:31 -07001659static inline void
1660init_task_pid(struct task_struct *task, enum pid_type type, struct pid *pid)
1661{
Eric W. Biederman2c470472017-09-26 13:06:43 -05001662 if (type == PIDTYPE_PID)
1663 task->thread_pid = pid;
1664 else
1665 task->signal->pids[type] = pid;
Oleg Nesterov81907732013-07-03 15:08:31 -07001666}
1667
Ingo Molnar6bfbaa52017-02-03 21:37:49 +01001668static inline void rcu_copy_process(struct task_struct *p)
1669{
1670#ifdef CONFIG_PREEMPT_RCU
1671 p->rcu_read_lock_nesting = 0;
1672 p->rcu_read_unlock_special.s = 0;
1673 p->rcu_blocked_node = NULL;
1674 INIT_LIST_HEAD(&p->rcu_node_entry);
1675#endif /* #ifdef CONFIG_PREEMPT_RCU */
1676#ifdef CONFIG_TASKS_RCU
1677 p->rcu_tasks_holdout = false;
1678 INIT_LIST_HEAD(&p->rcu_tasks_holdout_list);
1679 p->rcu_tasks_idle_cpu = -1;
1680#endif /* #ifdef CONFIG_TASKS_RCU */
Paul E. McKenneyd5f177d2020-03-09 19:56:53 -07001681#ifdef CONFIG_TASKS_TRACE_RCU
1682 p->trc_reader_nesting = 0;
Paul E. McKenney276c4102020-03-17 16:02:06 -07001683 p->trc_reader_special.s = 0;
Paul E. McKenneyd5f177d2020-03-09 19:56:53 -07001684 INIT_LIST_HEAD(&p->trc_holdout_list);
1685#endif /* #ifdef CONFIG_TASKS_TRACE_RCU */
Ingo Molnar6bfbaa52017-02-03 21:37:49 +01001686}
1687
Christian Brauner3695eae2019-07-28 00:22:29 +02001688struct pid *pidfd_pid(const struct file *file)
1689{
1690 if (file->f_op == &pidfd_fops)
1691 return file->private_data;
1692
1693 return ERR_PTR(-EBADF);
1694}
1695
Christian Braunerb3e583822019-03-27 13:04:15 +01001696static int pidfd_release(struct inode *inode, struct file *file)
1697{
1698 struct pid *pid = file->private_data;
1699
1700 file->private_data = NULL;
1701 put_pid(pid);
1702 return 0;
1703}
1704
1705#ifdef CONFIG_PROC_FS
Christian Kellner15d42eb2019-10-14 18:20:32 +02001706/**
1707 * pidfd_show_fdinfo - print information about a pidfd
1708 * @m: proc fdinfo file
1709 * @f: file referencing a pidfd
1710 *
1711 * Pid:
1712 * This function will print the pid that a given pidfd refers to in the
1713 * pid namespace of the procfs instance.
1714 * If the pid namespace of the process is not a descendant of the pid
1715 * namespace of the procfs instance 0 will be shown as its pid. This is
1716 * similar to calling getppid() on a process whose parent is outside of
1717 * its pid namespace.
1718 *
1719 * NSpid:
1720 * If pid namespaces are supported then this function will also print
1721 * the pid of a given pidfd refers to for all descendant pid namespaces
1722 * starting from the current pid namespace of the instance, i.e. the
1723 * Pid field and the first entry in the NSpid field will be identical.
1724 * If the pid namespace of the process is not a descendant of the pid
1725 * namespace of the procfs instance 0 will be shown as its first NSpid
1726 * entry and no others will be shown.
1727 * Note that this differs from the Pid and NSpid fields in
1728 * /proc/<pid>/status where Pid and NSpid are always shown relative to
1729 * the pid namespace of the procfs instance. The difference becomes
1730 * obvious when sending around a pidfd between pid namespaces from a
1731 * different branch of the tree, i.e. where no ancestoral relation is
1732 * present between the pid namespaces:
1733 * - create two new pid namespaces ns1 and ns2 in the initial pid
1734 * namespace (also take care to create new mount namespaces in the
1735 * new pid namespace and mount procfs)
1736 * - create a process with a pidfd in ns1
1737 * - send pidfd from ns1 to ns2
1738 * - read /proc/self/fdinfo/<pidfd> and observe that both Pid and NSpid
1739 * have exactly one entry, which is 0
1740 */
Christian Braunerb3e583822019-03-27 13:04:15 +01001741static void pidfd_show_fdinfo(struct seq_file *m, struct file *f)
1742{
Christian Braunerb3e583822019-03-27 13:04:15 +01001743 struct pid *pid = f->private_data;
Christian Brauner3d6d8da2019-10-17 12:18:28 +02001744 struct pid_namespace *ns;
1745 pid_t nr = -1;
Christian Braunerb3e583822019-03-27 13:04:15 +01001746
Christian Brauner3d6d8da2019-10-17 12:18:28 +02001747 if (likely(pid_has_task(pid, PIDTYPE_PID))) {
Alexey Gladkov9d78ede2020-05-18 20:07:38 +02001748 ns = proc_pid_ns(file_inode(m->file)->i_sb);
Christian Brauner3d6d8da2019-10-17 12:18:28 +02001749 nr = pid_nr_ns(pid, ns);
1750 }
1751
1752 seq_put_decimal_ll(m, "Pid:\t", nr);
Christian Kellner15d42eb2019-10-14 18:20:32 +02001753
1754#ifdef CONFIG_PID_NS
Christian Brauner3d6d8da2019-10-17 12:18:28 +02001755 seq_put_decimal_ll(m, "\nNSpid:\t", nr);
1756 if (nr > 0) {
Christian Kellner15d42eb2019-10-14 18:20:32 +02001757 int i;
1758
1759 /* If nr is non-zero it means that 'pid' is valid and that
1760 * ns, i.e. the pid namespace associated with the procfs
1761 * instance, is in the pid namespace hierarchy of pid.
1762 * Start at one below the already printed level.
1763 */
1764 for (i = ns->level + 1; i <= pid->level; i++)
Christian Brauner3d6d8da2019-10-17 12:18:28 +02001765 seq_put_decimal_ll(m, "\t", pid->numbers[i].nr);
Christian Kellner15d42eb2019-10-14 18:20:32 +02001766 }
1767#endif
Christian Braunerb3e583822019-03-27 13:04:15 +01001768 seq_putc(m, '\n');
1769}
1770#endif
1771
Joel Fernandes (Google)b53b0b92019-04-30 12:21:53 -04001772/*
1773 * Poll support for process exit notification.
1774 */
Luc Van Oostenryck9e777162019-11-20 01:33:20 +01001775static __poll_t pidfd_poll(struct file *file, struct poll_table_struct *pts)
Joel Fernandes (Google)b53b0b92019-04-30 12:21:53 -04001776{
Joel Fernandes (Google)b53b0b92019-04-30 12:21:53 -04001777 struct pid *pid = file->private_data;
Luc Van Oostenryck9e777162019-11-20 01:33:20 +01001778 __poll_t poll_flags = 0;
Joel Fernandes (Google)b53b0b92019-04-30 12:21:53 -04001779
1780 poll_wait(file, &pid->wait_pidfd, pts);
1781
Joel Fernandes (Google)b53b0b92019-04-30 12:21:53 -04001782 /*
1783 * Inform pollers only when the whole thread group exits.
1784 * If the thread group leader exits before all other threads in the
1785 * group, then poll(2) should block, similar to the wait(2) family.
1786 */
Eric W. Biederman38fd525a2020-07-01 07:30:06 -05001787 if (thread_group_exited(pid))
Luc Van Oostenryck9e777162019-11-20 01:33:20 +01001788 poll_flags = EPOLLIN | EPOLLRDNORM;
Joel Fernandes (Google)b53b0b92019-04-30 12:21:53 -04001789
1790 return poll_flags;
1791}
1792
Christian Braunerb3e583822019-03-27 13:04:15 +01001793const struct file_operations pidfd_fops = {
1794 .release = pidfd_release,
Joel Fernandes (Google)b53b0b92019-04-30 12:21:53 -04001795 .poll = pidfd_poll,
Christian Braunerb3e583822019-03-27 13:04:15 +01001796#ifdef CONFIG_PROC_FS
1797 .show_fdinfo = pidfd_show_fdinfo,
1798#endif
1799};
1800
Andrea Arcangelic3f3ce02019-05-14 15:40:46 -07001801static void __delayed_free_task(struct rcu_head *rhp)
1802{
1803 struct task_struct *tsk = container_of(rhp, struct task_struct, rcu);
1804
1805 free_task(tsk);
1806}
1807
1808static __always_inline void delayed_free_task(struct task_struct *tsk)
1809{
1810 if (IS_ENABLED(CONFIG_MEMCG))
1811 call_rcu(&tsk->rcu, __delayed_free_task);
1812 else
1813 free_task(tsk);
1814}
1815
Suren Baghdasaryan67197a42020-10-13 16:58:35 -07001816static void copy_oom_score_adj(u64 clone_flags, struct task_struct *tsk)
1817{
1818 /* Skip if kernel thread */
1819 if (!tsk->mm)
1820 return;
1821
1822 /* Skip if spawning a thread or using vfork */
1823 if ((clone_flags & (CLONE_VM | CLONE_THREAD | CLONE_VFORK)) != CLONE_VM)
1824 return;
1825
1826 /* We need to synchronize with __set_oom_adj */
1827 mutex_lock(&oom_adj_mutex);
1828 set_bit(MMF_MULTIPROCESS, &tsk->mm->flags);
1829 /* Update the values in case they were changed after copy_signal */
1830 tsk->signal->oom_score_adj = current->signal->oom_score_adj;
1831 tsk->signal->oom_score_adj_min = current->signal->oom_score_adj_min;
1832 mutex_unlock(&oom_adj_mutex);
1833}
1834
Frank Mayharf06febc2008-09-12 09:54:39 -07001835/*
Linus Torvalds1da177e2005-04-16 15:20:36 -07001836 * This creates a new process as a copy of the old one,
1837 * but does not actually start it yet.
1838 *
1839 * It copies the registers, and all the appropriate
1840 * parts of the process environment (as per the clone
1841 * flags). The actual kick-off is left to the caller.
1842 */
Emese Revfy0766f782016-06-20 20:42:34 +02001843static __latent_entropy struct task_struct *copy_process(
Roland McGrath09a05392008-07-25 19:45:47 -07001844 struct pid *pid,
Josh Triplett3033f14a2015-06-25 15:01:19 -07001845 int trace,
Christian Brauner7f192e32019-05-25 11:36:41 +02001846 int node,
1847 struct kernel_clone_args *args)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001848{
Christian Braunerb3e583822019-03-27 13:04:15 +01001849 int pidfd = -1, retval;
Mariusz Kozlowskia24efe62007-10-18 23:41:09 -07001850 struct task_struct *p;
Eric W. Biedermanc3ad2c32018-07-23 15:20:37 -05001851 struct multiprocess_signals delayed;
Al Viro6fd2fe42019-06-26 22:22:09 -04001852 struct file *pidfile = NULL;
Christian Brauner7f192e32019-05-25 11:36:41 +02001853 u64 clone_flags = args->flags;
Andrei Vagin769071a2019-11-12 01:26:52 +00001854 struct nsproxy *nsp = current->nsproxy;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001855
Marcos Paulo de Souza667b6092018-02-06 15:39:34 -08001856 /*
1857 * Don't allow sharing the root directory with processes in a different
1858 * namespace
1859 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001860 if ((clone_flags & (CLONE_NEWNS|CLONE_FS)) == (CLONE_NEWNS|CLONE_FS))
1861 return ERR_PTR(-EINVAL);
1862
Eric W. Biedermane66eded2013-03-13 11:51:49 -07001863 if ((clone_flags & (CLONE_NEWUSER|CLONE_FS)) == (CLONE_NEWUSER|CLONE_FS))
1864 return ERR_PTR(-EINVAL);
1865
Linus Torvalds1da177e2005-04-16 15:20:36 -07001866 /*
1867 * Thread groups must share signals as well, and detached threads
1868 * can only be started up within the thread group.
1869 */
1870 if ((clone_flags & CLONE_THREAD) && !(clone_flags & CLONE_SIGHAND))
1871 return ERR_PTR(-EINVAL);
1872
1873 /*
1874 * Shared signal handlers imply shared VM. By way of the above,
1875 * thread groups also imply shared VM. Blocking this case allows
1876 * for various simplifications in other code.
1877 */
1878 if ((clone_flags & CLONE_SIGHAND) && !(clone_flags & CLONE_VM))
1879 return ERR_PTR(-EINVAL);
1880
Sukadev Bhattiprolu123be072009-09-23 15:57:20 -07001881 /*
1882 * Siblings of global init remain as zombies on exit since they are
1883 * not reaped by their parent (swapper). To solve this and to avoid
1884 * multi-rooted process trees, prevent global and container-inits
1885 * from creating siblings.
1886 */
1887 if ((clone_flags & CLONE_PARENT) &&
1888 current->signal->flags & SIGNAL_UNKILLABLE)
1889 return ERR_PTR(-EINVAL);
1890
Eric W. Biederman8382fca2012-12-20 19:26:06 -08001891 /*
Oleg Nesterov40a0d322013-09-11 14:19:41 -07001892 * If the new process will be in a different pid or user namespace
Eric W. Biedermanfaf00da2015-08-10 18:25:44 -05001893 * do not allow it to share a thread group with the forking task.
Eric W. Biederman8382fca2012-12-20 19:26:06 -08001894 */
Eric W. Biedermanfaf00da2015-08-10 18:25:44 -05001895 if (clone_flags & CLONE_THREAD) {
Oleg Nesterov40a0d322013-09-11 14:19:41 -07001896 if ((clone_flags & (CLONE_NEWUSER | CLONE_NEWPID)) ||
Andrei Vagin769071a2019-11-12 01:26:52 +00001897 (task_active_pid_ns(current) != nsp->pid_ns_for_children))
1898 return ERR_PTR(-EINVAL);
1899 }
1900
1901 /*
1902 * If the new process will be in a different time namespace
1903 * do not allow it to share VM or a thread group with the forking task.
1904 */
1905 if (clone_flags & (CLONE_THREAD | CLONE_VM)) {
1906 if (nsp->time_ns != nsp->time_ns_for_children)
Oleg Nesterov40a0d322013-09-11 14:19:41 -07001907 return ERR_PTR(-EINVAL);
1908 }
Eric W. Biederman8382fca2012-12-20 19:26:06 -08001909
Christian Braunerb3e583822019-03-27 13:04:15 +01001910 if (clone_flags & CLONE_PIDFD) {
Christian Braunerb3e583822019-03-27 13:04:15 +01001911 /*
Christian Braunerb3e583822019-03-27 13:04:15 +01001912 * - CLONE_DETACHED is blocked so that we can potentially
1913 * reuse it later for CLONE_PIDFD.
1914 * - CLONE_THREAD is blocked until someone really needs it.
1915 */
Christian Brauner7f192e32019-05-25 11:36:41 +02001916 if (clone_flags & (CLONE_DETACHED | CLONE_THREAD))
Christian Braunerb3e583822019-03-27 13:04:15 +01001917 return ERR_PTR(-EINVAL);
Christian Braunerb3e583822019-03-27 13:04:15 +01001918 }
1919
Eric W. Biedermanc3ad2c32018-07-23 15:20:37 -05001920 /*
1921 * Force any signals received before this point to be delivered
1922 * before the fork happens. Collect up signals sent to multiple
1923 * processes that happen during the fork and delay them so that
1924 * they appear to happen after the fork.
1925 */
1926 sigemptyset(&delayed.signal);
1927 INIT_HLIST_NODE(&delayed.node);
1928
1929 spin_lock_irq(&current->sighand->siglock);
1930 if (!(clone_flags & CLONE_THREAD))
1931 hlist_add_head(&delayed.node, &current->signal->multiprocess);
1932 recalc_sigpending();
1933 spin_unlock_irq(&current->sighand->siglock);
1934 retval = -ERESTARTNOINTR;
1935 if (signal_pending(current))
1936 goto fork_out;
1937
Linus Torvalds1da177e2005-04-16 15:20:36 -07001938 retval = -ENOMEM;
Andi Kleen725fc622016-05-23 16:24:05 -07001939 p = dup_task_struct(current, node);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001940 if (!p)
1941 goto fork_out;
1942
Vegard Nossum4d6501dc2017-05-09 09:39:59 +02001943 /*
1944 * This _must_ happen before we call free_task(), i.e. before we jump
1945 * to any of the bad_fork_* labels. This is to avoid freeing
1946 * p->set_child_tid which is (ab)used as a kthread's data pointer for
1947 * kernel threads (PF_KTHREAD).
1948 */
Christian Brauner7f192e32019-05-25 11:36:41 +02001949 p->set_child_tid = (clone_flags & CLONE_CHILD_SETTID) ? args->child_tid : NULL;
Vegard Nossum4d6501dc2017-05-09 09:39:59 +02001950 /*
1951 * Clear TID on mm_release()?
1952 */
Christian Brauner7f192e32019-05-25 11:36:41 +02001953 p->clear_child_tid = (clone_flags & CLONE_CHILD_CLEARTID) ? args->child_tid : NULL;
Vegard Nossum4d6501dc2017-05-09 09:39:59 +02001954
Steven Rostedtf7e8b612009-06-02 16:39:48 -04001955 ftrace_graph_init_task(p);
1956
Peter Zijlstrabea493a2006-10-17 00:10:33 -07001957 rt_mutex_init_task(p);
1958
Peter Zijlstraa21ee602020-05-25 12:22:41 +02001959 lockdep_assert_irqs_enabled();
Ingo Molnard12c1a32008-07-14 12:09:28 +02001960#ifdef CONFIG_PROVE_LOCKING
Ingo Molnarde30a2b2006-07-03 00:24:42 -07001961 DEBUG_LOCKS_WARN_ON(!p->softirqs_enabled);
1962#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07001963 retval = -EAGAIN;
David Howells3b11a1d2008-11-14 10:39:26 +11001964 if (atomic_read(&p->real_cred->user->processes) >=
Jiri Slaby78d7d402010-03-05 13:42:54 -08001965 task_rlimit(p, RLIMIT_NPROC)) {
Eric Parisb57922b2013-07-03 15:08:29 -07001966 if (p->real_cred->user != INIT_USER &&
1967 !capable(CAP_SYS_RESOURCE) && !capable(CAP_SYS_ADMIN))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001968 goto bad_fork_free;
1969 }
Vasiliy Kulikov72fa5992011-08-08 19:02:04 +04001970 current->flags &= ~PF_NPROC_EXCEEDED;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001971
David Howellsf1752ee2008-11-14 10:39:17 +11001972 retval = copy_creds(p, clone_flags);
1973 if (retval < 0)
1974 goto bad_fork_free;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001975
1976 /*
1977 * If multiple threads are within copy_process(), then this check
1978 * triggers too late. This doesn't hurt, the check is only there
1979 * to stop root fork bombs.
1980 */
Li Zefan04ec93f2009-02-06 08:17:19 +00001981 retval = -EAGAIN;
Weilong Chenc17d1a32020-06-23 12:12:40 +08001982 if (data_race(nr_threads >= max_threads))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001983 goto bad_fork_cleanup_count;
1984
Shailabh Nagarca74e922006-07-14 00:24:36 -07001985 delayacct_tsk_init(p); /* Must remain after dup_task_struct() */
Peter Zijlstrac1de45c2016-11-28 23:03:05 -08001986 p->flags &= ~(PF_SUPERPRIV | PF_WQ_WORKER | PF_IDLE);
David Rientjes514ddb42014-04-07 15:37:27 -07001987 p->flags |= PF_FORKNOEXEC;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001988 INIT_LIST_HEAD(&p->children);
1989 INIT_LIST_HEAD(&p->sibling);
Paul E. McKenneyf41d9112009-08-22 13:56:52 -07001990 rcu_copy_process(p);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001991 p->vfork_done = NULL;
1992 spin_lock_init(&p->alloc_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001993
Linus Torvalds1da177e2005-04-16 15:20:36 -07001994 init_sigpending(&p->pending);
1995
Martin Schwidefsky64861632011-12-15 14:56:09 +01001996 p->utime = p->stime = p->gtime = 0;
Stanislaw Gruszka40565b52016-11-15 03:06:51 +01001997#ifdef CONFIG_ARCH_HAS_SCALED_CPUTIME
Martin Schwidefsky64861632011-12-15 14:56:09 +01001998 p->utimescaled = p->stimescaled = 0;
Stanislaw Gruszka40565b52016-11-15 03:06:51 +01001999#endif
Peter Zijlstra9d7fb042015-06-30 11:30:54 +02002000 prev_cputime_init(&p->prev_cputime);
2001
Frederic Weisbecker6a616712012-12-16 20:00:34 +01002002#ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN
Frederic Weisbeckerbac5b6b2017-06-29 19:15:10 +02002003 seqcount_init(&p->vtime.seqcount);
2004 p->vtime.starttime = 0;
2005 p->vtime.state = VTIME_INACTIVE;
Frederic Weisbecker6a616712012-12-16 20:00:34 +01002006#endif
2007
Jens Axboe0f212202020-09-13 13:09:39 -06002008#ifdef CONFIG_IO_URING
2009 p->io_uring = NULL;
2010#endif
2011
KAMEZAWA Hiroyukia3a2e762010-04-06 14:34:42 -07002012#if defined(SPLIT_RSS_COUNTING)
2013 memset(&p->rss_stat, 0, sizeof(p->rss_stat));
2014#endif
Balbir Singh172ba842007-07-09 18:52:00 +02002015
Arjan van de Ven69766752008-09-01 15:52:40 -07002016 p->default_timer_slack_ns = current->timer_slack_ns;
2017
Johannes Weinereb414682018-10-26 15:06:27 -07002018#ifdef CONFIG_PSI
2019 p->psi_flags = 0;
2020#endif
2021
Andrea Righi59954772008-07-27 17:29:15 +02002022 task_io_accounting_init(&p->ioac);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002023 acct_clear_integrals(p);
2024
Thomas Gleixner3a245c02019-08-21 21:09:06 +02002025 posix_cputimers_init(&p->posix_cputimers);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002026
Linus Torvalds1da177e2005-04-16 15:20:36 -07002027 p->io_context = NULL;
Richard Guy Briggsc0b0ae82018-05-12 21:58:21 -04002028 audit_set_context(p, NULL);
Paul Menageb4f48b62007-10-18 23:39:33 -07002029 cgroup_fork(p);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002030#ifdef CONFIG_NUMA
Lee Schermerhorn846a16b2008-04-28 02:13:09 -07002031 p->mempolicy = mpol_dup(p->mempolicy);
Daniel Rebelo de Oliveirafb0a6852011-07-26 16:08:39 -07002032 if (IS_ERR(p->mempolicy)) {
2033 retval = PTR_ERR(p->mempolicy);
2034 p->mempolicy = NULL;
Li Zefane8604cb2014-03-28 15:18:27 +08002035 goto bad_fork_cleanup_threadgroup_lock;
Daniel Rebelo de Oliveirafb0a6852011-07-26 16:08:39 -07002036 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002037#endif
Michal Hocko778d3b02011-07-26 16:08:30 -07002038#ifdef CONFIG_CPUSETS
2039 p->cpuset_mem_spread_rotor = NUMA_NO_NODE;
2040 p->cpuset_slab_spread_rotor = NUMA_NO_NODE;
Ahmed S. Darwishb7505862020-07-20 17:55:19 +02002041 seqcount_spinlock_init(&p->mems_allowed_seq, &p->alloc_lock);
Michal Hocko778d3b02011-07-26 16:08:30 -07002042#endif
Ingo Molnarde30a2b2006-07-03 00:24:42 -07002043#ifdef CONFIG_TRACE_IRQFLAGS
Marco Elver0584df92020-07-29 13:09:15 +02002044 memset(&p->irqtrace, 0, sizeof(p->irqtrace));
2045 p->irqtrace.hardirq_disable_ip = _THIS_IP_;
2046 p->irqtrace.softirq_enable_ip = _THIS_IP_;
2047 p->softirqs_enabled = 1;
2048 p->softirq_context = 0;
Ingo Molnarde30a2b2006-07-03 00:24:42 -07002049#endif
David Hildenbrand8bcbde52015-05-11 17:52:06 +02002050
2051 p->pagefault_disabled = 0;
2052
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07002053#ifdef CONFIG_LOCKDEP
Byungchul Parkb09be672017-08-07 16:12:52 +09002054 lockdep_init_task(p);
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07002055#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07002056
Ingo Molnar408894e2006-01-09 15:59:20 -08002057#ifdef CONFIG_DEBUG_MUTEXES
2058 p->blocked_on = NULL; /* not blocked yet */
2059#endif
Kent Overstreetcafe5632013-03-23 16:11:31 -07002060#ifdef CONFIG_BCACHE
2061 p->sequential_io = 0;
2062 p->sequential_io_avg = 0;
2063#endif
Markus Metzger0f481402009-04-03 16:43:48 +02002064
Srivatsa Vaddagiri3c90e6e2007-11-09 22:39:39 +01002065 /* Perform scheduler related setup. Assign this task to a CPU. */
Dario Faggioliaab03e02013-11-28 11:14:43 +01002066 retval = sched_fork(clone_flags, p);
2067 if (retval)
2068 goto bad_fork_cleanup_policy;
Peter Zijlstra6ab423e2009-05-25 14:45:27 +02002069
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002070 retval = perf_event_init_task(p);
Peter Zijlstra6ab423e2009-05-25 14:45:27 +02002071 if (retval)
2072 goto bad_fork_cleanup_policy;
Daniel Rebelo de Oliveirafb0a6852011-07-26 16:08:39 -07002073 retval = audit_alloc(p);
2074 if (retval)
Peter Zijlstra6c72e3502014-10-02 16:17:02 -07002075 goto bad_fork_cleanup_perf;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002076 /* copy all the process information */
Jack Millerab602f72014-08-08 14:23:19 -07002077 shm_init_task(p);
Tetsuo Handae4e55b42017-03-24 20:46:33 +09002078 retval = security_task_alloc(p, clone_flags);
Daniel Rebelo de Oliveirafb0a6852011-07-26 16:08:39 -07002079 if (retval)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002080 goto bad_fork_cleanup_audit;
Tetsuo Handae4e55b42017-03-24 20:46:33 +09002081 retval = copy_semundo(clone_flags, p);
2082 if (retval)
2083 goto bad_fork_cleanup_security;
Daniel Rebelo de Oliveirafb0a6852011-07-26 16:08:39 -07002084 retval = copy_files(clone_flags, p);
2085 if (retval)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002086 goto bad_fork_cleanup_semundo;
Daniel Rebelo de Oliveirafb0a6852011-07-26 16:08:39 -07002087 retval = copy_fs(clone_flags, p);
2088 if (retval)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002089 goto bad_fork_cleanup_files;
Daniel Rebelo de Oliveirafb0a6852011-07-26 16:08:39 -07002090 retval = copy_sighand(clone_flags, p);
2091 if (retval)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002092 goto bad_fork_cleanup_fs;
Daniel Rebelo de Oliveirafb0a6852011-07-26 16:08:39 -07002093 retval = copy_signal(clone_flags, p);
2094 if (retval)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002095 goto bad_fork_cleanup_sighand;
Daniel Rebelo de Oliveirafb0a6852011-07-26 16:08:39 -07002096 retval = copy_mm(clone_flags, p);
2097 if (retval)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002098 goto bad_fork_cleanup_signal;
Daniel Rebelo de Oliveirafb0a6852011-07-26 16:08:39 -07002099 retval = copy_namespaces(clone_flags, p);
2100 if (retval)
David Howellsd84f4f92008-11-14 10:39:23 +11002101 goto bad_fork_cleanup_mm;
Daniel Rebelo de Oliveirafb0a6852011-07-26 16:08:39 -07002102 retval = copy_io(clone_flags, p);
2103 if (retval)
Jens Axboefd0928d2008-01-24 08:52:45 +01002104 goto bad_fork_cleanup_namespaces;
Christian Brauner714acdb2020-06-11 11:04:15 +02002105 retval = copy_thread(clone_flags, args->stack, args->stack_size, p, args->tls);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002106 if (retval)
Jens Axboefd0928d2008-01-24 08:52:45 +01002107 goto bad_fork_cleanup_io;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002108
Alexander Popovafaef012018-08-17 01:16:58 +03002109 stackleak_task_init(p);
2110
Pavel Emelyanov425fb2b2007-10-18 23:40:07 -07002111 if (pid != &init_struct_pid) {
Adrian Reber49cb2fc2019-11-15 13:36:20 +01002112 pid = alloc_pid(p->nsproxy->pid_ns_for_children, args->set_tid,
2113 args->set_tid_size);
Michal Hocko35f71bc2015-04-16 12:47:38 -07002114 if (IS_ERR(pid)) {
2115 retval = PTR_ERR(pid);
Jiri Slaby0740aa52016-05-20 17:00:25 -07002116 goto bad_fork_cleanup_thread;
Michal Hocko35f71bc2015-04-16 12:47:38 -07002117 }
Pavel Emelyanov425fb2b2007-10-18 23:40:07 -07002118 }
2119
Christian Braunerb3e583822019-03-27 13:04:15 +01002120 /*
2121 * This has to happen after we've potentially unshared the file
2122 * descriptor table (so that the pidfd doesn't leak into the child
2123 * if the fd table isn't shared).
2124 */
2125 if (clone_flags & CLONE_PIDFD) {
Al Viro6fd2fe42019-06-26 22:22:09 -04002126 retval = get_unused_fd_flags(O_RDWR | O_CLOEXEC);
Christian Braunerb3e583822019-03-27 13:04:15 +01002127 if (retval < 0)
2128 goto bad_fork_free_pid;
2129
2130 pidfd = retval;
Al Viro6fd2fe42019-06-26 22:22:09 -04002131
2132 pidfile = anon_inode_getfile("[pidfd]", &pidfd_fops, pid,
2133 O_RDWR | O_CLOEXEC);
2134 if (IS_ERR(pidfile)) {
2135 put_unused_fd(pidfd);
Christian Brauner28dd29c2019-07-01 16:01:46 +02002136 retval = PTR_ERR(pidfile);
Al Viro6fd2fe42019-06-26 22:22:09 -04002137 goto bad_fork_free_pid;
2138 }
2139 get_pid(pid); /* held by pidfile now */
2140
Christian Brauner7f192e32019-05-25 11:36:41 +02002141 retval = put_user(pidfd, args->pidfd);
Christian Braunerb3e583822019-03-27 13:04:15 +01002142 if (retval)
2143 goto bad_fork_put_pidfd;
2144 }
2145
Jens Axboe73c10102011-03-08 13:19:51 +01002146#ifdef CONFIG_BLOCK
2147 p->plug = NULL;
2148#endif
Thomas Gleixnerba31c1a42019-11-06 22:55:36 +01002149 futex_init_task(p);
2150
Linus Torvalds1da177e2005-04-16 15:20:36 -07002151 /*
GOTO Masanorif9a38792006-03-13 21:20:44 -08002152 * sigaltstack should be cleared when sharing the same VM
2153 */
2154 if ((clone_flags & (CLONE_VM|CLONE_VFORK)) == CLONE_VM)
Stas Sergeev2a742132016-04-14 23:20:04 +03002155 sas_ss_reset(p);
GOTO Masanorif9a38792006-03-13 21:20:44 -08002156
2157 /*
Oleg Nesterov65808072009-12-15 16:47:16 -08002158 * Syscall tracing and stepping should be turned off in the
2159 * child regardless of CLONE_PTRACE.
Linus Torvalds1da177e2005-04-16 15:20:36 -07002160 */
Oleg Nesterov65808072009-12-15 16:47:16 -08002161 user_disable_single_step(p);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002162 clear_tsk_thread_flag(p, TIF_SYSCALL_TRACE);
Laurent Viviered75e8d2005-09-03 15:57:18 -07002163#ifdef TIF_SYSCALL_EMU
2164 clear_tsk_thread_flag(p, TIF_SYSCALL_EMU);
2165#endif
Lin Fenge02c9b02019-05-14 15:42:34 -07002166 clear_tsk_latency_tracing(p);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002167
Linus Torvalds1da177e2005-04-16 15:20:36 -07002168 /* ok, now we should be set up.. */
Oleg Nesterov18c830d2013-07-03 15:08:32 -07002169 p->pid = pid_nr(pid);
2170 if (clone_flags & CLONE_THREAD) {
Oleg Nesterov18c830d2013-07-03 15:08:32 -07002171 p->group_leader = current->group_leader;
2172 p->tgid = current->tgid;
2173 } else {
Oleg Nesterov18c830d2013-07-03 15:08:32 -07002174 p->group_leader = p;
2175 p->tgid = p->pid;
2176 }
Oleg Nesterov5f8aadd2012-03-14 19:55:38 +01002177
Wu Fengguang9d823e82011-06-11 18:10:12 -06002178 p->nr_dirtied = 0;
2179 p->nr_dirtied_pause = 128 >> (PAGE_SHIFT - 10);
Wu Fengguang83712352011-06-11 19:25:42 -06002180 p->dirty_paused_when = 0;
Wu Fengguang9d823e82011-06-11 18:10:12 -06002181
Oleg Nesterovbb8cbbf2013-11-13 15:36:12 +01002182 p->pdeath_signal = 0;
Oleg Nesterov47e65322006-03-28 16:11:25 -08002183 INIT_LIST_HEAD(&p->thread_group);
Al Viro158e1642012-06-27 09:24:13 +04002184 p->task_works = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002185
Oleg Nesterov18c830d2013-07-03 15:08:32 -07002186 /*
Aleksa Sarai7e476822015-06-09 21:32:09 +10002187 * Ensure that the cgroup subsystem policies allow the new process to be
Randy Dunlap7b7b8a22020-10-15 20:10:28 -07002188 * forked. It should be noted that the new process's css_set can be changed
Aleksa Sarai7e476822015-06-09 21:32:09 +10002189 * between here and cgroup_post_fork() if an organisation operation is in
2190 * progress.
2191 */
Christian Brauneref2c41c2020-02-05 14:26:22 +01002192 retval = cgroup_can_fork(p, args);
Aleksa Sarai7e476822015-06-09 21:32:09 +10002193 if (retval)
Christian Brauner5a5cf5c2020-02-05 14:26:20 +01002194 goto bad_fork_put_pidfd;
Aleksa Sarai7e476822015-06-09 21:32:09 +10002195
2196 /*
David Herrmann7b558512019-01-08 13:58:52 +01002197 * From this point on we must avoid any synchronous user-space
2198 * communication until we take the tasklist-lock. In particular, we do
2199 * not want user-space to be able to predict the process start-time by
2200 * stalling fork(2) after we recorded the start_time but before it is
2201 * visible to the system.
2202 */
2203
2204 p->start_time = ktime_get_ns();
Peter Zijlstracf25e242019-11-07 11:07:58 +01002205 p->start_boottime = ktime_get_boottime_ns();
David Herrmann7b558512019-01-08 13:58:52 +01002206
2207 /*
Oleg Nesterov18c830d2013-07-03 15:08:32 -07002208 * Make it visible to the rest of the system, but dont wake it up yet.
2209 * Need tasklist lock for parent etc handling!
2210 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07002211 write_lock_irq(&tasklist_lock);
2212
Linus Torvalds1da177e2005-04-16 15:20:36 -07002213 /* CLONE_PARENT re-uses the old parent */
Oleg Nesterov2d5516c2009-03-02 22:58:45 +01002214 if (clone_flags & (CLONE_PARENT|CLONE_THREAD)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002215 p->real_parent = current->real_parent;
Oleg Nesterov2d5516c2009-03-02 22:58:45 +01002216 p->parent_exec_id = current->parent_exec_id;
Eddy Wub4e00442020-11-07 14:47:22 +08002217 if (clone_flags & CLONE_THREAD)
2218 p->exit_signal = -1;
2219 else
2220 p->exit_signal = current->group_leader->exit_signal;
Oleg Nesterov2d5516c2009-03-02 22:58:45 +01002221 } else {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002222 p->real_parent = current;
Oleg Nesterov2d5516c2009-03-02 22:58:45 +01002223 p->parent_exec_id = current->self_exec_id;
Eddy Wub4e00442020-11-07 14:47:22 +08002224 p->exit_signal = args->exit_signal;
Oleg Nesterov2d5516c2009-03-02 22:58:45 +01002225 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002226
Josh Poimboeufd83a7cb2017-02-13 19:42:40 -06002227 klp_copy_process(p);
2228
Oleg Nesterov3f17da62006-02-15 22:13:24 +03002229 spin_lock(&current->sighand->siglock);
Oleg Nesterov4a2c7a72006-03-28 16:11:26 -08002230
2231 /*
Kees Cookdbd952122014-06-27 15:18:48 -07002232 * Copy seccomp details explicitly here, in case they were changed
2233 * before holding sighand lock.
2234 */
2235 copy_seccomp(p);
2236
Mathieu Desnoyersd7822b12018-06-02 08:43:54 -04002237 rseq_fork(p, clone_flags);
2238
Eric W. Biederman4ca1d3e2018-07-13 15:30:33 -05002239 /* Don't start children in a dying pid namespace */
Gargi Sharmae8cfbc22017-11-17 15:30:34 -08002240 if (unlikely(!(ns_of_pid(pid)->pid_allocated & PIDNS_ADDING))) {
Kirill Tkhai3fd37222017-05-12 19:11:31 +03002241 retval = -ENOMEM;
2242 goto bad_fork_cancel_cgroup;
2243 }
Oleg Nesterov4a2c7a72006-03-28 16:11:26 -08002244
Eric W. Biederman7673bf52018-07-23 08:01:10 -05002245 /* Let kill terminate clone/fork in the middle */
2246 if (fatal_signal_pending(current)) {
2247 retval = -EINTR;
2248 goto bad_fork_cancel_cgroup;
2249 }
2250
Al Viro6fd2fe42019-06-26 22:22:09 -04002251 /* past the last point of failure */
2252 if (pidfile)
2253 fd_install(pidfd, pidfile);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002254
Eric W. Biederman2c470472017-09-26 13:06:43 -05002255 init_task_pid_links(p);
Oleg Nesterov73b9ebf2006-03-28 16:11:07 -08002256 if (likely(p->pid)) {
Tejun Heo4b9d33e2011-06-17 16:50:38 +02002257 ptrace_init_task(p, (clone_flags & CLONE_PTRACE) || trace);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002258
Oleg Nesterov81907732013-07-03 15:08:31 -07002259 init_task_pid(p, PIDTYPE_PID, pid);
Oleg Nesterov73b9ebf2006-03-28 16:11:07 -08002260 if (thread_group_leader(p)) {
Eric W. Biederman6883f812017-06-04 04:32:13 -05002261 init_task_pid(p, PIDTYPE_TGID, pid);
Oleg Nesterov81907732013-07-03 15:08:31 -07002262 init_task_pid(p, PIDTYPE_PGID, task_pgrp(current));
2263 init_task_pid(p, PIDTYPE_SID, task_session(current));
2264
Eric W. Biederman1c4042c2010-07-12 17:10:36 -07002265 if (is_child_reaper(pid)) {
Eric W. Biederman17cf22c2010-03-02 14:51:53 -08002266 ns_of_pid(pid)->child_reaper = p;
Eric W. Biederman1c4042c2010-07-12 17:10:36 -07002267 p->signal->flags |= SIGNAL_UNKILLABLE;
2268 }
Eric W. Biedermanc3ad2c32018-07-23 15:20:37 -05002269 p->signal->shared_pending.signal = delayed.signal;
Alan Cox9c9f4de2008-10-13 10:37:26 +01002270 p->signal->tty = tty_kref_get(current->signal->tty);
Pavel Tikhomirov749860c2017-01-30 18:06:12 +03002271 /*
2272 * Inherit has_child_subreaper flag under the same
2273 * tasklist_lock with adding child to the process tree
2274 * for propagate_has_child_subreaper optimization.
2275 */
2276 p->signal->has_child_subreaper = p->real_parent->signal->has_child_subreaper ||
2277 p->real_parent->signal->is_child_subreaper;
Oleg Nesterov9cd80bb2009-12-17 15:27:15 -08002278 list_add_tail(&p->sibling, &p->real_parent->children);
Eric W. Biederman5e85d4a2006-04-18 22:20:16 -07002279 list_add_tail_rcu(&p->tasks, &init_task.tasks);
Eric W. Biederman6883f812017-06-04 04:32:13 -05002280 attach_pid(p, PIDTYPE_TGID);
Oleg Nesterov81907732013-07-03 15:08:31 -07002281 attach_pid(p, PIDTYPE_PGID);
2282 attach_pid(p, PIDTYPE_SID);
Christoph Lameter909ea962010-12-08 16:22:55 +01002283 __this_cpu_inc(process_counts);
Oleg Nesterov80628ca2013-07-03 15:08:30 -07002284 } else {
2285 current->signal->nr_threads++;
2286 atomic_inc(&current->signal->live);
Elena Reshetova60d4de32019-01-18 14:27:27 +02002287 refcount_inc(&current->signal->sigcnt);
Eric W. Biederman924de3b2018-07-23 13:38:00 -05002288 task_join_group_stop(p);
Oleg Nesterov80628ca2013-07-03 15:08:30 -07002289 list_add_tail_rcu(&p->thread_group,
2290 &p->group_leader->thread_group);
Oleg Nesterov0c740d02014-01-21 15:49:56 -08002291 list_add_tail_rcu(&p->thread_node,
2292 &p->signal->thread_head);
Oleg Nesterov73b9ebf2006-03-28 16:11:07 -08002293 }
Oleg Nesterov81907732013-07-03 15:08:31 -07002294 attach_pid(p, PIDTYPE_PID);
Oleg Nesterov73b9ebf2006-03-28 16:11:07 -08002295 nr_threads++;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002296 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002297 total_forks++;
Eric W. Biedermanc3ad2c32018-07-23 15:20:37 -05002298 hlist_del_init(&delayed.node);
Oleg Nesterov3f17da62006-02-15 22:13:24 +03002299 spin_unlock(&current->sighand->siglock);
Oleg Nesterov4af42062014-04-13 20:58:54 +02002300 syscall_tracepoint_update(p);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002301 write_unlock_irq(&tasklist_lock);
Oleg Nesterov4af42062014-04-13 20:58:54 +02002302
Andrew Mortonc13cf852005-11-28 13:43:48 -08002303 proc_fork_connector(p);
Qais Yousef13685c42020-07-16 12:03:45 +01002304 sched_post_fork(p);
Christian Brauneref2c41c2020-02-05 14:26:22 +01002305 cgroup_post_fork(p, args);
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002306 perf_event_fork(p);
KAMEZAWA Hiroyuki43d2b112012-01-10 15:08:09 -08002307
2308 trace_task_newtask(p, clone_flags);
Oleg Nesterov3ab67962013-10-16 19:39:37 +02002309 uprobe_copy_process(p, clone_flags);
KAMEZAWA Hiroyuki43d2b112012-01-10 15:08:09 -08002310
Suren Baghdasaryan67197a42020-10-13 16:58:35 -07002311 copy_oom_score_adj(clone_flags, p);
2312
Linus Torvalds1da177e2005-04-16 15:20:36 -07002313 return p;
2314
Aleksa Sarai7e476822015-06-09 21:32:09 +10002315bad_fork_cancel_cgroup:
Kirill Tkhai3fd37222017-05-12 19:11:31 +03002316 spin_unlock(&current->sighand->siglock);
2317 write_unlock_irq(&tasklist_lock);
Christian Brauneref2c41c2020-02-05 14:26:22 +01002318 cgroup_cancel_fork(p, args);
Christian Braunerb3e583822019-03-27 13:04:15 +01002319bad_fork_put_pidfd:
Al Viro6fd2fe42019-06-26 22:22:09 -04002320 if (clone_flags & CLONE_PIDFD) {
2321 fput(pidfile);
2322 put_unused_fd(pidfd);
2323 }
Pavel Emelyanov425fb2b2007-10-18 23:40:07 -07002324bad_fork_free_pid:
2325 if (pid != &init_struct_pid)
2326 free_pid(pid);
Jiri Slaby0740aa52016-05-20 17:00:25 -07002327bad_fork_cleanup_thread:
2328 exit_thread(p);
Jens Axboefd0928d2008-01-24 08:52:45 +01002329bad_fork_cleanup_io:
Louis Rillingb69f2292009-12-04 14:52:42 +01002330 if (p->io_context)
2331 exit_io_context(p);
Serge E. Hallynab516012006-10-02 02:18:06 -07002332bad_fork_cleanup_namespaces:
Linus Torvalds444f3782007-01-30 13:35:18 -08002333 exit_task_namespaces(p);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002334bad_fork_cleanup_mm:
Andrea Arcangelic3f3ce02019-05-14 15:40:46 -07002335 if (p->mm) {
2336 mm_clear_owner(p->mm, p);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002337 mmput(p->mm);
Andrea Arcangelic3f3ce02019-05-14 15:40:46 -07002338 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002339bad_fork_cleanup_signal:
Oleg Nesterov4ab6c082009-08-26 14:29:24 -07002340 if (!(clone_flags & CLONE_THREAD))
Mike Galbraith1c5354d2011-01-05 11:16:04 +01002341 free_signal_struct(p->signal);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002342bad_fork_cleanup_sighand:
Oleg Nesterova7e53282006-03-28 16:11:27 -08002343 __cleanup_sighand(p->sighand);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002344bad_fork_cleanup_fs:
2345 exit_fs(p); /* blocking */
2346bad_fork_cleanup_files:
2347 exit_files(p); /* blocking */
2348bad_fork_cleanup_semundo:
2349 exit_sem(p);
Tetsuo Handae4e55b42017-03-24 20:46:33 +09002350bad_fork_cleanup_security:
2351 security_task_free(p);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002352bad_fork_cleanup_audit:
2353 audit_free(p);
Peter Zijlstra6c72e3502014-10-02 16:17:02 -07002354bad_fork_cleanup_perf:
Ingo Molnarcdd6c482009-09-21 12:02:48 +02002355 perf_event_free_task(p);
Peter Zijlstra6c72e3502014-10-02 16:17:02 -07002356bad_fork_cleanup_policy:
Byungchul Parkb09be672017-08-07 16:12:52 +09002357 lockdep_free_task(p);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002358#ifdef CONFIG_NUMA
Lee Schermerhornf0be3d32008-04-28 02:13:08 -07002359 mpol_put(p->mempolicy);
Li Zefane8604cb2014-03-28 15:18:27 +08002360bad_fork_cleanup_threadgroup_lock:
Linus Torvalds1da177e2005-04-16 15:20:36 -07002361#endif
Shailabh Nagar35df17c2006-08-31 21:27:38 -07002362 delayacct_tsk_free(p);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002363bad_fork_cleanup_count:
David Howellsd84f4f92008-11-14 10:39:23 +11002364 atomic_dec(&p->cred->user->processes);
David Howellse0e81732009-09-02 09:13:40 +01002365 exit_creds(p);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002366bad_fork_free:
Andy Lutomirski405c0752016-10-31 08:11:43 -07002367 p->state = TASK_DEAD;
Andy Lutomirski68f24b082016-09-15 22:45:48 -07002368 put_task_stack(p);
Andrea Arcangelic3f3ce02019-05-14 15:40:46 -07002369 delayed_free_task(p);
Oleg Nesterovfe7d37d2006-01-08 01:04:02 -08002370fork_out:
Eric W. Biedermanc3ad2c32018-07-23 15:20:37 -05002371 spin_lock_irq(&current->sighand->siglock);
2372 hlist_del_init(&delayed.node);
2373 spin_unlock_irq(&current->sighand->siglock);
Oleg Nesterovfe7d37d2006-01-08 01:04:02 -08002374 return ERR_PTR(retval);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002375}
2376
Eric W. Biederman2c470472017-09-26 13:06:43 -05002377static inline void init_idle_pids(struct task_struct *idle)
Oleg Nesterovf106eee2010-05-26 14:44:11 -07002378{
2379 enum pid_type type;
2380
2381 for (type = PIDTYPE_PID; type < PIDTYPE_MAX; ++type) {
Eric W. Biederman2c470472017-09-26 13:06:43 -05002382 INIT_HLIST_NODE(&idle->pid_links[type]); /* not really needed */
2383 init_task_pid(idle, type, &init_struct_pid);
Oleg Nesterovf106eee2010-05-26 14:44:11 -07002384 }
2385}
2386
Paul Gortmaker0db06282013-06-19 14:53:51 -04002387struct task_struct *fork_idle(int cpu)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002388{
Ingo Molnar36c8b582006-07-03 00:25:41 -07002389 struct task_struct *task;
Christian Brauner7f192e32019-05-25 11:36:41 +02002390 struct kernel_clone_args args = {
2391 .flags = CLONE_VM,
2392 };
2393
2394 task = copy_process(&init_struct_pid, 0, cpu_to_node(cpu), &args);
Oleg Nesterovf106eee2010-05-26 14:44:11 -07002395 if (!IS_ERR(task)) {
Eric W. Biederman2c470472017-09-26 13:06:43 -05002396 init_idle_pids(task);
Akinobu Mita753ca4f2006-11-25 11:09:34 -08002397 init_idle(task, cpu);
Oleg Nesterovf106eee2010-05-26 14:44:11 -07002398 }
Oleg Nesterov73b9ebf2006-03-28 16:11:07 -08002399
Linus Torvalds1da177e2005-04-16 15:20:36 -07002400 return task;
2401}
2402
Nadav Amit13585fa2019-04-25 17:11:25 -07002403struct mm_struct *copy_init_mm(void)
2404{
2405 return dup_mm(NULL, &init_mm);
2406}
2407
Linus Torvalds1da177e2005-04-16 15:20:36 -07002408/*
2409 * Ok, this is the main fork-routine.
2410 *
2411 * It copies the process, and if successful kick-starts
2412 * it and waits for it to finish using the VM if required.
Eugene Syromiatnikova0eb9ab2019-09-11 18:45:40 +01002413 *
2414 * args->exit_signal is expected to be checked for sanity by the caller.
Linus Torvalds1da177e2005-04-16 15:20:36 -07002415 */
Christian Braunercad69672020-08-19 12:46:45 +02002416pid_t kernel_clone(struct kernel_clone_args *args)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002417{
Christian Brauner7f192e32019-05-25 11:36:41 +02002418 u64 clone_flags = args->flags;
Marcos Paulo de Souza9f5325a2018-02-06 15:39:30 -08002419 struct completion vfork;
2420 struct pid *pid;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002421 struct task_struct *p;
2422 int trace = 0;
Christian Braunercad69672020-08-19 12:46:45 +02002423 pid_t nr;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002424
Andrew Mortonbdff7462008-02-04 22:27:22 -08002425 /*
Christian Brauner3af85882020-06-08 17:28:50 +02002426 * For legacy clone() calls, CLONE_PIDFD uses the parent_tid argument
2427 * to return the pidfd. Hence, CLONE_PIDFD and CLONE_PARENT_SETTID are
2428 * mutually exclusive. With clone3() CLONE_PIDFD has grown a separate
2429 * field in struct clone_args and it still doesn't make sense to have
2430 * them both point at the same memory location. Performing this check
2431 * here has the advantage that we don't need to have a separate helper
2432 * to check for legacy clone().
2433 */
2434 if ((args->flags & CLONE_PIDFD) &&
2435 (args->flags & CLONE_PARENT_SETTID) &&
2436 (args->pidfd == args->parent_tid))
2437 return -EINVAL;
2438
2439 /*
Tejun Heo4b9d33e2011-06-17 16:50:38 +02002440 * Determine whether and which event to report to ptracer. When
2441 * called from kernel_thread or CLONE_UNTRACED is explicitly
2442 * requested, no event is reported; otherwise, report if the event
2443 * for the type of forking is enabled.
Roland McGrath09a05392008-07-25 19:45:47 -07002444 */
Al Viroe80d6662012-10-22 23:10:08 -04002445 if (!(clone_flags & CLONE_UNTRACED)) {
Tejun Heo4b9d33e2011-06-17 16:50:38 +02002446 if (clone_flags & CLONE_VFORK)
2447 trace = PTRACE_EVENT_VFORK;
Christian Brauner7f192e32019-05-25 11:36:41 +02002448 else if (args->exit_signal != SIGCHLD)
Tejun Heo4b9d33e2011-06-17 16:50:38 +02002449 trace = PTRACE_EVENT_CLONE;
2450 else
2451 trace = PTRACE_EVENT_FORK;
2452
2453 if (likely(!ptrace_event_enabled(current, trace)))
2454 trace = 0;
2455 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002456
Christian Brauner7f192e32019-05-25 11:36:41 +02002457 p = copy_process(NULL, trace, NUMA_NO_NODE, args);
Emese Revfy38addce2016-06-20 20:41:19 +02002458 add_latent_entropy();
Marcos Paulo de Souza9f5325a2018-02-06 15:39:30 -08002459
2460 if (IS_ERR(p))
2461 return PTR_ERR(p);
2462
Linus Torvalds1da177e2005-04-16 15:20:36 -07002463 /*
2464 * Do this prior waking up the new thread - the thread pointer
2465 * might get invalid after that point, if the thread exits quickly.
2466 */
Marcos Paulo de Souza9f5325a2018-02-06 15:39:30 -08002467 trace_sched_process_fork(current, p);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002468
Marcos Paulo de Souza9f5325a2018-02-06 15:39:30 -08002469 pid = get_task_pid(p, PIDTYPE_PID);
2470 nr = pid_vnr(pid);
Mathieu Desnoyers0a16b602008-07-18 12:16:17 -04002471
Marcos Paulo de Souza9f5325a2018-02-06 15:39:30 -08002472 if (clone_flags & CLONE_PARENT_SETTID)
Christian Brauner7f192e32019-05-25 11:36:41 +02002473 put_user(nr, args->parent_tid);
Pavel Emelyanov30e49c22007-10-18 23:40:10 -07002474
Marcos Paulo de Souza9f5325a2018-02-06 15:39:30 -08002475 if (clone_flags & CLONE_VFORK) {
2476 p->vfork_done = &vfork;
2477 init_completion(&vfork);
2478 get_task_struct(p);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002479 }
Marcos Paulo de Souza9f5325a2018-02-06 15:39:30 -08002480
2481 wake_up_new_task(p);
2482
2483 /* forking complete and child started to run, tell ptracer */
2484 if (unlikely(trace))
2485 ptrace_event_pid(trace, pid);
2486
2487 if (clone_flags & CLONE_VFORK) {
2488 if (!wait_for_vfork_done(p, &vfork))
2489 ptrace_event_pid(PTRACE_EVENT_VFORK_DONE, pid);
2490 }
2491
2492 put_pid(pid);
Eric W. Biederman92476d72006-03-31 02:31:42 -08002493 return nr;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002494}
2495
Al Viro2aa3a7f2012-09-21 19:55:31 -04002496/*
2497 * Create a kernel thread.
2498 */
2499pid_t kernel_thread(int (*fn)(void *), void *arg, unsigned long flags)
2500{
Christian Brauner7f192e32019-05-25 11:36:41 +02002501 struct kernel_clone_args args = {
Christian Brauner3f2c7882020-05-07 12:32:14 +02002502 .flags = ((lower_32_bits(flags) | CLONE_VM |
2503 CLONE_UNTRACED) & ~CSIGNAL),
2504 .exit_signal = (lower_32_bits(flags) & CSIGNAL),
Christian Brauner7f192e32019-05-25 11:36:41 +02002505 .stack = (unsigned long)fn,
2506 .stack_size = (unsigned long)arg,
2507 };
2508
Christian Braunercad69672020-08-19 12:46:45 +02002509 return kernel_clone(&args);
Al Viro2aa3a7f2012-09-21 19:55:31 -04002510}
Al Viro2aa3a7f2012-09-21 19:55:31 -04002511
Al Virod2125042012-10-23 13:17:59 -04002512#ifdef __ARCH_WANT_SYS_FORK
2513SYSCALL_DEFINE0(fork)
2514{
2515#ifdef CONFIG_MMU
Christian Brauner7f192e32019-05-25 11:36:41 +02002516 struct kernel_clone_args args = {
2517 .exit_signal = SIGCHLD,
2518 };
2519
Christian Braunercad69672020-08-19 12:46:45 +02002520 return kernel_clone(&args);
Al Virod2125042012-10-23 13:17:59 -04002521#else
2522 /* can not support in nommu mode */
Daeseok Youn5d59e182014-01-23 15:55:47 -08002523 return -EINVAL;
Al Virod2125042012-10-23 13:17:59 -04002524#endif
2525}
2526#endif
2527
2528#ifdef __ARCH_WANT_SYS_VFORK
2529SYSCALL_DEFINE0(vfork)
2530{
Christian Brauner7f192e32019-05-25 11:36:41 +02002531 struct kernel_clone_args args = {
2532 .flags = CLONE_VFORK | CLONE_VM,
2533 .exit_signal = SIGCHLD,
2534 };
2535
Christian Braunercad69672020-08-19 12:46:45 +02002536 return kernel_clone(&args);
Al Virod2125042012-10-23 13:17:59 -04002537}
2538#endif
2539
2540#ifdef __ARCH_WANT_SYS_CLONE
2541#ifdef CONFIG_CLONE_BACKWARDS
2542SYSCALL_DEFINE5(clone, unsigned long, clone_flags, unsigned long, newsp,
2543 int __user *, parent_tidptr,
Josh Triplett3033f14a2015-06-25 15:01:19 -07002544 unsigned long, tls,
Al Virod2125042012-10-23 13:17:59 -04002545 int __user *, child_tidptr)
2546#elif defined(CONFIG_CLONE_BACKWARDS2)
2547SYSCALL_DEFINE5(clone, unsigned long, newsp, unsigned long, clone_flags,
2548 int __user *, parent_tidptr,
2549 int __user *, child_tidptr,
Josh Triplett3033f14a2015-06-25 15:01:19 -07002550 unsigned long, tls)
Michal Simekdfa97712013-08-13 16:00:53 -07002551#elif defined(CONFIG_CLONE_BACKWARDS3)
2552SYSCALL_DEFINE6(clone, unsigned long, clone_flags, unsigned long, newsp,
2553 int, stack_size,
2554 int __user *, parent_tidptr,
2555 int __user *, child_tidptr,
Josh Triplett3033f14a2015-06-25 15:01:19 -07002556 unsigned long, tls)
Al Virod2125042012-10-23 13:17:59 -04002557#else
2558SYSCALL_DEFINE5(clone, unsigned long, clone_flags, unsigned long, newsp,
2559 int __user *, parent_tidptr,
2560 int __user *, child_tidptr,
Josh Triplett3033f14a2015-06-25 15:01:19 -07002561 unsigned long, tls)
Al Virod2125042012-10-23 13:17:59 -04002562#endif
2563{
Christian Brauner7f192e32019-05-25 11:36:41 +02002564 struct kernel_clone_args args = {
Christian Brauner3f2c7882020-05-07 12:32:14 +02002565 .flags = (lower_32_bits(clone_flags) & ~CSIGNAL),
Christian Brauner7f192e32019-05-25 11:36:41 +02002566 .pidfd = parent_tidptr,
2567 .child_tid = child_tidptr,
2568 .parent_tid = parent_tidptr,
Christian Brauner3f2c7882020-05-07 12:32:14 +02002569 .exit_signal = (lower_32_bits(clone_flags) & CSIGNAL),
Christian Brauner7f192e32019-05-25 11:36:41 +02002570 .stack = newsp,
2571 .tls = tls,
2572 };
2573
Christian Braunercad69672020-08-19 12:46:45 +02002574 return kernel_clone(&args);
Christian Brauner7f192e32019-05-25 11:36:41 +02002575}
Christian Braunerd68dbb02019-06-21 01:26:35 +02002576#endif
Christian Brauner7f192e32019-05-25 11:36:41 +02002577
Christian Braunerd68dbb02019-06-21 01:26:35 +02002578#ifdef __ARCH_WANT_SYS_CLONE3
Amanieu d'Antrasdd499f72020-01-02 18:24:13 +01002579
Christian Brauner7f192e32019-05-25 11:36:41 +02002580noinline static int copy_clone_args_from_user(struct kernel_clone_args *kargs,
2581 struct clone_args __user *uargs,
Aleksa Saraif14c2342019-10-01 11:10:53 +10002582 size_t usize)
Christian Brauner7f192e32019-05-25 11:36:41 +02002583{
Aleksa Saraif14c2342019-10-01 11:10:53 +10002584 int err;
Christian Brauner7f192e32019-05-25 11:36:41 +02002585 struct clone_args args;
Adrian Reber49cb2fc2019-11-15 13:36:20 +01002586 pid_t *kset_tid = kargs->set_tid;
Christian Brauner7f192e32019-05-25 11:36:41 +02002587
Eugene Syromiatnikova966dcf2020-04-12 22:26:58 +02002588 BUILD_BUG_ON(offsetofend(struct clone_args, tls) !=
2589 CLONE_ARGS_SIZE_VER0);
2590 BUILD_BUG_ON(offsetofend(struct clone_args, set_tid_size) !=
2591 CLONE_ARGS_SIZE_VER1);
2592 BUILD_BUG_ON(offsetofend(struct clone_args, cgroup) !=
2593 CLONE_ARGS_SIZE_VER2);
2594 BUILD_BUG_ON(sizeof(struct clone_args) != CLONE_ARGS_SIZE_VER2);
2595
Aleksa Saraif14c2342019-10-01 11:10:53 +10002596 if (unlikely(usize > PAGE_SIZE))
Christian Brauner7f192e32019-05-25 11:36:41 +02002597 return -E2BIG;
Aleksa Saraif14c2342019-10-01 11:10:53 +10002598 if (unlikely(usize < CLONE_ARGS_SIZE_VER0))
Christian Brauner7f192e32019-05-25 11:36:41 +02002599 return -EINVAL;
2600
Aleksa Saraif14c2342019-10-01 11:10:53 +10002601 err = copy_struct_from_user(&args, sizeof(args), uargs, usize);
2602 if (err)
2603 return err;
Christian Brauner7f192e32019-05-25 11:36:41 +02002604
Adrian Reber49cb2fc2019-11-15 13:36:20 +01002605 if (unlikely(args.set_tid_size > MAX_PID_NS_LEVEL))
2606 return -EINVAL;
2607
2608 if (unlikely(!args.set_tid && args.set_tid_size > 0))
2609 return -EINVAL;
2610
2611 if (unlikely(args.set_tid && args.set_tid_size == 0))
2612 return -EINVAL;
2613
Eugene Syromiatnikova0eb9ab2019-09-11 18:45:40 +01002614 /*
2615 * Verify that higher 32bits of exit_signal are unset and that
2616 * it is a valid signal
2617 */
2618 if (unlikely((args.exit_signal & ~((u64)CSIGNAL)) ||
2619 !valid_signal(args.exit_signal)))
2620 return -EINVAL;
2621
Eugene Syromiatnikov62173872020-04-12 22:31:23 +02002622 if ((args.flags & CLONE_INTO_CGROUP) &&
2623 (args.cgroup > INT_MAX || usize < CLONE_ARGS_SIZE_VER2))
Christian Brauneref2c41c2020-02-05 14:26:22 +01002624 return -EINVAL;
2625
Christian Brauner7f192e32019-05-25 11:36:41 +02002626 *kargs = (struct kernel_clone_args){
2627 .flags = args.flags,
2628 .pidfd = u64_to_user_ptr(args.pidfd),
2629 .child_tid = u64_to_user_ptr(args.child_tid),
2630 .parent_tid = u64_to_user_ptr(args.parent_tid),
2631 .exit_signal = args.exit_signal,
2632 .stack = args.stack,
2633 .stack_size = args.stack_size,
2634 .tls = args.tls,
Adrian Reber49cb2fc2019-11-15 13:36:20 +01002635 .set_tid_size = args.set_tid_size,
Christian Brauneref2c41c2020-02-05 14:26:22 +01002636 .cgroup = args.cgroup,
Christian Brauner7f192e32019-05-25 11:36:41 +02002637 };
2638
Adrian Reber49cb2fc2019-11-15 13:36:20 +01002639 if (args.set_tid &&
2640 copy_from_user(kset_tid, u64_to_user_ptr(args.set_tid),
2641 (kargs->set_tid_size * sizeof(pid_t))))
2642 return -EFAULT;
2643
2644 kargs->set_tid = kset_tid;
2645
Christian Brauner7f192e32019-05-25 11:36:41 +02002646 return 0;
2647}
2648
Christian Braunerfa729c42019-10-31 12:36:08 +01002649/**
2650 * clone3_stack_valid - check and prepare stack
2651 * @kargs: kernel clone args
2652 *
2653 * Verify that the stack arguments userspace gave us are sane.
2654 * In addition, set the stack direction for userspace since it's easy for us to
2655 * determine.
2656 */
2657static inline bool clone3_stack_valid(struct kernel_clone_args *kargs)
2658{
2659 if (kargs->stack == 0) {
2660 if (kargs->stack_size > 0)
2661 return false;
2662 } else {
2663 if (kargs->stack_size == 0)
2664 return false;
2665
2666 if (!access_ok((void __user *)kargs->stack, kargs->stack_size))
2667 return false;
2668
2669#if !defined(CONFIG_STACK_GROWSUP) && !defined(CONFIG_IA64)
2670 kargs->stack += kargs->stack_size;
2671#endif
2672 }
2673
2674 return true;
2675}
2676
2677static bool clone3_args_valid(struct kernel_clone_args *kargs)
Christian Brauner7f192e32019-05-25 11:36:41 +02002678{
Christian Braunerb612e5d2019-10-14 12:45:37 +02002679 /* Verify that no unknown flags are passed along. */
Christian Brauneref2c41c2020-02-05 14:26:22 +01002680 if (kargs->flags &
2681 ~(CLONE_LEGACY_FLAGS | CLONE_CLEAR_SIGHAND | CLONE_INTO_CGROUP))
Christian Brauner7f192e32019-05-25 11:36:41 +02002682 return false;
2683
2684 /*
2685 * - make the CLONE_DETACHED bit reuseable for clone3
2686 * - make the CSIGNAL bits reuseable for clone3
2687 */
2688 if (kargs->flags & (CLONE_DETACHED | CSIGNAL))
2689 return false;
2690
Christian Braunerb612e5d2019-10-14 12:45:37 +02002691 if ((kargs->flags & (CLONE_SIGHAND | CLONE_CLEAR_SIGHAND)) ==
2692 (CLONE_SIGHAND | CLONE_CLEAR_SIGHAND))
2693 return false;
2694
Christian Brauner7f192e32019-05-25 11:36:41 +02002695 if ((kargs->flags & (CLONE_THREAD | CLONE_PARENT)) &&
2696 kargs->exit_signal)
2697 return false;
2698
Christian Braunerfa729c42019-10-31 12:36:08 +01002699 if (!clone3_stack_valid(kargs))
2700 return false;
2701
Christian Brauner7f192e32019-05-25 11:36:41 +02002702 return true;
2703}
2704
Christian Brauner501bd012019-09-27 17:28:42 +02002705/**
2706 * clone3 - create a new process with specific properties
2707 * @uargs: argument structure
2708 * @size: size of @uargs
2709 *
2710 * clone3() is the extensible successor to clone()/clone2().
2711 * It takes a struct as argument that is versioned by its size.
2712 *
2713 * Return: On success, a positive PID for the child process.
2714 * On error, a negative errno number.
2715 */
Christian Brauner7f192e32019-05-25 11:36:41 +02002716SYSCALL_DEFINE2(clone3, struct clone_args __user *, uargs, size_t, size)
2717{
2718 int err;
2719
2720 struct kernel_clone_args kargs;
Adrian Reber49cb2fc2019-11-15 13:36:20 +01002721 pid_t set_tid[MAX_PID_NS_LEVEL];
2722
2723 kargs.set_tid = set_tid;
Christian Brauner7f192e32019-05-25 11:36:41 +02002724
2725 err = copy_clone_args_from_user(&kargs, uargs, size);
2726 if (err)
2727 return err;
2728
2729 if (!clone3_args_valid(&kargs))
2730 return -EINVAL;
2731
Christian Braunercad69672020-08-19 12:46:45 +02002732 return kernel_clone(&kargs);
Al Virod2125042012-10-23 13:17:59 -04002733}
2734#endif
2735
Oleg Nesterov0f1b92c2017-01-30 18:06:11 +03002736void walk_process_tree(struct task_struct *top, proc_visitor visitor, void *data)
2737{
2738 struct task_struct *leader, *parent, *child;
2739 int res;
2740
2741 read_lock(&tasklist_lock);
2742 leader = top = top->group_leader;
2743down:
2744 for_each_thread(leader, parent) {
2745 list_for_each_entry(child, &parent->children, sibling) {
2746 res = visitor(child, data);
2747 if (res) {
2748 if (res < 0)
2749 goto out;
2750 leader = child;
2751 goto down;
2752 }
2753up:
2754 ;
2755 }
2756 }
2757
2758 if (leader != top) {
2759 child = leader;
2760 parent = child->real_parent;
2761 leader = parent->group_leader;
2762 goto up;
2763 }
2764out:
2765 read_unlock(&tasklist_lock);
2766}
2767
Ravikiran G Thirumalai5fd63b32006-01-11 22:46:15 +01002768#ifndef ARCH_MIN_MMSTRUCT_ALIGN
2769#define ARCH_MIN_MMSTRUCT_ALIGN 0
2770#endif
2771
Alexey Dobriyan51cc5062008-07-25 19:45:34 -07002772static void sighand_ctor(void *data)
Oleg Nesterovaa1757f2006-03-28 16:11:12 -08002773{
2774 struct sighand_struct *sighand = data;
2775
Christoph Lametera35afb82007-05-16 22:10:57 -07002776 spin_lock_init(&sighand->siglock);
Davide Libenzib8fceee2007-09-20 12:40:16 -07002777 init_waitqueue_head(&sighand->signalfd_wqh);
Oleg Nesterovaa1757f2006-03-28 16:11:12 -08002778}
2779
Linus Torvalds1da177e2005-04-16 15:20:36 -07002780void __init proc_caches_init(void)
2781{
Rik van Rielc1a2f7f2018-07-16 15:03:31 -04002782 unsigned int mm_size;
2783
Linus Torvalds1da177e2005-04-16 15:20:36 -07002784 sighand_cachep = kmem_cache_create("sighand_cache",
2785 sizeof(struct sighand_struct), 0,
Paul E. McKenney5f0d5a32017-01-18 02:53:44 -08002786 SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_TYPESAFE_BY_RCU|
Levin, Alexander (Sasha Levin)75f296d2017-11-15 17:35:54 -08002787 SLAB_ACCOUNT, sighand_ctor);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002788 signal_cachep = kmem_cache_create("signal_cache",
2789 sizeof(struct signal_struct), 0,
Levin, Alexander (Sasha Levin)75f296d2017-11-15 17:35:54 -08002790 SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_ACCOUNT,
Vladimir Davydov5d097052016-01-14 15:18:21 -08002791 NULL);
Paul Mundt20c2df82007-07-20 10:11:58 +09002792 files_cachep = kmem_cache_create("files_cache",
Linus Torvalds1da177e2005-04-16 15:20:36 -07002793 sizeof(struct files_struct), 0,
Levin, Alexander (Sasha Levin)75f296d2017-11-15 17:35:54 -08002794 SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_ACCOUNT,
Vladimir Davydov5d097052016-01-14 15:18:21 -08002795 NULL);
Paul Mundt20c2df82007-07-20 10:11:58 +09002796 fs_cachep = kmem_cache_create("fs_cache",
Linus Torvalds1da177e2005-04-16 15:20:36 -07002797 sizeof(struct fs_struct), 0,
Levin, Alexander (Sasha Levin)75f296d2017-11-15 17:35:54 -08002798 SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_ACCOUNT,
Vladimir Davydov5d097052016-01-14 15:18:21 -08002799 NULL);
Rik van Rielc1a2f7f2018-07-16 15:03:31 -04002800
Linus Torvalds6345d242011-05-29 11:32:28 -07002801 /*
Rik van Rielc1a2f7f2018-07-16 15:03:31 -04002802 * The mm_cpumask is located at the end of mm_struct, and is
2803 * dynamically sized based on the maximum CPU number this system
2804 * can have, taking hotplug into account (nr_cpu_ids).
Linus Torvalds6345d242011-05-29 11:32:28 -07002805 */
Rik van Rielc1a2f7f2018-07-16 15:03:31 -04002806 mm_size = sizeof(struct mm_struct) + cpumask_size();
2807
David Windsor07dcd7f2017-08-15 16:45:00 -07002808 mm_cachep = kmem_cache_create_usercopy("mm_struct",
Rik van Rielc1a2f7f2018-07-16 15:03:31 -04002809 mm_size, ARCH_MIN_MMSTRUCT_ALIGN,
Levin, Alexander (Sasha Levin)75f296d2017-11-15 17:35:54 -08002810 SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_ACCOUNT,
David Windsor07dcd7f2017-08-15 16:45:00 -07002811 offsetof(struct mm_struct, saved_auxv),
2812 sizeof_field(struct mm_struct, saved_auxv),
Vladimir Davydov5d097052016-01-14 15:18:21 -08002813 NULL);
2814 vm_area_cachep = KMEM_CACHE(vm_area_struct, SLAB_PANIC|SLAB_ACCOUNT);
David Howells8feae132009-01-08 12:04:47 +00002815 mmap_init();
Al Viro66577192011-06-28 15:41:10 -04002816 nsproxy_cache_init();
Linus Torvalds1da177e2005-04-16 15:20:36 -07002817}
JANAK DESAIcf2e3402006-02-07 12:58:58 -08002818
JANAK DESAIcf2e3402006-02-07 12:58:58 -08002819/*
Oleg Nesterov9bfb23f2011-03-22 16:34:09 -07002820 * Check constraints on flags passed to the unshare system call.
JANAK DESAIcf2e3402006-02-07 12:58:58 -08002821 */
Oleg Nesterov9bfb23f2011-03-22 16:34:09 -07002822static int check_unshare_flags(unsigned long unshare_flags)
JANAK DESAIcf2e3402006-02-07 12:58:58 -08002823{
Oleg Nesterov9bfb23f2011-03-22 16:34:09 -07002824 if (unshare_flags & ~(CLONE_THREAD|CLONE_FS|CLONE_NEWNS|CLONE_SIGHAND|
2825 CLONE_VM|CLONE_FILES|CLONE_SYSVSEM|
Eric W. Biederman50804fe2010-03-02 15:41:50 -08002826 CLONE_NEWUTS|CLONE_NEWIPC|CLONE_NEWNET|
Andrei Vagin769071a2019-11-12 01:26:52 +00002827 CLONE_NEWUSER|CLONE_NEWPID|CLONE_NEWCGROUP|
2828 CLONE_NEWTIME))
JANAK DESAIcf2e3402006-02-07 12:58:58 -08002829 return -EINVAL;
Oleg Nesterov9bfb23f2011-03-22 16:34:09 -07002830 /*
Eric W. Biederman12c641a2015-08-10 17:35:07 -05002831 * Not implemented, but pretend it works if there is nothing
2832 * to unshare. Note that unsharing the address space or the
2833 * signal handlers also need to unshare the signal queues (aka
2834 * CLONE_THREAD).
Oleg Nesterov9bfb23f2011-03-22 16:34:09 -07002835 */
2836 if (unshare_flags & (CLONE_THREAD | CLONE_SIGHAND | CLONE_VM)) {
Eric W. Biederman12c641a2015-08-10 17:35:07 -05002837 if (!thread_group_empty(current))
2838 return -EINVAL;
2839 }
2840 if (unshare_flags & (CLONE_SIGHAND | CLONE_VM)) {
Elena Reshetovad036bda2019-01-18 14:27:26 +02002841 if (refcount_read(&current->sighand->count) > 1)
Eric W. Biederman12c641a2015-08-10 17:35:07 -05002842 return -EINVAL;
2843 }
2844 if (unshare_flags & CLONE_VM) {
2845 if (!current_is_single_threaded())
Oleg Nesterov9bfb23f2011-03-22 16:34:09 -07002846 return -EINVAL;
2847 }
JANAK DESAIcf2e3402006-02-07 12:58:58 -08002848
2849 return 0;
2850}
2851
2852/*
JANAK DESAI99d14192006-02-07 12:58:59 -08002853 * Unshare the filesystem structure if it is being shared
JANAK DESAIcf2e3402006-02-07 12:58:58 -08002854 */
2855static int unshare_fs(unsigned long unshare_flags, struct fs_struct **new_fsp)
2856{
2857 struct fs_struct *fs = current->fs;
2858
Al Viro498052b2009-03-30 07:20:30 -04002859 if (!(unshare_flags & CLONE_FS) || !fs)
2860 return 0;
2861
2862 /* don't need lock here; in the worst case we'll do useless copy */
2863 if (fs->users == 1)
2864 return 0;
2865
2866 *new_fsp = copy_fs_struct(fs);
2867 if (!*new_fsp)
2868 return -ENOMEM;
JANAK DESAIcf2e3402006-02-07 12:58:58 -08002869
2870 return 0;
2871}
2872
2873/*
JANAK DESAIa016f332006-02-07 12:59:02 -08002874 * Unshare file descriptor table if it is being shared
JANAK DESAIcf2e3402006-02-07 12:58:58 -08002875 */
Christian Brauner60997c32020-06-03 21:48:55 +02002876int unshare_fd(unsigned long unshare_flags, unsigned int max_fds,
2877 struct files_struct **new_fdp)
JANAK DESAIcf2e3402006-02-07 12:58:58 -08002878{
2879 struct files_struct *fd = current->files;
JANAK DESAIa016f332006-02-07 12:59:02 -08002880 int error = 0;
JANAK DESAIcf2e3402006-02-07 12:58:58 -08002881
2882 if ((unshare_flags & CLONE_FILES) &&
JANAK DESAIa016f332006-02-07 12:59:02 -08002883 (fd && atomic_read(&fd->count) > 1)) {
Christian Brauner60997c32020-06-03 21:48:55 +02002884 *new_fdp = dup_fd(fd, max_fds, &error);
JANAK DESAIa016f332006-02-07 12:59:02 -08002885 if (!*new_fdp)
2886 return error;
2887 }
JANAK DESAIcf2e3402006-02-07 12:58:58 -08002888
2889 return 0;
2890}
2891
2892/*
JANAK DESAIcf2e3402006-02-07 12:58:58 -08002893 * unshare allows a process to 'unshare' part of the process
2894 * context which was originally shared using clone. copy_*
Christian Braunercad69672020-08-19 12:46:45 +02002895 * functions used by kernel_clone() cannot be used here directly
JANAK DESAIcf2e3402006-02-07 12:58:58 -08002896 * because they modify an inactive task_struct that is being
2897 * constructed. Here we are modifying the current, active,
2898 * task_struct.
2899 */
Dominik Brodowski9b321052018-03-11 11:34:42 +01002900int ksys_unshare(unsigned long unshare_flags)
JANAK DESAIcf2e3402006-02-07 12:58:58 -08002901{
JANAK DESAIcf2e3402006-02-07 12:58:58 -08002902 struct fs_struct *fs, *new_fs = NULL;
JANAK DESAIcf2e3402006-02-07 12:58:58 -08002903 struct files_struct *fd, *new_fd = NULL;
Eric W. Biedermanb2e0d9872012-07-26 05:15:35 -07002904 struct cred *new_cred = NULL;
Pavel Emelyanovcf7b7082007-10-18 23:39:54 -07002905 struct nsproxy *new_nsproxy = NULL;
Manfred Spraul9edff4a2008-04-29 01:00:57 -07002906 int do_sysvsem = 0;
Oleg Nesterov9bfb23f2011-03-22 16:34:09 -07002907 int err;
JANAK DESAIcf2e3402006-02-07 12:58:58 -08002908
Eric W. Biederman50804fe2010-03-02 15:41:50 -08002909 /*
Eric W. Biedermanfaf00da2015-08-10 18:25:44 -05002910 * If unsharing a user namespace must also unshare the thread group
2911 * and unshare the filesystem root and working directories.
Eric W. Biedermanb2e0d9872012-07-26 05:15:35 -07002912 */
2913 if (unshare_flags & CLONE_NEWUSER)
Eric W. Biedermane66eded2013-03-13 11:51:49 -07002914 unshare_flags |= CLONE_THREAD | CLONE_FS;
Eric W. Biedermanb2e0d9872012-07-26 05:15:35 -07002915 /*
Eric W. Biederman50804fe2010-03-02 15:41:50 -08002916 * If unsharing vm, must also unshare signal handlers.
2917 */
2918 if (unshare_flags & CLONE_VM)
2919 unshare_flags |= CLONE_SIGHAND;
Manfred Spraul6013f672008-04-29 01:00:59 -07002920 /*
Eric W. Biederman12c641a2015-08-10 17:35:07 -05002921 * If unsharing a signal handlers, must also unshare the signal queues.
2922 */
2923 if (unshare_flags & CLONE_SIGHAND)
2924 unshare_flags |= CLONE_THREAD;
2925 /*
Oleg Nesterov9bfb23f2011-03-22 16:34:09 -07002926 * If unsharing namespace, must also unshare filesystem information.
2927 */
2928 if (unshare_flags & CLONE_NEWNS)
2929 unshare_flags |= CLONE_FS;
Eric W. Biederman50804fe2010-03-02 15:41:50 -08002930
2931 err = check_unshare_flags(unshare_flags);
2932 if (err)
2933 goto bad_unshare_out;
Oleg Nesterov9bfb23f2011-03-22 16:34:09 -07002934 /*
Manfred Spraul6013f672008-04-29 01:00:59 -07002935 * CLONE_NEWIPC must also detach from the undolist: after switching
2936 * to a new ipc namespace, the semaphore arrays from the old
2937 * namespace are unreachable.
2938 */
2939 if (unshare_flags & (CLONE_NEWIPC|CLONE_SYSVSEM))
Manfred Spraul9edff4a2008-04-29 01:00:57 -07002940 do_sysvsem = 1;
Daniel Rebelo de Oliveirafb0a6852011-07-26 16:08:39 -07002941 err = unshare_fs(unshare_flags, &new_fs);
2942 if (err)
Oleg Nesterov9bfb23f2011-03-22 16:34:09 -07002943 goto bad_unshare_out;
Christian Brauner60997c32020-06-03 21:48:55 +02002944 err = unshare_fd(unshare_flags, NR_OPEN_MAX, &new_fd);
Daniel Rebelo de Oliveirafb0a6852011-07-26 16:08:39 -07002945 if (err)
Oleg Nesterov9bfb23f2011-03-22 16:34:09 -07002946 goto bad_unshare_cleanup_fs;
Eric W. Biedermanb2e0d9872012-07-26 05:15:35 -07002947 err = unshare_userns(unshare_flags, &new_cred);
Daniel Rebelo de Oliveirafb0a6852011-07-26 16:08:39 -07002948 if (err)
Manfred Spraul9edff4a2008-04-29 01:00:57 -07002949 goto bad_unshare_cleanup_fd;
Eric W. Biedermanb2e0d9872012-07-26 05:15:35 -07002950 err = unshare_nsproxy_namespaces(unshare_flags, &new_nsproxy,
2951 new_cred, new_fs);
2952 if (err)
2953 goto bad_unshare_cleanup_cred;
JANAK DESAIcf2e3402006-02-07 12:58:58 -08002954
Eric W. Biedermanb2e0d9872012-07-26 05:15:35 -07002955 if (new_fs || new_fd || do_sysvsem || new_cred || new_nsproxy) {
Manfred Spraul9edff4a2008-04-29 01:00:57 -07002956 if (do_sysvsem) {
2957 /*
2958 * CLONE_SYSVSEM is equivalent to sys_exit().
2959 */
2960 exit_sem(current);
2961 }
Jack Millerab602f72014-08-08 14:23:19 -07002962 if (unshare_flags & CLONE_NEWIPC) {
2963 /* Orphan segments in old ns (see sem above). */
2964 exit_shm(current);
2965 shm_init_task(current);
2966 }
Serge E. Hallynab516012006-10-02 02:18:06 -07002967
Alan Cox6f977e62013-02-27 17:03:23 -08002968 if (new_nsproxy)
Pavel Emelyanovcf7b7082007-10-18 23:39:54 -07002969 switch_task_namespaces(current, new_nsproxy);
JANAK DESAIcf2e3402006-02-07 12:58:58 -08002970
Pavel Emelyanovcf7b7082007-10-18 23:39:54 -07002971 task_lock(current);
2972
JANAK DESAIcf2e3402006-02-07 12:58:58 -08002973 if (new_fs) {
2974 fs = current->fs;
Nick Piggin2a4419b2010-08-18 04:37:33 +10002975 spin_lock(&fs->lock);
JANAK DESAIcf2e3402006-02-07 12:58:58 -08002976 current->fs = new_fs;
Al Viro498052b2009-03-30 07:20:30 -04002977 if (--fs->users)
2978 new_fs = NULL;
2979 else
2980 new_fs = fs;
Nick Piggin2a4419b2010-08-18 04:37:33 +10002981 spin_unlock(&fs->lock);
JANAK DESAIcf2e3402006-02-07 12:58:58 -08002982 }
2983
JANAK DESAIcf2e3402006-02-07 12:58:58 -08002984 if (new_fd) {
2985 fd = current->files;
2986 current->files = new_fd;
2987 new_fd = fd;
2988 }
2989
2990 task_unlock(current);
Eric W. Biedermanb2e0d9872012-07-26 05:15:35 -07002991
2992 if (new_cred) {
2993 /* Install the new user namespace */
2994 commit_creds(new_cred);
2995 new_cred = NULL;
2996 }
JANAK DESAIcf2e3402006-02-07 12:58:58 -08002997 }
2998
Hari Bathinie4222672017-03-08 02:11:36 +05302999 perf_event_namespaces(current);
3000
Eric W. Biedermanb2e0d9872012-07-26 05:15:35 -07003001bad_unshare_cleanup_cred:
3002 if (new_cred)
3003 put_cred(new_cred);
JANAK DESAIcf2e3402006-02-07 12:58:58 -08003004bad_unshare_cleanup_fd:
3005 if (new_fd)
3006 put_files_struct(new_fd);
3007
JANAK DESAIcf2e3402006-02-07 12:58:58 -08003008bad_unshare_cleanup_fs:
3009 if (new_fs)
Al Viro498052b2009-03-30 07:20:30 -04003010 free_fs_struct(new_fs);
JANAK DESAIcf2e3402006-02-07 12:58:58 -08003011
JANAK DESAIcf2e3402006-02-07 12:58:58 -08003012bad_unshare_out:
3013 return err;
3014}
Al Viro3b125382008-04-22 05:31:30 -04003015
Dominik Brodowski9b321052018-03-11 11:34:42 +01003016SYSCALL_DEFINE1(unshare, unsigned long, unshare_flags)
3017{
3018 return ksys_unshare(unshare_flags);
3019}
3020
Al Viro3b125382008-04-22 05:31:30 -04003021/*
3022 * Helper to unshare the files of the current task.
3023 * We don't want to expose copy_files internals to
3024 * the exec layer of the kernel.
3025 */
3026
3027int unshare_files(struct files_struct **displaced)
3028{
3029 struct task_struct *task = current;
Al Viro50704512008-04-26 05:25:00 +01003030 struct files_struct *copy = NULL;
Al Viro3b125382008-04-22 05:31:30 -04003031 int error;
3032
Christian Brauner60997c32020-06-03 21:48:55 +02003033 error = unshare_fd(CLONE_FILES, NR_OPEN_MAX, &copy);
Al Viro3b125382008-04-22 05:31:30 -04003034 if (error || !copy) {
3035 *displaced = NULL;
3036 return error;
3037 }
3038 *displaced = task->files;
3039 task_lock(task);
3040 task->files = copy;
3041 task_unlock(task);
3042 return 0;
3043}
Heinrich Schuchardt16db3d32015-04-16 12:47:50 -07003044
3045int sysctl_max_threads(struct ctl_table *table, int write,
Tobias Klauserb0daa2c2020-09-04 16:35:49 -07003046 void *buffer, size_t *lenp, loff_t *ppos)
Heinrich Schuchardt16db3d32015-04-16 12:47:50 -07003047{
3048 struct ctl_table t;
3049 int ret;
3050 int threads = max_threads;
Michal Hockob0f53db2019-10-06 17:58:19 -07003051 int min = 1;
Heinrich Schuchardt16db3d32015-04-16 12:47:50 -07003052 int max = MAX_THREADS;
3053
3054 t = *table;
3055 t.data = &threads;
3056 t.extra1 = &min;
3057 t.extra2 = &max;
3058
3059 ret = proc_dointvec_minmax(&t, write, buffer, lenp, ppos);
3060 if (ret || !write)
3061 return ret;
3062
Michal Hockob0f53db2019-10-06 17:58:19 -07003063 max_threads = threads;
Heinrich Schuchardt16db3d32015-04-16 12:47:50 -07003064
3065 return 0;
3066}