blob: 88f609f1c0c3ad71b40ffc1286800a9bfeb8e7c3 [file] [log] [blame]
Alexei Starovoitov99c55f72014-09-26 00:16:57 -07001/* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com
2 *
3 * This program is free software; you can redistribute it and/or
4 * modify it under the terms of version 2 of the GNU General Public
5 * License as published by the Free Software Foundation.
6 *
7 * This program is distributed in the hope that it will be useful, but
8 * WITHOUT ANY WARRANTY; without even the implied warranty of
9 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
10 * General Public License for more details.
11 */
12#include <linux/bpf.h>
13#include <linux/syscalls.h>
14#include <linux/slab.h>
15#include <linux/anon_inodes.h>
Alexei Starovoitovdb20fd22014-09-26 00:16:59 -070016#include <linux/file.h>
Alexei Starovoitov09756af2014-09-26 00:17:00 -070017#include <linux/license.h>
18#include <linux/filter.h>
Alexei Starovoitov25415172015-03-25 12:49:20 -070019#include <linux/version.h>
Mickaël Salaün535e7b4b2016-11-13 19:44:03 +010020#include <linux/kernel.h>
Alexei Starovoitov99c55f72014-09-26 00:16:57 -070021
Alexei Starovoitovb121d1e2016-03-07 21:57:13 -080022DEFINE_PER_CPU(int, bpf_prog_active);
23
Alexei Starovoitov1be7f752015-10-07 22:23:21 -070024int sysctl_unprivileged_bpf_disabled __read_mostly;
25
Alexei Starovoitov99c55f72014-09-26 00:16:57 -070026static LIST_HEAD(bpf_map_types);
27
28static struct bpf_map *find_and_alloc_map(union bpf_attr *attr)
29{
30 struct bpf_map_type_list *tl;
31 struct bpf_map *map;
32
33 list_for_each_entry(tl, &bpf_map_types, list_node) {
34 if (tl->type == attr->map_type) {
35 map = tl->ops->map_alloc(attr);
36 if (IS_ERR(map))
37 return map;
38 map->ops = tl->ops;
39 map->map_type = attr->map_type;
40 return map;
41 }
42 }
43 return ERR_PTR(-EINVAL);
44}
45
46/* boot time registration of different map implementations */
47void bpf_register_map_type(struct bpf_map_type_list *tl)
48{
49 list_add(&tl->list_node, &bpf_map_types);
50}
51
Alexei Starovoitov6c905982016-03-07 21:57:15 -080052int bpf_map_precharge_memlock(u32 pages)
53{
54 struct user_struct *user = get_current_user();
55 unsigned long memlock_limit, cur;
56
57 memlock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
58 cur = atomic_long_read(&user->locked_vm);
59 free_uid(user);
60 if (cur + pages > memlock_limit)
61 return -EPERM;
62 return 0;
63}
64
Alexei Starovoitovaaac3ba2015-10-07 22:23:22 -070065static int bpf_map_charge_memlock(struct bpf_map *map)
66{
67 struct user_struct *user = get_current_user();
68 unsigned long memlock_limit;
69
70 memlock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
71
72 atomic_long_add(map->pages, &user->locked_vm);
73
74 if (atomic_long_read(&user->locked_vm) > memlock_limit) {
75 atomic_long_sub(map->pages, &user->locked_vm);
76 free_uid(user);
77 return -EPERM;
78 }
79 map->user = user;
80 return 0;
81}
82
83static void bpf_map_uncharge_memlock(struct bpf_map *map)
84{
85 struct user_struct *user = map->user;
86
87 atomic_long_sub(map->pages, &user->locked_vm);
88 free_uid(user);
89}
90
Alexei Starovoitov99c55f72014-09-26 00:16:57 -070091/* called from workqueue */
92static void bpf_map_free_deferred(struct work_struct *work)
93{
94 struct bpf_map *map = container_of(work, struct bpf_map, work);
95
Alexei Starovoitovaaac3ba2015-10-07 22:23:22 -070096 bpf_map_uncharge_memlock(map);
Alexei Starovoitov99c55f72014-09-26 00:16:57 -070097 /* implementation dependent freeing */
98 map->ops->map_free(map);
99}
100
Daniel Borkmannc9da1612015-11-24 21:28:15 +0100101static void bpf_map_put_uref(struct bpf_map *map)
102{
103 if (atomic_dec_and_test(&map->usercnt)) {
104 if (map->map_type == BPF_MAP_TYPE_PROG_ARRAY)
105 bpf_fd_array_map_clear(map);
106 }
107}
108
Alexei Starovoitov99c55f72014-09-26 00:16:57 -0700109/* decrement map refcnt and schedule it for freeing via workqueue
110 * (unrelying map implementation ops->map_free() might sleep)
111 */
112void bpf_map_put(struct bpf_map *map)
113{
114 if (atomic_dec_and_test(&map->refcnt)) {
115 INIT_WORK(&map->work, bpf_map_free_deferred);
116 schedule_work(&map->work);
117 }
118}
119
Daniel Borkmannc9da1612015-11-24 21:28:15 +0100120void bpf_map_put_with_uref(struct bpf_map *map)
121{
122 bpf_map_put_uref(map);
123 bpf_map_put(map);
124}
125
Alexei Starovoitov99c55f72014-09-26 00:16:57 -0700126static int bpf_map_release(struct inode *inode, struct file *filp)
127{
Daniel Borkmann61d1b6a2016-06-15 22:47:12 +0200128 struct bpf_map *map = filp->private_data;
129
130 if (map->ops->map_release)
131 map->ops->map_release(map, filp);
132
133 bpf_map_put_with_uref(map);
Alexei Starovoitov99c55f72014-09-26 00:16:57 -0700134 return 0;
135}
136
Daniel Borkmannf99bf202015-11-19 11:56:22 +0100137#ifdef CONFIG_PROC_FS
138static void bpf_map_show_fdinfo(struct seq_file *m, struct file *filp)
139{
140 const struct bpf_map *map = filp->private_data;
Daniel Borkmann21116b72016-11-26 01:28:07 +0100141 const struct bpf_array *array;
142 u32 owner_prog_type = 0;
143
144 if (map->map_type == BPF_MAP_TYPE_PROG_ARRAY) {
145 array = container_of(map, struct bpf_array, map);
146 owner_prog_type = array->owner_prog_type;
147 }
Daniel Borkmannf99bf202015-11-19 11:56:22 +0100148
149 seq_printf(m,
150 "map_type:\t%u\n"
151 "key_size:\t%u\n"
152 "value_size:\t%u\n"
Daniel Borkmann322cea22016-03-25 00:30:25 +0100153 "max_entries:\t%u\n"
Daniel Borkmann21116b72016-11-26 01:28:07 +0100154 "map_flags:\t%#x\n"
155 "memlock:\t%llu\n",
Daniel Borkmannf99bf202015-11-19 11:56:22 +0100156 map->map_type,
157 map->key_size,
158 map->value_size,
Daniel Borkmann322cea22016-03-25 00:30:25 +0100159 map->max_entries,
Daniel Borkmann21116b72016-11-26 01:28:07 +0100160 map->map_flags,
161 map->pages * 1ULL << PAGE_SHIFT);
162
163 if (owner_prog_type)
164 seq_printf(m, "owner_prog_type:\t%u\n",
165 owner_prog_type);
Daniel Borkmannf99bf202015-11-19 11:56:22 +0100166}
167#endif
168
Alexei Starovoitov99c55f72014-09-26 00:16:57 -0700169static const struct file_operations bpf_map_fops = {
Daniel Borkmannf99bf202015-11-19 11:56:22 +0100170#ifdef CONFIG_PROC_FS
171 .show_fdinfo = bpf_map_show_fdinfo,
172#endif
173 .release = bpf_map_release,
Alexei Starovoitov99c55f72014-09-26 00:16:57 -0700174};
175
Daniel Borkmannb2197752015-10-29 14:58:09 +0100176int bpf_map_new_fd(struct bpf_map *map)
Daniel Borkmannaa797812015-10-29 14:58:06 +0100177{
178 return anon_inode_getfd("bpf-map", &bpf_map_fops, map,
179 O_RDWR | O_CLOEXEC);
180}
181
Alexei Starovoitov99c55f72014-09-26 00:16:57 -0700182/* helper macro to check that unused fields 'union bpf_attr' are zero */
183#define CHECK_ATTR(CMD) \
184 memchr_inv((void *) &attr->CMD##_LAST_FIELD + \
185 sizeof(attr->CMD##_LAST_FIELD), 0, \
186 sizeof(*attr) - \
187 offsetof(union bpf_attr, CMD##_LAST_FIELD) - \
188 sizeof(attr->CMD##_LAST_FIELD)) != NULL
189
Alexei Starovoitov6c905982016-03-07 21:57:15 -0800190#define BPF_MAP_CREATE_LAST_FIELD map_flags
Alexei Starovoitov99c55f72014-09-26 00:16:57 -0700191/* called via syscall */
192static int map_create(union bpf_attr *attr)
193{
194 struct bpf_map *map;
195 int err;
196
197 err = CHECK_ATTR(BPF_MAP_CREATE);
198 if (err)
199 return -EINVAL;
200
201 /* find map type and init map: hashtable vs rbtree vs bloom vs ... */
202 map = find_and_alloc_map(attr);
203 if (IS_ERR(map))
204 return PTR_ERR(map);
205
206 atomic_set(&map->refcnt, 1);
Daniel Borkmannc9da1612015-11-24 21:28:15 +0100207 atomic_set(&map->usercnt, 1);
Alexei Starovoitov99c55f72014-09-26 00:16:57 -0700208
Alexei Starovoitovaaac3ba2015-10-07 22:23:22 -0700209 err = bpf_map_charge_memlock(map);
210 if (err)
Daniel Borkmann20b2b242016-11-04 00:56:31 +0100211 goto free_map_nouncharge;
Alexei Starovoitovaaac3ba2015-10-07 22:23:22 -0700212
Daniel Borkmannaa797812015-10-29 14:58:06 +0100213 err = bpf_map_new_fd(map);
Alexei Starovoitov99c55f72014-09-26 00:16:57 -0700214 if (err < 0)
215 /* failed to allocate fd */
216 goto free_map;
217
218 return err;
219
220free_map:
Daniel Borkmann20b2b242016-11-04 00:56:31 +0100221 bpf_map_uncharge_memlock(map);
222free_map_nouncharge:
Alexei Starovoitov99c55f72014-09-26 00:16:57 -0700223 map->ops->map_free(map);
224 return err;
225}
226
Alexei Starovoitovdb20fd22014-09-26 00:16:59 -0700227/* if error is returned, fd is released.
228 * On success caller should complete fd access with matching fdput()
229 */
Daniel Borkmannc2101292015-10-29 14:58:07 +0100230struct bpf_map *__bpf_map_get(struct fd f)
Alexei Starovoitovdb20fd22014-09-26 00:16:59 -0700231{
Alexei Starovoitovdb20fd22014-09-26 00:16:59 -0700232 if (!f.file)
233 return ERR_PTR(-EBADF);
Alexei Starovoitovdb20fd22014-09-26 00:16:59 -0700234 if (f.file->f_op != &bpf_map_fops) {
235 fdput(f);
236 return ERR_PTR(-EINVAL);
237 }
238
Daniel Borkmannc2101292015-10-29 14:58:07 +0100239 return f.file->private_data;
240}
241
Alexei Starovoitov92117d82016-04-27 18:56:20 -0700242/* prog's and map's refcnt limit */
243#define BPF_MAX_REFCNT 32768
244
245struct bpf_map *bpf_map_inc(struct bpf_map *map, bool uref)
Daniel Borkmannc9da1612015-11-24 21:28:15 +0100246{
Alexei Starovoitov92117d82016-04-27 18:56:20 -0700247 if (atomic_inc_return(&map->refcnt) > BPF_MAX_REFCNT) {
248 atomic_dec(&map->refcnt);
249 return ERR_PTR(-EBUSY);
250 }
Daniel Borkmannc9da1612015-11-24 21:28:15 +0100251 if (uref)
252 atomic_inc(&map->usercnt);
Alexei Starovoitov92117d82016-04-27 18:56:20 -0700253 return map;
Daniel Borkmannc9da1612015-11-24 21:28:15 +0100254}
255
256struct bpf_map *bpf_map_get_with_uref(u32 ufd)
Daniel Borkmannc2101292015-10-29 14:58:07 +0100257{
258 struct fd f = fdget(ufd);
259 struct bpf_map *map;
260
261 map = __bpf_map_get(f);
262 if (IS_ERR(map))
263 return map;
264
Alexei Starovoitov92117d82016-04-27 18:56:20 -0700265 map = bpf_map_inc(map, true);
Daniel Borkmannc2101292015-10-29 14:58:07 +0100266 fdput(f);
Alexei Starovoitovdb20fd22014-09-26 00:16:59 -0700267
268 return map;
269}
270
Alexei Starovoitovb8cdc052016-03-09 18:56:49 -0800271int __weak bpf_stackmap_copy(struct bpf_map *map, void *key, void *value)
272{
273 return -ENOTSUPP;
274}
275
Alexei Starovoitovdb20fd22014-09-26 00:16:59 -0700276/* last field in 'union bpf_attr' used by this command */
277#define BPF_MAP_LOOKUP_ELEM_LAST_FIELD value
278
279static int map_lookup_elem(union bpf_attr *attr)
280{
Mickaël Salaün535e7b4b2016-11-13 19:44:03 +0100281 void __user *ukey = u64_to_user_ptr(attr->key);
282 void __user *uvalue = u64_to_user_ptr(attr->value);
Alexei Starovoitovdb20fd22014-09-26 00:16:59 -0700283 int ufd = attr->map_fd;
Alexei Starovoitovdb20fd22014-09-26 00:16:59 -0700284 struct bpf_map *map;
Alexei Starovoitov8ebe6672015-01-22 17:11:08 -0800285 void *key, *value, *ptr;
Alexei Starovoitov15a07b32016-02-01 22:39:55 -0800286 u32 value_size;
Daniel Borkmann592867b2015-09-08 18:00:09 +0200287 struct fd f;
Alexei Starovoitovdb20fd22014-09-26 00:16:59 -0700288 int err;
289
290 if (CHECK_ATTR(BPF_MAP_LOOKUP_ELEM))
291 return -EINVAL;
292
Daniel Borkmann592867b2015-09-08 18:00:09 +0200293 f = fdget(ufd);
Daniel Borkmannc2101292015-10-29 14:58:07 +0100294 map = __bpf_map_get(f);
Alexei Starovoitovdb20fd22014-09-26 00:16:59 -0700295 if (IS_ERR(map))
296 return PTR_ERR(map);
297
298 err = -ENOMEM;
299 key = kmalloc(map->key_size, GFP_USER);
300 if (!key)
301 goto err_put;
302
303 err = -EFAULT;
304 if (copy_from_user(key, ukey, map->key_size) != 0)
305 goto free_key;
306
Alexei Starovoitov15a07b32016-02-01 22:39:55 -0800307 if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH ||
Martin KaFai Lau8f844932016-11-11 10:55:10 -0800308 map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH ||
Alexei Starovoitov15a07b32016-02-01 22:39:55 -0800309 map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY)
310 value_size = round_up(map->value_size, 8) * num_possible_cpus();
311 else
312 value_size = map->value_size;
313
Alexei Starovoitov8ebe6672015-01-22 17:11:08 -0800314 err = -ENOMEM;
Alexei Starovoitov15a07b32016-02-01 22:39:55 -0800315 value = kmalloc(value_size, GFP_USER | __GFP_NOWARN);
Alexei Starovoitovdb20fd22014-09-26 00:16:59 -0700316 if (!value)
Alexei Starovoitov8ebe6672015-01-22 17:11:08 -0800317 goto free_key;
318
Martin KaFai Lau8f844932016-11-11 10:55:10 -0800319 if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH ||
320 map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH) {
Alexei Starovoitov15a07b32016-02-01 22:39:55 -0800321 err = bpf_percpu_hash_copy(map, key, value);
322 } else if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) {
323 err = bpf_percpu_array_copy(map, key, value);
Alexei Starovoitov557c0c62016-03-07 21:57:17 -0800324 } else if (map->map_type == BPF_MAP_TYPE_STACK_TRACE) {
325 err = bpf_stackmap_copy(map, key, value);
Alexei Starovoitov15a07b32016-02-01 22:39:55 -0800326 } else {
327 rcu_read_lock();
328 ptr = map->ops->map_lookup_elem(map, key);
329 if (ptr)
330 memcpy(value, ptr, value_size);
331 rcu_read_unlock();
332 err = ptr ? 0 : -ENOENT;
333 }
Alexei Starovoitov8ebe6672015-01-22 17:11:08 -0800334
Alexei Starovoitov15a07b32016-02-01 22:39:55 -0800335 if (err)
Alexei Starovoitov8ebe6672015-01-22 17:11:08 -0800336 goto free_value;
Alexei Starovoitovdb20fd22014-09-26 00:16:59 -0700337
338 err = -EFAULT;
Alexei Starovoitov15a07b32016-02-01 22:39:55 -0800339 if (copy_to_user(uvalue, value, value_size) != 0)
Alexei Starovoitov8ebe6672015-01-22 17:11:08 -0800340 goto free_value;
Alexei Starovoitovdb20fd22014-09-26 00:16:59 -0700341
342 err = 0;
343
Alexei Starovoitov8ebe6672015-01-22 17:11:08 -0800344free_value:
345 kfree(value);
Alexei Starovoitovdb20fd22014-09-26 00:16:59 -0700346free_key:
347 kfree(key);
348err_put:
349 fdput(f);
350 return err;
351}
352
Alexei Starovoitov3274f522014-11-13 17:36:44 -0800353#define BPF_MAP_UPDATE_ELEM_LAST_FIELD flags
Alexei Starovoitovdb20fd22014-09-26 00:16:59 -0700354
355static int map_update_elem(union bpf_attr *attr)
356{
Mickaël Salaün535e7b4b2016-11-13 19:44:03 +0100357 void __user *ukey = u64_to_user_ptr(attr->key);
358 void __user *uvalue = u64_to_user_ptr(attr->value);
Alexei Starovoitovdb20fd22014-09-26 00:16:59 -0700359 int ufd = attr->map_fd;
Alexei Starovoitovdb20fd22014-09-26 00:16:59 -0700360 struct bpf_map *map;
361 void *key, *value;
Alexei Starovoitov15a07b32016-02-01 22:39:55 -0800362 u32 value_size;
Daniel Borkmann592867b2015-09-08 18:00:09 +0200363 struct fd f;
Alexei Starovoitovdb20fd22014-09-26 00:16:59 -0700364 int err;
365
366 if (CHECK_ATTR(BPF_MAP_UPDATE_ELEM))
367 return -EINVAL;
368
Daniel Borkmann592867b2015-09-08 18:00:09 +0200369 f = fdget(ufd);
Daniel Borkmannc2101292015-10-29 14:58:07 +0100370 map = __bpf_map_get(f);
Alexei Starovoitovdb20fd22014-09-26 00:16:59 -0700371 if (IS_ERR(map))
372 return PTR_ERR(map);
373
374 err = -ENOMEM;
375 key = kmalloc(map->key_size, GFP_USER);
376 if (!key)
377 goto err_put;
378
379 err = -EFAULT;
380 if (copy_from_user(key, ukey, map->key_size) != 0)
381 goto free_key;
382
Alexei Starovoitov15a07b32016-02-01 22:39:55 -0800383 if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH ||
Martin KaFai Lau8f844932016-11-11 10:55:10 -0800384 map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH ||
Alexei Starovoitov15a07b32016-02-01 22:39:55 -0800385 map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY)
386 value_size = round_up(map->value_size, 8) * num_possible_cpus();
387 else
388 value_size = map->value_size;
389
Alexei Starovoitovdb20fd22014-09-26 00:16:59 -0700390 err = -ENOMEM;
Alexei Starovoitov15a07b32016-02-01 22:39:55 -0800391 value = kmalloc(value_size, GFP_USER | __GFP_NOWARN);
Alexei Starovoitovdb20fd22014-09-26 00:16:59 -0700392 if (!value)
393 goto free_key;
394
395 err = -EFAULT;
Alexei Starovoitov15a07b32016-02-01 22:39:55 -0800396 if (copy_from_user(value, uvalue, value_size) != 0)
Alexei Starovoitovdb20fd22014-09-26 00:16:59 -0700397 goto free_value;
398
Alexei Starovoitovb121d1e2016-03-07 21:57:13 -0800399 /* must increment bpf_prog_active to avoid kprobe+bpf triggering from
400 * inside bpf map update or delete otherwise deadlocks are possible
401 */
402 preempt_disable();
403 __this_cpu_inc(bpf_prog_active);
Martin KaFai Lau8f844932016-11-11 10:55:10 -0800404 if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH ||
405 map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH) {
Alexei Starovoitov15a07b32016-02-01 22:39:55 -0800406 err = bpf_percpu_hash_update(map, key, value, attr->flags);
407 } else if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) {
408 err = bpf_percpu_array_update(map, key, value, attr->flags);
Daniel Borkmannd056a782016-06-15 22:47:13 +0200409 } else if (map->map_type == BPF_MAP_TYPE_PERF_EVENT_ARRAY ||
Martin KaFai Lau4ed8ec52016-06-30 10:28:43 -0700410 map->map_type == BPF_MAP_TYPE_PROG_ARRAY ||
411 map->map_type == BPF_MAP_TYPE_CGROUP_ARRAY) {
Daniel Borkmannd056a782016-06-15 22:47:13 +0200412 rcu_read_lock();
413 err = bpf_fd_array_map_update_elem(map, f.file, key, value,
414 attr->flags);
415 rcu_read_unlock();
Alexei Starovoitov15a07b32016-02-01 22:39:55 -0800416 } else {
417 rcu_read_lock();
418 err = map->ops->map_update_elem(map, key, value, attr->flags);
419 rcu_read_unlock();
420 }
Alexei Starovoitovb121d1e2016-03-07 21:57:13 -0800421 __this_cpu_dec(bpf_prog_active);
422 preempt_enable();
Alexei Starovoitovdb20fd22014-09-26 00:16:59 -0700423
424free_value:
425 kfree(value);
426free_key:
427 kfree(key);
428err_put:
429 fdput(f);
430 return err;
431}
432
433#define BPF_MAP_DELETE_ELEM_LAST_FIELD key
434
435static int map_delete_elem(union bpf_attr *attr)
436{
Mickaël Salaün535e7b4b2016-11-13 19:44:03 +0100437 void __user *ukey = u64_to_user_ptr(attr->key);
Alexei Starovoitovdb20fd22014-09-26 00:16:59 -0700438 int ufd = attr->map_fd;
Alexei Starovoitovdb20fd22014-09-26 00:16:59 -0700439 struct bpf_map *map;
Daniel Borkmann592867b2015-09-08 18:00:09 +0200440 struct fd f;
Alexei Starovoitovdb20fd22014-09-26 00:16:59 -0700441 void *key;
442 int err;
443
444 if (CHECK_ATTR(BPF_MAP_DELETE_ELEM))
445 return -EINVAL;
446
Daniel Borkmann592867b2015-09-08 18:00:09 +0200447 f = fdget(ufd);
Daniel Borkmannc2101292015-10-29 14:58:07 +0100448 map = __bpf_map_get(f);
Alexei Starovoitovdb20fd22014-09-26 00:16:59 -0700449 if (IS_ERR(map))
450 return PTR_ERR(map);
451
452 err = -ENOMEM;
453 key = kmalloc(map->key_size, GFP_USER);
454 if (!key)
455 goto err_put;
456
457 err = -EFAULT;
458 if (copy_from_user(key, ukey, map->key_size) != 0)
459 goto free_key;
460
Alexei Starovoitovb121d1e2016-03-07 21:57:13 -0800461 preempt_disable();
462 __this_cpu_inc(bpf_prog_active);
Alexei Starovoitovdb20fd22014-09-26 00:16:59 -0700463 rcu_read_lock();
464 err = map->ops->map_delete_elem(map, key);
465 rcu_read_unlock();
Alexei Starovoitovb121d1e2016-03-07 21:57:13 -0800466 __this_cpu_dec(bpf_prog_active);
467 preempt_enable();
Alexei Starovoitovdb20fd22014-09-26 00:16:59 -0700468
469free_key:
470 kfree(key);
471err_put:
472 fdput(f);
473 return err;
474}
475
476/* last field in 'union bpf_attr' used by this command */
477#define BPF_MAP_GET_NEXT_KEY_LAST_FIELD next_key
478
479static int map_get_next_key(union bpf_attr *attr)
480{
Mickaël Salaün535e7b4b2016-11-13 19:44:03 +0100481 void __user *ukey = u64_to_user_ptr(attr->key);
482 void __user *unext_key = u64_to_user_ptr(attr->next_key);
Alexei Starovoitovdb20fd22014-09-26 00:16:59 -0700483 int ufd = attr->map_fd;
Alexei Starovoitovdb20fd22014-09-26 00:16:59 -0700484 struct bpf_map *map;
485 void *key, *next_key;
Daniel Borkmann592867b2015-09-08 18:00:09 +0200486 struct fd f;
Alexei Starovoitovdb20fd22014-09-26 00:16:59 -0700487 int err;
488
489 if (CHECK_ATTR(BPF_MAP_GET_NEXT_KEY))
490 return -EINVAL;
491
Daniel Borkmann592867b2015-09-08 18:00:09 +0200492 f = fdget(ufd);
Daniel Borkmannc2101292015-10-29 14:58:07 +0100493 map = __bpf_map_get(f);
Alexei Starovoitovdb20fd22014-09-26 00:16:59 -0700494 if (IS_ERR(map))
495 return PTR_ERR(map);
496
497 err = -ENOMEM;
498 key = kmalloc(map->key_size, GFP_USER);
499 if (!key)
500 goto err_put;
501
502 err = -EFAULT;
503 if (copy_from_user(key, ukey, map->key_size) != 0)
504 goto free_key;
505
506 err = -ENOMEM;
507 next_key = kmalloc(map->key_size, GFP_USER);
508 if (!next_key)
509 goto free_key;
510
511 rcu_read_lock();
512 err = map->ops->map_get_next_key(map, key, next_key);
513 rcu_read_unlock();
514 if (err)
515 goto free_next_key;
516
517 err = -EFAULT;
518 if (copy_to_user(unext_key, next_key, map->key_size) != 0)
519 goto free_next_key;
520
521 err = 0;
522
523free_next_key:
524 kfree(next_key);
525free_key:
526 kfree(key);
527err_put:
528 fdput(f);
529 return err;
530}
531
Alexei Starovoitov09756af2014-09-26 00:17:00 -0700532static LIST_HEAD(bpf_prog_types);
533
534static int find_prog_type(enum bpf_prog_type type, struct bpf_prog *prog)
535{
536 struct bpf_prog_type_list *tl;
537
538 list_for_each_entry(tl, &bpf_prog_types, list_node) {
539 if (tl->type == type) {
540 prog->aux->ops = tl->ops;
Daniel Borkmann24701ec2015-03-01 12:31:47 +0100541 prog->type = type;
Alexei Starovoitov09756af2014-09-26 00:17:00 -0700542 return 0;
543 }
544 }
Daniel Borkmann24701ec2015-03-01 12:31:47 +0100545
Alexei Starovoitov09756af2014-09-26 00:17:00 -0700546 return -EINVAL;
547}
548
549void bpf_register_prog_type(struct bpf_prog_type_list *tl)
550{
551 list_add(&tl->list_node, &bpf_prog_types);
552}
553
Alexei Starovoitov0a542a82014-09-26 00:17:01 -0700554/* fixup insn->imm field of bpf_call instructions:
555 * if (insn->imm == BPF_FUNC_map_lookup_elem)
556 * insn->imm = bpf_map_lookup_elem - __bpf_call_base;
557 * else if (insn->imm == BPF_FUNC_map_update_elem)
558 * insn->imm = bpf_map_update_elem - __bpf_call_base;
559 * else ...
560 *
561 * this function is called after eBPF program passed verification
562 */
563static void fixup_bpf_calls(struct bpf_prog *prog)
564{
565 const struct bpf_func_proto *fn;
566 int i;
567
568 for (i = 0; i < prog->len; i++) {
569 struct bpf_insn *insn = &prog->insnsi[i];
570
571 if (insn->code == (BPF_JMP | BPF_CALL)) {
572 /* we reach here when program has bpf_call instructions
573 * and it passed bpf_check(), means that
574 * ops->get_func_proto must have been supplied, check it
575 */
576 BUG_ON(!prog->aux->ops->get_func_proto);
577
Daniel Borkmannc46646d2015-09-30 01:41:51 +0200578 if (insn->imm == BPF_FUNC_get_route_realm)
579 prog->dst_needed = 1;
Daniel Borkmann3ad00402015-10-08 01:20:39 +0200580 if (insn->imm == BPF_FUNC_get_prandom_u32)
581 bpf_user_rnd_init_once();
Alexei Starovoitov04fd61ab2015-05-19 16:59:03 -0700582 if (insn->imm == BPF_FUNC_tail_call) {
583 /* mark bpf_tail_call as different opcode
584 * to avoid conditional branch in
585 * interpeter for every normal call
586 * and to prevent accidental JITing by
587 * JIT compiler that doesn't support
588 * bpf_tail_call yet
589 */
590 insn->imm = 0;
591 insn->code |= BPF_X;
592 continue;
593 }
594
Alexei Starovoitov0a542a82014-09-26 00:17:01 -0700595 fn = prog->aux->ops->get_func_proto(insn->imm);
596 /* all functions that have prototype and verifier allowed
597 * programs to call them, must be real in-kernel functions
598 */
599 BUG_ON(!fn->func);
600 insn->imm = fn->func - __bpf_call_base;
601 }
602 }
603}
604
Alexei Starovoitov09756af2014-09-26 00:17:00 -0700605/* drop refcnt on maps used by eBPF program and free auxilary data */
606static void free_used_maps(struct bpf_prog_aux *aux)
607{
608 int i;
609
610 for (i = 0; i < aux->used_map_cnt; i++)
611 bpf_map_put(aux->used_maps[i]);
612
613 kfree(aux->used_maps);
614}
615
Alexei Starovoitovaaac3ba2015-10-07 22:23:22 -0700616static int bpf_prog_charge_memlock(struct bpf_prog *prog)
617{
618 struct user_struct *user = get_current_user();
619 unsigned long memlock_limit;
620
621 memlock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
622
623 atomic_long_add(prog->pages, &user->locked_vm);
624 if (atomic_long_read(&user->locked_vm) > memlock_limit) {
625 atomic_long_sub(prog->pages, &user->locked_vm);
626 free_uid(user);
627 return -EPERM;
628 }
629 prog->aux->user = user;
630 return 0;
631}
632
633static void bpf_prog_uncharge_memlock(struct bpf_prog *prog)
634{
635 struct user_struct *user = prog->aux->user;
636
637 atomic_long_sub(prog->pages, &user->locked_vm);
638 free_uid(user);
639}
640
Daniel Borkmann1aacde32016-06-30 17:24:43 +0200641static void __bpf_prog_put_rcu(struct rcu_head *rcu)
Alexei Starovoitovabf2e7d2015-05-28 19:26:02 -0700642{
643 struct bpf_prog_aux *aux = container_of(rcu, struct bpf_prog_aux, rcu);
644
645 free_used_maps(aux);
Alexei Starovoitovaaac3ba2015-10-07 22:23:22 -0700646 bpf_prog_uncharge_memlock(aux->prog);
Alexei Starovoitovabf2e7d2015-05-28 19:26:02 -0700647 bpf_prog_free(aux->prog);
648}
649
Alexei Starovoitov09756af2014-09-26 00:17:00 -0700650void bpf_prog_put(struct bpf_prog *prog)
651{
Daniel Borkmanne9d8afa2015-10-29 14:58:08 +0100652 if (atomic_dec_and_test(&prog->aux->refcnt))
Daniel Borkmann1aacde32016-06-30 17:24:43 +0200653 call_rcu(&prog->aux->rcu, __bpf_prog_put_rcu);
Alexei Starovoitov09756af2014-09-26 00:17:00 -0700654}
Daniel Borkmanne2e9b652015-03-01 12:31:48 +0100655EXPORT_SYMBOL_GPL(bpf_prog_put);
Alexei Starovoitov09756af2014-09-26 00:17:00 -0700656
657static int bpf_prog_release(struct inode *inode, struct file *filp)
658{
659 struct bpf_prog *prog = filp->private_data;
660
Daniel Borkmann1aacde32016-06-30 17:24:43 +0200661 bpf_prog_put(prog);
Alexei Starovoitov09756af2014-09-26 00:17:00 -0700662 return 0;
663}
664
Daniel Borkmann7bd509e2016-12-04 23:19:41 +0100665#ifdef CONFIG_PROC_FS
666static void bpf_prog_show_fdinfo(struct seq_file *m, struct file *filp)
667{
668 const struct bpf_prog *prog = filp->private_data;
669 char prog_digest[sizeof(prog->digest) * 2 + 1] = { };
670
671 bin2hex(prog_digest, prog->digest, sizeof(prog->digest));
672 seq_printf(m,
673 "prog_type:\t%u\n"
674 "prog_jited:\t%u\n"
675 "prog_digest:\t%s\n"
676 "memlock:\t%llu\n",
677 prog->type,
678 prog->jited,
679 prog_digest,
680 prog->pages * 1ULL << PAGE_SHIFT);
681}
682#endif
683
Alexei Starovoitov09756af2014-09-26 00:17:00 -0700684static const struct file_operations bpf_prog_fops = {
Daniel Borkmann7bd509e2016-12-04 23:19:41 +0100685#ifdef CONFIG_PROC_FS
686 .show_fdinfo = bpf_prog_show_fdinfo,
687#endif
688 .release = bpf_prog_release,
Alexei Starovoitov09756af2014-09-26 00:17:00 -0700689};
690
Daniel Borkmannb2197752015-10-29 14:58:09 +0100691int bpf_prog_new_fd(struct bpf_prog *prog)
Daniel Borkmannaa797812015-10-29 14:58:06 +0100692{
693 return anon_inode_getfd("bpf-prog", &bpf_prog_fops, prog,
694 O_RDWR | O_CLOEXEC);
695}
696
Daniel Borkmann113214b2016-06-30 17:24:44 +0200697static struct bpf_prog *____bpf_prog_get(struct fd f)
Alexei Starovoitov09756af2014-09-26 00:17:00 -0700698{
Alexei Starovoitov09756af2014-09-26 00:17:00 -0700699 if (!f.file)
700 return ERR_PTR(-EBADF);
Alexei Starovoitov09756af2014-09-26 00:17:00 -0700701 if (f.file->f_op != &bpf_prog_fops) {
702 fdput(f);
703 return ERR_PTR(-EINVAL);
704 }
705
Daniel Borkmannc2101292015-10-29 14:58:07 +0100706 return f.file->private_data;
Alexei Starovoitov09756af2014-09-26 00:17:00 -0700707}
708
Brenden Blanco59d36562016-07-19 12:16:46 -0700709struct bpf_prog *bpf_prog_add(struct bpf_prog *prog, int i)
Alexei Starovoitov92117d82016-04-27 18:56:20 -0700710{
Brenden Blanco59d36562016-07-19 12:16:46 -0700711 if (atomic_add_return(i, &prog->aux->refcnt) > BPF_MAX_REFCNT) {
712 atomic_sub(i, &prog->aux->refcnt);
Alexei Starovoitov92117d82016-04-27 18:56:20 -0700713 return ERR_PTR(-EBUSY);
714 }
715 return prog;
716}
Brenden Blanco59d36562016-07-19 12:16:46 -0700717EXPORT_SYMBOL_GPL(bpf_prog_add);
718
Daniel Borkmannc5405942016-11-09 22:02:34 +0100719void bpf_prog_sub(struct bpf_prog *prog, int i)
720{
721 /* Only to be used for undoing previous bpf_prog_add() in some
722 * error path. We still know that another entity in our call
723 * path holds a reference to the program, thus atomic_sub() can
724 * be safely used in such cases!
725 */
726 WARN_ON(atomic_sub_return(i, &prog->aux->refcnt) == 0);
727}
728EXPORT_SYMBOL_GPL(bpf_prog_sub);
729
Brenden Blanco59d36562016-07-19 12:16:46 -0700730struct bpf_prog *bpf_prog_inc(struct bpf_prog *prog)
731{
732 return bpf_prog_add(prog, 1);
733}
Daniel Borkmann97bc4022016-11-19 01:45:00 +0100734EXPORT_SYMBOL_GPL(bpf_prog_inc);
Alexei Starovoitov92117d82016-04-27 18:56:20 -0700735
Daniel Borkmann113214b2016-06-30 17:24:44 +0200736static struct bpf_prog *__bpf_prog_get(u32 ufd, enum bpf_prog_type *type)
Alexei Starovoitov09756af2014-09-26 00:17:00 -0700737{
738 struct fd f = fdget(ufd);
739 struct bpf_prog *prog;
740
Daniel Borkmann113214b2016-06-30 17:24:44 +0200741 prog = ____bpf_prog_get(f);
Alexei Starovoitov09756af2014-09-26 00:17:00 -0700742 if (IS_ERR(prog))
743 return prog;
Daniel Borkmann113214b2016-06-30 17:24:44 +0200744 if (type && prog->type != *type) {
745 prog = ERR_PTR(-EINVAL);
746 goto out;
747 }
Alexei Starovoitov09756af2014-09-26 00:17:00 -0700748
Alexei Starovoitov92117d82016-04-27 18:56:20 -0700749 prog = bpf_prog_inc(prog);
Daniel Borkmann113214b2016-06-30 17:24:44 +0200750out:
Alexei Starovoitov09756af2014-09-26 00:17:00 -0700751 fdput(f);
752 return prog;
753}
Daniel Borkmann113214b2016-06-30 17:24:44 +0200754
755struct bpf_prog *bpf_prog_get(u32 ufd)
756{
757 return __bpf_prog_get(ufd, NULL);
758}
759
760struct bpf_prog *bpf_prog_get_type(u32 ufd, enum bpf_prog_type type)
761{
762 return __bpf_prog_get(ufd, &type);
763}
764EXPORT_SYMBOL_GPL(bpf_prog_get_type);
Alexei Starovoitov09756af2014-09-26 00:17:00 -0700765
766/* last field in 'union bpf_attr' used by this command */
Alexei Starovoitov25415172015-03-25 12:49:20 -0700767#define BPF_PROG_LOAD_LAST_FIELD kern_version
Alexei Starovoitov09756af2014-09-26 00:17:00 -0700768
769static int bpf_prog_load(union bpf_attr *attr)
770{
771 enum bpf_prog_type type = attr->prog_type;
772 struct bpf_prog *prog;
773 int err;
774 char license[128];
775 bool is_gpl;
776
777 if (CHECK_ATTR(BPF_PROG_LOAD))
778 return -EINVAL;
779
780 /* copy eBPF program license from user space */
Mickaël Salaün535e7b4b2016-11-13 19:44:03 +0100781 if (strncpy_from_user(license, u64_to_user_ptr(attr->license),
Alexei Starovoitov09756af2014-09-26 00:17:00 -0700782 sizeof(license) - 1) < 0)
783 return -EFAULT;
784 license[sizeof(license) - 1] = 0;
785
786 /* eBPF programs must be GPL compatible to use GPL-ed functions */
787 is_gpl = license_is_gpl_compatible(license);
788
Daniel Borkmannef0915c2016-12-07 01:15:44 +0100789 if (attr->insn_cnt == 0 || attr->insn_cnt > BPF_MAXINSNS)
790 return -E2BIG;
Alexei Starovoitov09756af2014-09-26 00:17:00 -0700791
Alexei Starovoitov25415172015-03-25 12:49:20 -0700792 if (type == BPF_PROG_TYPE_KPROBE &&
793 attr->kern_version != LINUX_VERSION_CODE)
794 return -EINVAL;
795
Alexei Starovoitov1be7f752015-10-07 22:23:21 -0700796 if (type != BPF_PROG_TYPE_SOCKET_FILTER && !capable(CAP_SYS_ADMIN))
797 return -EPERM;
798
Alexei Starovoitov09756af2014-09-26 00:17:00 -0700799 /* plain bpf_prog allocation */
800 prog = bpf_prog_alloc(bpf_prog_size(attr->insn_cnt), GFP_USER);
801 if (!prog)
802 return -ENOMEM;
803
Alexei Starovoitovaaac3ba2015-10-07 22:23:22 -0700804 err = bpf_prog_charge_memlock(prog);
805 if (err)
806 goto free_prog_nouncharge;
807
Alexei Starovoitov09756af2014-09-26 00:17:00 -0700808 prog->len = attr->insn_cnt;
809
810 err = -EFAULT;
Mickaël Salaün535e7b4b2016-11-13 19:44:03 +0100811 if (copy_from_user(prog->insns, u64_to_user_ptr(attr->insns),
Alexei Starovoitov09756af2014-09-26 00:17:00 -0700812 prog->len * sizeof(struct bpf_insn)) != 0)
813 goto free_prog;
814
815 prog->orig_prog = NULL;
Daniel Borkmanna91263d2015-09-30 01:41:50 +0200816 prog->jited = 0;
Alexei Starovoitov09756af2014-09-26 00:17:00 -0700817
818 atomic_set(&prog->aux->refcnt, 1);
Daniel Borkmanna91263d2015-09-30 01:41:50 +0200819 prog->gpl_compatible = is_gpl ? 1 : 0;
Alexei Starovoitov09756af2014-09-26 00:17:00 -0700820
821 /* find program type: socket_filter vs tracing_filter */
822 err = find_prog_type(type, prog);
823 if (err < 0)
824 goto free_prog;
825
826 /* run eBPF verifier */
Alexei Starovoitov9bac3d62015-03-13 11:57:42 -0700827 err = bpf_check(&prog, attr);
Alexei Starovoitov09756af2014-09-26 00:17:00 -0700828 if (err < 0)
829 goto free_used_maps;
830
Alexei Starovoitov0a542a82014-09-26 00:17:01 -0700831 /* fixup BPF_CALL->imm field */
832 fixup_bpf_calls(prog);
833
Alexei Starovoitov09756af2014-09-26 00:17:00 -0700834 /* eBPF program is ready to be JITed */
Daniel Borkmannd1c55ab2016-05-13 19:08:31 +0200835 prog = bpf_prog_select_runtime(prog, &err);
Alexei Starovoitov04fd61ab2015-05-19 16:59:03 -0700836 if (err < 0)
837 goto free_used_maps;
Alexei Starovoitov09756af2014-09-26 00:17:00 -0700838
Daniel Borkmannaa797812015-10-29 14:58:06 +0100839 err = bpf_prog_new_fd(prog);
Alexei Starovoitov09756af2014-09-26 00:17:00 -0700840 if (err < 0)
841 /* failed to allocate fd */
842 goto free_used_maps;
843
844 return err;
845
846free_used_maps:
847 free_used_maps(prog->aux);
848free_prog:
Alexei Starovoitovaaac3ba2015-10-07 22:23:22 -0700849 bpf_prog_uncharge_memlock(prog);
850free_prog_nouncharge:
Alexei Starovoitov09756af2014-09-26 00:17:00 -0700851 bpf_prog_free(prog);
852 return err;
853}
854
Daniel Borkmannb2197752015-10-29 14:58:09 +0100855#define BPF_OBJ_LAST_FIELD bpf_fd
856
857static int bpf_obj_pin(const union bpf_attr *attr)
858{
859 if (CHECK_ATTR(BPF_OBJ))
860 return -EINVAL;
861
Mickaël Salaün535e7b4b2016-11-13 19:44:03 +0100862 return bpf_obj_pin_user(attr->bpf_fd, u64_to_user_ptr(attr->pathname));
Daniel Borkmannb2197752015-10-29 14:58:09 +0100863}
864
865static int bpf_obj_get(const union bpf_attr *attr)
866{
867 if (CHECK_ATTR(BPF_OBJ) || attr->bpf_fd != 0)
868 return -EINVAL;
869
Mickaël Salaün535e7b4b2016-11-13 19:44:03 +0100870 return bpf_obj_get_user(u64_to_user_ptr(attr->pathname));
Daniel Borkmannb2197752015-10-29 14:58:09 +0100871}
872
Daniel Mackf4324552016-11-23 16:52:27 +0100873#ifdef CONFIG_CGROUP_BPF
874
875#define BPF_PROG_ATTACH_LAST_FIELD attach_type
876
877static int bpf_prog_attach(const union bpf_attr *attr)
878{
879 struct bpf_prog *prog;
880 struct cgroup *cgrp;
David Ahernb2cd1252016-12-01 08:48:03 -0800881 enum bpf_prog_type ptype;
Daniel Mackf4324552016-11-23 16:52:27 +0100882
883 if (!capable(CAP_NET_ADMIN))
884 return -EPERM;
885
886 if (CHECK_ATTR(BPF_PROG_ATTACH))
887 return -EINVAL;
888
889 switch (attr->attach_type) {
890 case BPF_CGROUP_INET_INGRESS:
891 case BPF_CGROUP_INET_EGRESS:
David Ahernb2cd1252016-12-01 08:48:03 -0800892 ptype = BPF_PROG_TYPE_CGROUP_SKB;
Daniel Mackf4324552016-11-23 16:52:27 +0100893 break;
David Ahern610236582016-12-01 08:48:04 -0800894 case BPF_CGROUP_INET_SOCK_CREATE:
895 ptype = BPF_PROG_TYPE_CGROUP_SOCK;
896 break;
Daniel Mackf4324552016-11-23 16:52:27 +0100897 default:
898 return -EINVAL;
899 }
900
David Ahernb2cd1252016-12-01 08:48:03 -0800901 prog = bpf_prog_get_type(attr->attach_bpf_fd, ptype);
902 if (IS_ERR(prog))
903 return PTR_ERR(prog);
904
905 cgrp = cgroup_get_from_fd(attr->target_fd);
906 if (IS_ERR(cgrp)) {
907 bpf_prog_put(prog);
908 return PTR_ERR(cgrp);
909 }
910
911 cgroup_bpf_update(cgrp, prog, attr->attach_type);
912 cgroup_put(cgrp);
913
Daniel Mackf4324552016-11-23 16:52:27 +0100914 return 0;
915}
916
917#define BPF_PROG_DETACH_LAST_FIELD attach_type
918
919static int bpf_prog_detach(const union bpf_attr *attr)
920{
921 struct cgroup *cgrp;
922
923 if (!capable(CAP_NET_ADMIN))
924 return -EPERM;
925
926 if (CHECK_ATTR(BPF_PROG_DETACH))
927 return -EINVAL;
928
929 switch (attr->attach_type) {
930 case BPF_CGROUP_INET_INGRESS:
931 case BPF_CGROUP_INET_EGRESS:
David Ahern610236582016-12-01 08:48:04 -0800932 case BPF_CGROUP_INET_SOCK_CREATE:
Daniel Mackf4324552016-11-23 16:52:27 +0100933 cgrp = cgroup_get_from_fd(attr->target_fd);
934 if (IS_ERR(cgrp))
935 return PTR_ERR(cgrp);
936
937 cgroup_bpf_update(cgrp, NULL, attr->attach_type);
938 cgroup_put(cgrp);
939 break;
940
941 default:
942 return -EINVAL;
943 }
944
945 return 0;
946}
947#endif /* CONFIG_CGROUP_BPF */
948
Alexei Starovoitov99c55f72014-09-26 00:16:57 -0700949SYSCALL_DEFINE3(bpf, int, cmd, union bpf_attr __user *, uattr, unsigned int, size)
950{
951 union bpf_attr attr = {};
952 int err;
953
Alexei Starovoitov1be7f752015-10-07 22:23:21 -0700954 if (!capable(CAP_SYS_ADMIN) && sysctl_unprivileged_bpf_disabled)
Alexei Starovoitov99c55f72014-09-26 00:16:57 -0700955 return -EPERM;
956
957 if (!access_ok(VERIFY_READ, uattr, 1))
958 return -EFAULT;
959
960 if (size > PAGE_SIZE) /* silly large */
961 return -E2BIG;
962
963 /* If we're handed a bigger struct than we know of,
964 * ensure all the unknown bits are 0 - i.e. new
965 * user-space does not rely on any kernel feature
966 * extensions we dont know about yet.
967 */
968 if (size > sizeof(attr)) {
969 unsigned char __user *addr;
970 unsigned char __user *end;
971 unsigned char val;
972
973 addr = (void __user *)uattr + sizeof(attr);
974 end = (void __user *)uattr + size;
975
976 for (; addr < end; addr++) {
977 err = get_user(val, addr);
978 if (err)
979 return err;
980 if (val)
981 return -E2BIG;
982 }
983 size = sizeof(attr);
984 }
985
986 /* copy attributes from user space, may be less than sizeof(bpf_attr) */
987 if (copy_from_user(&attr, uattr, size) != 0)
988 return -EFAULT;
989
990 switch (cmd) {
991 case BPF_MAP_CREATE:
992 err = map_create(&attr);
993 break;
Alexei Starovoitovdb20fd22014-09-26 00:16:59 -0700994 case BPF_MAP_LOOKUP_ELEM:
995 err = map_lookup_elem(&attr);
996 break;
997 case BPF_MAP_UPDATE_ELEM:
998 err = map_update_elem(&attr);
999 break;
1000 case BPF_MAP_DELETE_ELEM:
1001 err = map_delete_elem(&attr);
1002 break;
1003 case BPF_MAP_GET_NEXT_KEY:
1004 err = map_get_next_key(&attr);
1005 break;
Alexei Starovoitov09756af2014-09-26 00:17:00 -07001006 case BPF_PROG_LOAD:
1007 err = bpf_prog_load(&attr);
1008 break;
Daniel Borkmannb2197752015-10-29 14:58:09 +01001009 case BPF_OBJ_PIN:
1010 err = bpf_obj_pin(&attr);
1011 break;
1012 case BPF_OBJ_GET:
1013 err = bpf_obj_get(&attr);
1014 break;
Daniel Mackf4324552016-11-23 16:52:27 +01001015
1016#ifdef CONFIG_CGROUP_BPF
1017 case BPF_PROG_ATTACH:
1018 err = bpf_prog_attach(&attr);
1019 break;
1020 case BPF_PROG_DETACH:
1021 err = bpf_prog_detach(&attr);
1022 break;
1023#endif
1024
Alexei Starovoitov99c55f72014-09-26 00:16:57 -07001025 default:
1026 err = -EINVAL;
1027 break;
1028 }
1029
1030 return err;
1031}