blob: 3dc3c90496905d5c2b3120843c49413205c9372c [file] [log] [blame]
Seth Jenningsb700e7f2014-12-16 11:58:19 -06001/*
2 * core.c - Kernel Live Patching Core
3 *
4 * Copyright (C) 2014 Seth Jennings <sjenning@redhat.com>
5 * Copyright (C) 2014 SUSE
6 *
7 * This program is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU General Public License
9 * as published by the Free Software Foundation; either version 2
10 * of the License, or (at your option) any later version.
11 *
12 * This program is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 * GNU General Public License for more details.
16 *
17 * You should have received a copy of the GNU General Public License
18 * along with this program; if not, see <http://www.gnu.org/licenses/>.
19 */
20
21#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
22
23#include <linux/module.h>
24#include <linux/kernel.h>
25#include <linux/mutex.h>
26#include <linux/slab.h>
Seth Jenningsb700e7f2014-12-16 11:58:19 -060027#include <linux/list.h>
28#include <linux/kallsyms.h>
29#include <linux/livepatch.h>
Jessica Yu425595a2016-03-22 20:03:18 -040030#include <linux/elf.h>
31#include <linux/moduleloader.h>
Josh Poimboeufb56b36e2015-12-03 16:33:26 -060032#include <asm/cacheflush.h>
Josh Poimboeufc349cdca2017-02-13 19:42:37 -060033#include "patch.h"
Josh Poimboeufd83a7cb2017-02-13 19:42:40 -060034#include "transition.h"
Seth Jenningsb700e7f2014-12-16 11:58:19 -060035
Josh Poimboeuf3c33f5b2015-01-20 09:26:19 -060036/*
Josh Poimboeufd83a7cb2017-02-13 19:42:40 -060037 * klp_mutex is a coarse lock which serializes access to klp data. All
38 * accesses to klp-related variables and structures must have mutex protection,
39 * except within the following functions which carefully avoid the need for it:
40 *
41 * - klp_ftrace_handler()
42 * - klp_update_patch_state()
Josh Poimboeuf3c33f5b2015-01-20 09:26:19 -060043 */
Josh Poimboeufd83a7cb2017-02-13 19:42:40 -060044DEFINE_MUTEX(klp_mutex);
Josh Poimboeuf3c33f5b2015-01-20 09:26:19 -060045
Seth Jenningsb700e7f2014-12-16 11:58:19 -060046static LIST_HEAD(klp_patches);
47
48static struct kobject *klp_root_kobj;
49
50static bool klp_is_module(struct klp_object *obj)
51{
52 return obj->name;
53}
54
55static bool klp_is_object_loaded(struct klp_object *obj)
56{
57 return !obj->name || obj->mod;
58}
59
60/* sets obj->mod if object is not vmlinux and module is found */
61static void klp_find_object_module(struct klp_object *obj)
62{
Petr Mladek8cb2c2d2015-03-12 12:55:13 +010063 struct module *mod;
64
Seth Jenningsb700e7f2014-12-16 11:58:19 -060065 if (!klp_is_module(obj))
66 return;
67
68 mutex_lock(&module_mutex);
69 /*
Petr Mladek8cb2c2d2015-03-12 12:55:13 +010070 * We do not want to block removal of patched modules and therefore
71 * we do not take a reference here. The patches are removed by
Jessica Yu7e545d62016-03-16 20:55:39 -040072 * klp_module_going() instead.
Seth Jenningsb700e7f2014-12-16 11:58:19 -060073 */
Petr Mladek8cb2c2d2015-03-12 12:55:13 +010074 mod = find_module(obj->name);
75 /*
Jessica Yu7e545d62016-03-16 20:55:39 -040076 * Do not mess work of klp_module_coming() and klp_module_going().
77 * Note that the patch might still be needed before klp_module_going()
Petr Mladek8cb2c2d2015-03-12 12:55:13 +010078 * is called. Module functions can be called even in the GOING state
79 * until mod->exit() finishes. This is especially important for
80 * patches that modify semantic of the functions.
81 */
82 if (mod && mod->klp_alive)
83 obj->mod = mod;
84
Seth Jenningsb700e7f2014-12-16 11:58:19 -060085 mutex_unlock(&module_mutex);
86}
87
Seth Jenningsb700e7f2014-12-16 11:58:19 -060088static bool klp_is_patch_registered(struct klp_patch *patch)
89{
90 struct klp_patch *mypatch;
91
92 list_for_each_entry(mypatch, &klp_patches, list)
93 if (mypatch == patch)
94 return true;
95
96 return false;
97}
98
99static bool klp_initialized(void)
100{
Nicholas Mc Guiree76ff062015-05-11 07:52:29 +0200101 return !!klp_root_kobj;
Seth Jenningsb700e7f2014-12-16 11:58:19 -0600102}
103
104struct klp_find_arg {
105 const char *objname;
106 const char *name;
107 unsigned long addr;
Seth Jenningsb700e7f2014-12-16 11:58:19 -0600108 unsigned long count;
Chris J Argesb2b018e2015-12-01 20:40:54 -0600109 unsigned long pos;
Seth Jenningsb700e7f2014-12-16 11:58:19 -0600110};
111
112static int klp_find_callback(void *data, const char *name,
113 struct module *mod, unsigned long addr)
114{
115 struct klp_find_arg *args = data;
116
117 if ((mod && !args->objname) || (!mod && args->objname))
118 return 0;
119
120 if (strcmp(args->name, name))
121 return 0;
122
123 if (args->objname && strcmp(args->objname, mod->name))
124 return 0;
125
Seth Jenningsb700e7f2014-12-16 11:58:19 -0600126 args->addr = addr;
127 args->count++;
128
Chris J Argesb2b018e2015-12-01 20:40:54 -0600129 /*
130 * Finish the search when the symbol is found for the desired position
131 * or the position is not defined for a non-unique symbol.
132 */
133 if ((args->pos && (args->count == args->pos)) ||
134 (!args->pos && (args->count > 1)))
135 return 1;
136
Seth Jenningsb700e7f2014-12-16 11:58:19 -0600137 return 0;
138}
139
140static int klp_find_object_symbol(const char *objname, const char *name,
Chris J Argesb2b018e2015-12-01 20:40:54 -0600141 unsigned long sympos, unsigned long *addr)
Seth Jenningsb700e7f2014-12-16 11:58:19 -0600142{
143 struct klp_find_arg args = {
144 .objname = objname,
145 .name = name,
146 .addr = 0,
Chris J Argesb2b018e2015-12-01 20:40:54 -0600147 .count = 0,
148 .pos = sympos,
Seth Jenningsb700e7f2014-12-16 11:58:19 -0600149 };
150
Miroslav Benes9a1bd632015-06-01 17:48:37 +0200151 mutex_lock(&module_mutex);
Seth Jenningsb700e7f2014-12-16 11:58:19 -0600152 kallsyms_on_each_symbol(klp_find_callback, &args);
Miroslav Benes9a1bd632015-06-01 17:48:37 +0200153 mutex_unlock(&module_mutex);
Seth Jenningsb700e7f2014-12-16 11:58:19 -0600154
Chris J Argesb2b018e2015-12-01 20:40:54 -0600155 /*
156 * Ensure an address was found. If sympos is 0, ensure symbol is unique;
157 * otherwise ensure the symbol position count matches sympos.
158 */
159 if (args.addr == 0)
Seth Jenningsb700e7f2014-12-16 11:58:19 -0600160 pr_err("symbol '%s' not found in symbol table\n", name);
Chris J Argesb2b018e2015-12-01 20:40:54 -0600161 else if (args.count > 1 && sympos == 0) {
Petr Mladekf995b5f2016-03-09 15:20:59 +0100162 pr_err("unresolvable ambiguity for symbol '%s' in object '%s'\n",
163 name, objname);
Chris J Argesb2b018e2015-12-01 20:40:54 -0600164 } else if (sympos != args.count && sympos > 0) {
165 pr_err("symbol position %lu for symbol '%s' in object '%s' not found\n",
166 sympos, name, objname ? objname : "vmlinux");
167 } else {
Seth Jenningsb700e7f2014-12-16 11:58:19 -0600168 *addr = args.addr;
169 return 0;
170 }
171
172 *addr = 0;
173 return -EINVAL;
174}
175
Jessica Yu425595a2016-03-22 20:03:18 -0400176static int klp_resolve_symbols(Elf_Shdr *relasec, struct module *pmod)
Seth Jenningsb700e7f2014-12-16 11:58:19 -0600177{
Jessica Yu425595a2016-03-22 20:03:18 -0400178 int i, cnt, vmlinux, ret;
179 char objname[MODULE_NAME_LEN];
180 char symname[KSYM_NAME_LEN];
181 char *strtab = pmod->core_kallsyms.strtab;
182 Elf_Rela *relas;
183 Elf_Sym *sym;
184 unsigned long sympos, addr;
Seth Jenningsb700e7f2014-12-16 11:58:19 -0600185
Chris J Argesb2b018e2015-12-01 20:40:54 -0600186 /*
Jessica Yu425595a2016-03-22 20:03:18 -0400187 * Since the field widths for objname and symname in the sscanf()
188 * call are hard-coded and correspond to MODULE_NAME_LEN and
189 * KSYM_NAME_LEN respectively, we must make sure that MODULE_NAME_LEN
190 * and KSYM_NAME_LEN have the values we expect them to have.
191 *
192 * Because the value of MODULE_NAME_LEN can differ among architectures,
193 * we use the smallest/strictest upper bound possible (56, based on
194 * the current definition of MODULE_NAME_LEN) to prevent overflows.
Chris J Argesb2b018e2015-12-01 20:40:54 -0600195 */
Jessica Yu425595a2016-03-22 20:03:18 -0400196 BUILD_BUG_ON(MODULE_NAME_LEN < 56 || KSYM_NAME_LEN != 128);
197
198 relas = (Elf_Rela *) relasec->sh_addr;
199 /* For each rela in this klp relocation section */
200 for (i = 0; i < relasec->sh_size / sizeof(Elf_Rela); i++) {
201 sym = pmod->core_kallsyms.symtab + ELF_R_SYM(relas[i].r_info);
202 if (sym->st_shndx != SHN_LIVEPATCH) {
203 pr_err("symbol %s is not marked as a livepatch symbol",
204 strtab + sym->st_name);
205 return -EINVAL;
206 }
207
208 /* Format: .klp.sym.objname.symname,sympos */
209 cnt = sscanf(strtab + sym->st_name,
210 ".klp.sym.%55[^.].%127[^,],%lu",
211 objname, symname, &sympos);
212 if (cnt != 3) {
213 pr_err("symbol %s has an incorrectly formatted name",
214 strtab + sym->st_name);
215 return -EINVAL;
216 }
217
218 /* klp_find_object_symbol() treats a NULL objname as vmlinux */
219 vmlinux = !strcmp(objname, "vmlinux");
220 ret = klp_find_object_symbol(vmlinux ? NULL : objname,
221 symname, sympos, &addr);
222 if (ret)
223 return ret;
224
225 sym->st_value = addr;
226 }
227
228 return 0;
Seth Jenningsb700e7f2014-12-16 11:58:19 -0600229}
230
231static int klp_write_object_relocations(struct module *pmod,
232 struct klp_object *obj)
233{
Jessica Yu425595a2016-03-22 20:03:18 -0400234 int i, cnt, ret = 0;
235 const char *objname, *secname;
236 char sec_objname[MODULE_NAME_LEN];
237 Elf_Shdr *sec;
Seth Jenningsb700e7f2014-12-16 11:58:19 -0600238
239 if (WARN_ON(!klp_is_object_loaded(obj)))
240 return -EINVAL;
241
Jessica Yu425595a2016-03-22 20:03:18 -0400242 objname = klp_is_module(obj) ? obj->name : "vmlinux";
Seth Jenningsb700e7f2014-12-16 11:58:19 -0600243
Jessica Yu425595a2016-03-22 20:03:18 -0400244 /* For each klp relocation section */
245 for (i = 1; i < pmod->klp_info->hdr.e_shnum; i++) {
246 sec = pmod->klp_info->sechdrs + i;
247 secname = pmod->klp_info->secstrings + sec->sh_name;
248 if (!(sec->sh_flags & SHF_RELA_LIVEPATCH))
249 continue;
Josh Poimboeufb56b36e2015-12-03 16:33:26 -0600250
Jessica Yu425595a2016-03-22 20:03:18 -0400251 /*
252 * Format: .klp.rela.sec_objname.section_name
253 * See comment in klp_resolve_symbols() for an explanation
254 * of the selected field width value.
255 */
256 cnt = sscanf(secname, ".klp.rela.%55[^.]", sec_objname);
257 if (cnt != 1) {
258 pr_err("section %s has an incorrectly formatted name",
259 secname);
260 ret = -EINVAL;
261 break;
Seth Jenningsb700e7f2014-12-16 11:58:19 -0600262 }
Jessica Yu425595a2016-03-22 20:03:18 -0400263
264 if (strcmp(objname, sec_objname))
265 continue;
266
267 ret = klp_resolve_symbols(sec, pmod);
268 if (ret)
269 break;
270
271 ret = apply_relocate_add(pmod->klp_info->sechdrs,
272 pmod->core_kallsyms.strtab,
273 pmod->klp_info->symndx, i, pmod);
274 if (ret)
275 break;
Seth Jenningsb700e7f2014-12-16 11:58:19 -0600276 }
277
Josh Poimboeufb56b36e2015-12-03 16:33:26 -0600278 return ret;
Seth Jenningsb700e7f2014-12-16 11:58:19 -0600279}
280
Seth Jenningsb700e7f2014-12-16 11:58:19 -0600281static int __klp_disable_patch(struct klp_patch *patch)
282{
Josh Poimboeufd83a7cb2017-02-13 19:42:40 -0600283 if (klp_transition_patch)
284 return -EBUSY;
Seth Jenningsb700e7f2014-12-16 11:58:19 -0600285
Josh Poimboeuf83a90bb2015-01-20 09:26:18 -0600286 /* enforce stacking: only the last enabled patch can be disabled */
287 if (!list_is_last(&patch->list, &klp_patches) &&
Josh Poimboeuf0dade9f2017-02-13 19:42:35 -0600288 list_next_entry(patch, list)->enabled)
Josh Poimboeuf83a90bb2015-01-20 09:26:18 -0600289 return -EBUSY;
290
Josh Poimboeufd83a7cb2017-02-13 19:42:40 -0600291 klp_init_transition(patch, KLP_UNPATCHED);
Seth Jenningsb700e7f2014-12-16 11:58:19 -0600292
Josh Poimboeufd83a7cb2017-02-13 19:42:40 -0600293 /*
294 * Enforce the order of the func->transition writes in
295 * klp_init_transition() and the TIF_PATCH_PENDING writes in
296 * klp_start_transition(). In the rare case where klp_ftrace_handler()
297 * is called shortly after klp_update_patch_state() switches the task,
298 * this ensures the handler sees that func->transition is set.
299 */
300 smp_wmb();
Seth Jenningsb700e7f2014-12-16 11:58:19 -0600301
Josh Poimboeufd83a7cb2017-02-13 19:42:40 -0600302 klp_start_transition();
303 klp_try_complete_transition();
Josh Poimboeuf0dade9f2017-02-13 19:42:35 -0600304 patch->enabled = false;
Seth Jenningsb700e7f2014-12-16 11:58:19 -0600305
306 return 0;
307}
308
309/**
310 * klp_disable_patch() - disables a registered patch
311 * @patch: The registered, enabled patch to be disabled
312 *
313 * Unregisters the patched functions from ftrace.
314 *
315 * Return: 0 on success, otherwise error
316 */
317int klp_disable_patch(struct klp_patch *patch)
318{
319 int ret;
320
321 mutex_lock(&klp_mutex);
322
323 if (!klp_is_patch_registered(patch)) {
324 ret = -EINVAL;
325 goto err;
326 }
327
Josh Poimboeuf0dade9f2017-02-13 19:42:35 -0600328 if (!patch->enabled) {
Seth Jenningsb700e7f2014-12-16 11:58:19 -0600329 ret = -EINVAL;
330 goto err;
331 }
332
333 ret = __klp_disable_patch(patch);
334
335err:
336 mutex_unlock(&klp_mutex);
337 return ret;
338}
339EXPORT_SYMBOL_GPL(klp_disable_patch);
340
341static int __klp_enable_patch(struct klp_patch *patch)
342{
343 struct klp_object *obj;
344 int ret;
345
Josh Poimboeufd83a7cb2017-02-13 19:42:40 -0600346 if (klp_transition_patch)
347 return -EBUSY;
348
Josh Poimboeuf0dade9f2017-02-13 19:42:35 -0600349 if (WARN_ON(patch->enabled))
Seth Jenningsb700e7f2014-12-16 11:58:19 -0600350 return -EINVAL;
351
Josh Poimboeuf83a90bb2015-01-20 09:26:18 -0600352 /* enforce stacking: only the first disabled patch can be enabled */
353 if (patch->list.prev != &klp_patches &&
Josh Poimboeuf0dade9f2017-02-13 19:42:35 -0600354 !list_prev_entry(patch, list)->enabled)
Josh Poimboeuf83a90bb2015-01-20 09:26:18 -0600355 return -EBUSY;
356
Seth Jenningsb700e7f2014-12-16 11:58:19 -0600357 pr_notice("enabling patch '%s'\n", patch->mod->name);
358
Josh Poimboeufd83a7cb2017-02-13 19:42:40 -0600359 klp_init_transition(patch, KLP_PATCHED);
360
361 /*
362 * Enforce the order of the func->transition writes in
363 * klp_init_transition() and the ops->func_stack writes in
364 * klp_patch_object(), so that klp_ftrace_handler() will see the
365 * func->transition updates before the handler is registered and the
366 * new funcs become visible to the handler.
367 */
368 smp_wmb();
369
Jiri Slaby8cdd0432015-05-19 12:01:19 +0200370 klp_for_each_object(patch, obj) {
Seth Jenningsb700e7f2014-12-16 11:58:19 -0600371 if (!klp_is_object_loaded(obj))
372 continue;
373
Josh Poimboeuf0dade9f2017-02-13 19:42:35 -0600374 ret = klp_patch_object(obj);
Josh Poimboeufd83a7cb2017-02-13 19:42:40 -0600375 if (ret) {
376 pr_warn("failed to enable patch '%s'\n",
377 patch->mod->name);
378
379 klp_cancel_transition();
380 return ret;
381 }
Seth Jenningsb700e7f2014-12-16 11:58:19 -0600382 }
383
Josh Poimboeufd83a7cb2017-02-13 19:42:40 -0600384 klp_start_transition();
385 klp_try_complete_transition();
Josh Poimboeuf0dade9f2017-02-13 19:42:35 -0600386 patch->enabled = true;
Seth Jenningsb700e7f2014-12-16 11:58:19 -0600387
388 return 0;
Seth Jenningsb700e7f2014-12-16 11:58:19 -0600389}
390
391/**
392 * klp_enable_patch() - enables a registered patch
393 * @patch: The registered, disabled patch to be enabled
394 *
395 * Performs the needed symbol lookups and code relocations,
396 * then registers the patched functions with ftrace.
397 *
398 * Return: 0 on success, otherwise error
399 */
400int klp_enable_patch(struct klp_patch *patch)
401{
402 int ret;
403
404 mutex_lock(&klp_mutex);
405
406 if (!klp_is_patch_registered(patch)) {
407 ret = -EINVAL;
408 goto err;
409 }
410
411 ret = __klp_enable_patch(patch);
412
413err:
414 mutex_unlock(&klp_mutex);
415 return ret;
416}
417EXPORT_SYMBOL_GPL(klp_enable_patch);
418
419/*
420 * Sysfs Interface
421 *
422 * /sys/kernel/livepatch
423 * /sys/kernel/livepatch/<patch>
424 * /sys/kernel/livepatch/<patch>/enabled
Josh Poimboeufd83a7cb2017-02-13 19:42:40 -0600425 * /sys/kernel/livepatch/<patch>/transition
Seth Jenningsb700e7f2014-12-16 11:58:19 -0600426 * /sys/kernel/livepatch/<patch>/<object>
Chris J Arges444f9e92015-12-01 20:40:56 -0600427 * /sys/kernel/livepatch/<patch>/<object>/<function,sympos>
Seth Jenningsb700e7f2014-12-16 11:58:19 -0600428 */
429
430static ssize_t enabled_store(struct kobject *kobj, struct kobj_attribute *attr,
431 const char *buf, size_t count)
432{
433 struct klp_patch *patch;
434 int ret;
Josh Poimboeuf68ae4b22017-02-13 19:42:38 -0600435 bool enabled;
Seth Jenningsb700e7f2014-12-16 11:58:19 -0600436
Josh Poimboeuf68ae4b22017-02-13 19:42:38 -0600437 ret = kstrtobool(buf, &enabled);
Seth Jenningsb700e7f2014-12-16 11:58:19 -0600438 if (ret)
Josh Poimboeuf68ae4b22017-02-13 19:42:38 -0600439 return ret;
Seth Jenningsb700e7f2014-12-16 11:58:19 -0600440
441 patch = container_of(kobj, struct klp_patch, kobj);
442
443 mutex_lock(&klp_mutex);
444
Josh Poimboeuf68ae4b22017-02-13 19:42:38 -0600445 if (patch->enabled == enabled) {
Seth Jenningsb700e7f2014-12-16 11:58:19 -0600446 /* already in requested state */
447 ret = -EINVAL;
448 goto err;
449 }
450
Josh Poimboeufd83a7cb2017-02-13 19:42:40 -0600451 if (patch == klp_transition_patch) {
452 klp_reverse_transition();
453 } else if (enabled) {
Seth Jenningsb700e7f2014-12-16 11:58:19 -0600454 ret = __klp_enable_patch(patch);
455 if (ret)
456 goto err;
457 } else {
458 ret = __klp_disable_patch(patch);
459 if (ret)
460 goto err;
461 }
462
463 mutex_unlock(&klp_mutex);
464
465 return count;
466
467err:
468 mutex_unlock(&klp_mutex);
469 return ret;
470}
471
472static ssize_t enabled_show(struct kobject *kobj,
473 struct kobj_attribute *attr, char *buf)
474{
475 struct klp_patch *patch;
476
477 patch = container_of(kobj, struct klp_patch, kobj);
Josh Poimboeuf0dade9f2017-02-13 19:42:35 -0600478 return snprintf(buf, PAGE_SIZE-1, "%d\n", patch->enabled);
Seth Jenningsb700e7f2014-12-16 11:58:19 -0600479}
480
Josh Poimboeufd83a7cb2017-02-13 19:42:40 -0600481static ssize_t transition_show(struct kobject *kobj,
482 struct kobj_attribute *attr, char *buf)
483{
484 struct klp_patch *patch;
485
486 patch = container_of(kobj, struct klp_patch, kobj);
487 return snprintf(buf, PAGE_SIZE-1, "%d\n",
488 patch == klp_transition_patch);
489}
490
Seth Jenningsb700e7f2014-12-16 11:58:19 -0600491static struct kobj_attribute enabled_kobj_attr = __ATTR_RW(enabled);
Josh Poimboeufd83a7cb2017-02-13 19:42:40 -0600492static struct kobj_attribute transition_kobj_attr = __ATTR_RO(transition);
Seth Jenningsb700e7f2014-12-16 11:58:19 -0600493static struct attribute *klp_patch_attrs[] = {
494 &enabled_kobj_attr.attr,
Josh Poimboeufd83a7cb2017-02-13 19:42:40 -0600495 &transition_kobj_attr.attr,
Seth Jenningsb700e7f2014-12-16 11:58:19 -0600496 NULL
497};
498
499static void klp_kobj_release_patch(struct kobject *kobj)
500{
501 /*
502 * Once we have a consistency model we'll need to module_put() the
503 * patch module here. See klp_register_patch() for more details.
504 */
505}
506
507static struct kobj_type klp_ktype_patch = {
508 .release = klp_kobj_release_patch,
509 .sysfs_ops = &kobj_sysfs_ops,
510 .default_attrs = klp_patch_attrs,
511};
512
Miroslav Benescad706d2015-05-19 12:01:18 +0200513static void klp_kobj_release_object(struct kobject *kobj)
514{
515}
516
517static struct kobj_type klp_ktype_object = {
518 .release = klp_kobj_release_object,
519 .sysfs_ops = &kobj_sysfs_ops,
520};
521
Seth Jenningsb700e7f2014-12-16 11:58:19 -0600522static void klp_kobj_release_func(struct kobject *kobj)
523{
Seth Jenningsb700e7f2014-12-16 11:58:19 -0600524}
525
526static struct kobj_type klp_ktype_func = {
527 .release = klp_kobj_release_func,
528 .sysfs_ops = &kobj_sysfs_ops,
529};
530
531/*
532 * Free all functions' kobjects in the array up to some limit. When limit is
533 * NULL, all kobjects are freed.
534 */
535static void klp_free_funcs_limited(struct klp_object *obj,
536 struct klp_func *limit)
537{
538 struct klp_func *func;
539
540 for (func = obj->funcs; func->old_name && func != limit; func++)
541 kobject_put(&func->kobj);
542}
543
544/* Clean up when a patched object is unloaded */
545static void klp_free_object_loaded(struct klp_object *obj)
546{
547 struct klp_func *func;
548
549 obj->mod = NULL;
550
Jiri Slaby8cdd0432015-05-19 12:01:19 +0200551 klp_for_each_func(obj, func)
Seth Jenningsb700e7f2014-12-16 11:58:19 -0600552 func->old_addr = 0;
553}
554
555/*
556 * Free all objects' kobjects in the array up to some limit. When limit is
557 * NULL, all kobjects are freed.
558 */
559static void klp_free_objects_limited(struct klp_patch *patch,
560 struct klp_object *limit)
561{
562 struct klp_object *obj;
563
564 for (obj = patch->objs; obj->funcs && obj != limit; obj++) {
565 klp_free_funcs_limited(obj, NULL);
Miroslav Benescad706d2015-05-19 12:01:18 +0200566 kobject_put(&obj->kobj);
Seth Jenningsb700e7f2014-12-16 11:58:19 -0600567 }
568}
569
570static void klp_free_patch(struct klp_patch *patch)
571{
572 klp_free_objects_limited(patch, NULL);
573 if (!list_empty(&patch->list))
574 list_del(&patch->list);
575 kobject_put(&patch->kobj);
576}
577
578static int klp_init_func(struct klp_object *obj, struct klp_func *func)
579{
Miroslav Benesf09d9082016-04-28 16:34:08 +0200580 if (!func->old_name || !func->new_func)
581 return -EINVAL;
582
Josh Poimboeuf3c33f5b2015-01-20 09:26:19 -0600583 INIT_LIST_HEAD(&func->stack_node);
Josh Poimboeuf0dade9f2017-02-13 19:42:35 -0600584 func->patched = false;
Josh Poimboeufd83a7cb2017-02-13 19:42:40 -0600585 func->transition = false;
Seth Jenningsb700e7f2014-12-16 11:58:19 -0600586
Chris J Arges444f9e92015-12-01 20:40:56 -0600587 /* The format for the sysfs directory is <function,sympos> where sympos
588 * is the nth occurrence of this symbol in kallsyms for the patched
589 * object. If the user selects 0 for old_sympos, then 1 will be used
590 * since a unique symbol will be the first occurrence.
591 */
Josh Poimboeuf3c33f5b2015-01-20 09:26:19 -0600592 return kobject_init_and_add(&func->kobj, &klp_ktype_func,
Chris J Arges444f9e92015-12-01 20:40:56 -0600593 &obj->kobj, "%s,%lu", func->old_name,
594 func->old_sympos ? func->old_sympos : 1);
Seth Jenningsb700e7f2014-12-16 11:58:19 -0600595}
596
Jessica Yu255e7322016-08-17 20:58:28 -0400597/* Arches may override this to finish any remaining arch-specific tasks */
598void __weak arch_klp_init_object_loaded(struct klp_patch *patch,
599 struct klp_object *obj)
600{
601}
602
Seth Jenningsb700e7f2014-12-16 11:58:19 -0600603/* parts of the initialization that is done only when the object is loaded */
604static int klp_init_object_loaded(struct klp_patch *patch,
605 struct klp_object *obj)
606{
607 struct klp_func *func;
608 int ret;
609
Jessica Yu255e7322016-08-17 20:58:28 -0400610 module_disable_ro(patch->mod);
Jessica Yu425595a2016-03-22 20:03:18 -0400611 ret = klp_write_object_relocations(patch->mod, obj);
Jessica Yu255e7322016-08-17 20:58:28 -0400612 if (ret) {
613 module_enable_ro(patch->mod, true);
Jessica Yu425595a2016-03-22 20:03:18 -0400614 return ret;
Jessica Yu255e7322016-08-17 20:58:28 -0400615 }
616
617 arch_klp_init_object_loaded(patch, obj);
618 module_enable_ro(patch->mod, true);
Seth Jenningsb700e7f2014-12-16 11:58:19 -0600619
Jiri Slaby8cdd0432015-05-19 12:01:19 +0200620 klp_for_each_func(obj, func) {
Chris J Argesb2b018e2015-12-01 20:40:54 -0600621 ret = klp_find_object_symbol(obj->name, func->old_name,
622 func->old_sympos,
623 &func->old_addr);
Seth Jenningsb700e7f2014-12-16 11:58:19 -0600624 if (ret)
625 return ret;
Josh Poimboeuff5e547f2017-02-13 19:42:39 -0600626
627 ret = kallsyms_lookup_size_offset(func->old_addr,
628 &func->old_size, NULL);
629 if (!ret) {
630 pr_err("kallsyms size lookup failed for '%s'\n",
631 func->old_name);
632 return -ENOENT;
633 }
634
635 ret = kallsyms_lookup_size_offset((unsigned long)func->new_func,
636 &func->new_size, NULL);
637 if (!ret) {
638 pr_err("kallsyms size lookup failed for '%s' replacement\n",
639 func->old_name);
640 return -ENOENT;
641 }
Seth Jenningsb700e7f2014-12-16 11:58:19 -0600642 }
643
644 return 0;
645}
646
647static int klp_init_object(struct klp_patch *patch, struct klp_object *obj)
648{
649 struct klp_func *func;
650 int ret;
651 const char *name;
652
653 if (!obj->funcs)
654 return -EINVAL;
655
Josh Poimboeuf0dade9f2017-02-13 19:42:35 -0600656 obj->patched = false;
Petr Mladek8cb2c2d2015-03-12 12:55:13 +0100657 obj->mod = NULL;
Seth Jenningsb700e7f2014-12-16 11:58:19 -0600658
659 klp_find_object_module(obj);
660
661 name = klp_is_module(obj) ? obj->name : "vmlinux";
Miroslav Benescad706d2015-05-19 12:01:18 +0200662 ret = kobject_init_and_add(&obj->kobj, &klp_ktype_object,
663 &patch->kobj, "%s", name);
664 if (ret)
665 return ret;
Seth Jenningsb700e7f2014-12-16 11:58:19 -0600666
Jiri Slaby8cdd0432015-05-19 12:01:19 +0200667 klp_for_each_func(obj, func) {
Seth Jenningsb700e7f2014-12-16 11:58:19 -0600668 ret = klp_init_func(obj, func);
669 if (ret)
670 goto free;
671 }
672
673 if (klp_is_object_loaded(obj)) {
674 ret = klp_init_object_loaded(patch, obj);
675 if (ret)
676 goto free;
677 }
678
679 return 0;
680
681free:
682 klp_free_funcs_limited(obj, func);
Miroslav Benescad706d2015-05-19 12:01:18 +0200683 kobject_put(&obj->kobj);
Seth Jenningsb700e7f2014-12-16 11:58:19 -0600684 return ret;
685}
686
687static int klp_init_patch(struct klp_patch *patch)
688{
689 struct klp_object *obj;
690 int ret;
691
692 if (!patch->objs)
693 return -EINVAL;
694
695 mutex_lock(&klp_mutex);
696
Josh Poimboeuf0dade9f2017-02-13 19:42:35 -0600697 patch->enabled = false;
Seth Jenningsb700e7f2014-12-16 11:58:19 -0600698
699 ret = kobject_init_and_add(&patch->kobj, &klp_ktype_patch,
Jiri Kosinae0b561e2015-02-15 10:03:20 +0100700 klp_root_kobj, "%s", patch->mod->name);
Seth Jenningsb700e7f2014-12-16 11:58:19 -0600701 if (ret)
702 goto unlock;
703
Jiri Slaby8cdd0432015-05-19 12:01:19 +0200704 klp_for_each_object(patch, obj) {
Seth Jenningsb700e7f2014-12-16 11:58:19 -0600705 ret = klp_init_object(patch, obj);
706 if (ret)
707 goto free;
708 }
709
Josh Poimboeuf99590ba2015-01-09 14:03:04 -0600710 list_add_tail(&patch->list, &klp_patches);
Seth Jenningsb700e7f2014-12-16 11:58:19 -0600711
712 mutex_unlock(&klp_mutex);
713
714 return 0;
715
716free:
717 klp_free_objects_limited(patch, obj);
718 kobject_put(&patch->kobj);
719unlock:
720 mutex_unlock(&klp_mutex);
721 return ret;
722}
723
724/**
725 * klp_unregister_patch() - unregisters a patch
726 * @patch: Disabled patch to be unregistered
727 *
728 * Frees the data structures and removes the sysfs interface.
729 *
730 * Return: 0 on success, otherwise error
731 */
732int klp_unregister_patch(struct klp_patch *patch)
733{
734 int ret = 0;
735
736 mutex_lock(&klp_mutex);
737
738 if (!klp_is_patch_registered(patch)) {
739 ret = -EINVAL;
740 goto out;
741 }
742
Josh Poimboeuf0dade9f2017-02-13 19:42:35 -0600743 if (patch->enabled) {
Seth Jenningsb700e7f2014-12-16 11:58:19 -0600744 ret = -EBUSY;
745 goto out;
746 }
747
748 klp_free_patch(patch);
749
750out:
751 mutex_unlock(&klp_mutex);
752 return ret;
753}
754EXPORT_SYMBOL_GPL(klp_unregister_patch);
755
756/**
757 * klp_register_patch() - registers a patch
758 * @patch: Patch to be registered
759 *
760 * Initializes the data structure associated with the patch and
761 * creates the sysfs interface.
762 *
763 * Return: 0 on success, otherwise error
764 */
765int klp_register_patch(struct klp_patch *patch)
766{
767 int ret;
768
Seth Jenningsb700e7f2014-12-16 11:58:19 -0600769 if (!patch || !patch->mod)
770 return -EINVAL;
771
Jessica Yu425595a2016-03-22 20:03:18 -0400772 if (!is_livepatch_module(patch->mod)) {
773 pr_err("module %s is not marked as a livepatch module",
774 patch->mod->name);
775 return -EINVAL;
776 }
777
Seth Jenningsb700e7f2014-12-16 11:58:19 -0600778 if (!klp_initialized())
779 return -ENODEV;
780
Seth Jenningsb700e7f2014-12-16 11:58:19 -0600781 /*
Josh Poimboeufd83a7cb2017-02-13 19:42:40 -0600782 * Architectures without reliable stack traces have to set
783 * patch->immediate because there's currently no way to patch kthreads
784 * with the consistency model.
785 */
786 if (!klp_have_reliable_stack() && !patch->immediate) {
787 pr_err("This architecture doesn't have support for the livepatch consistency model.\n");
788 return -ENOSYS;
789 }
790
791 /*
Seth Jenningsb700e7f2014-12-16 11:58:19 -0600792 * A reference is taken on the patch module to prevent it from being
793 * unloaded. Right now, we don't allow patch modules to unload since
794 * there is currently no method to determine if a thread is still
795 * running in the patched code contained in the patch module once
796 * the ftrace registration is successful.
797 */
798 if (!try_module_get(patch->mod))
799 return -ENODEV;
800
801 ret = klp_init_patch(patch);
802 if (ret)
803 module_put(patch->mod);
804
805 return ret;
806}
807EXPORT_SYMBOL_GPL(klp_register_patch);
808
Jessica Yu7e545d62016-03-16 20:55:39 -0400809int klp_module_coming(struct module *mod)
Seth Jenningsb700e7f2014-12-16 11:58:19 -0600810{
Minfei Huang36e505c2015-05-15 10:22:48 +0800811 int ret;
Seth Jenningsb700e7f2014-12-16 11:58:19 -0600812 struct klp_patch *patch;
813 struct klp_object *obj;
814
Jessica Yu7e545d62016-03-16 20:55:39 -0400815 if (WARN_ON(mod->state != MODULE_STATE_COMING))
816 return -EINVAL;
Seth Jenningsb700e7f2014-12-16 11:58:19 -0600817
818 mutex_lock(&klp_mutex);
Petr Mladek8cb2c2d2015-03-12 12:55:13 +0100819 /*
Jessica Yu7e545d62016-03-16 20:55:39 -0400820 * Each module has to know that klp_module_coming()
821 * has been called. We never know what module will
822 * get patched by a new patch.
Petr Mladek8cb2c2d2015-03-12 12:55:13 +0100823 */
Jessica Yu7e545d62016-03-16 20:55:39 -0400824 mod->klp_alive = true;
Petr Mladek8cb2c2d2015-03-12 12:55:13 +0100825
Seth Jenningsb700e7f2014-12-16 11:58:19 -0600826 list_for_each_entry(patch, &klp_patches, list) {
Jiri Slaby8cdd0432015-05-19 12:01:19 +0200827 klp_for_each_object(patch, obj) {
Seth Jenningsb700e7f2014-12-16 11:58:19 -0600828 if (!klp_is_module(obj) || strcmp(obj->name, mod->name))
829 continue;
830
Jessica Yu7e545d62016-03-16 20:55:39 -0400831 obj->mod = mod;
832
833 ret = klp_init_object_loaded(patch, obj);
834 if (ret) {
835 pr_warn("failed to initialize patch '%s' for module '%s' (%d)\n",
836 patch->mod->name, obj->mod->name, ret);
837 goto err;
838 }
839
Josh Poimboeufd83a7cb2017-02-13 19:42:40 -0600840 /*
841 * Only patch the module if the patch is enabled or is
842 * in transition.
843 */
844 if (!patch->enabled && patch != klp_transition_patch)
Jessica Yu7e545d62016-03-16 20:55:39 -0400845 break;
846
847 pr_notice("applying patch '%s' to loading module '%s'\n",
848 patch->mod->name, obj->mod->name);
849
Josh Poimboeuf0dade9f2017-02-13 19:42:35 -0600850 ret = klp_patch_object(obj);
Jessica Yu7e545d62016-03-16 20:55:39 -0400851 if (ret) {
852 pr_warn("failed to apply patch '%s' to module '%s' (%d)\n",
853 patch->mod->name, obj->mod->name, ret);
854 goto err;
855 }
Seth Jenningsb700e7f2014-12-16 11:58:19 -0600856
857 break;
858 }
859 }
860
861 mutex_unlock(&klp_mutex);
862
863 return 0;
Jessica Yu7e545d62016-03-16 20:55:39 -0400864
865err:
866 /*
867 * If a patch is unsuccessfully applied, return
868 * error to the module loader.
869 */
870 pr_warn("patch '%s' failed for module '%s', refusing to load module '%s'\n",
871 patch->mod->name, obj->mod->name, obj->mod->name);
872 mod->klp_alive = false;
873 klp_free_object_loaded(obj);
874 mutex_unlock(&klp_mutex);
875
876 return ret;
Seth Jenningsb700e7f2014-12-16 11:58:19 -0600877}
878
Jessica Yu7e545d62016-03-16 20:55:39 -0400879void klp_module_going(struct module *mod)
880{
881 struct klp_patch *patch;
882 struct klp_object *obj;
883
884 if (WARN_ON(mod->state != MODULE_STATE_GOING &&
885 mod->state != MODULE_STATE_COMING))
886 return;
887
888 mutex_lock(&klp_mutex);
889 /*
890 * Each module has to know that klp_module_going()
891 * has been called. We never know what module will
892 * get patched by a new patch.
893 */
894 mod->klp_alive = false;
895
896 list_for_each_entry(patch, &klp_patches, list) {
897 klp_for_each_object(patch, obj) {
898 if (!klp_is_module(obj) || strcmp(obj->name, mod->name))
899 continue;
900
Josh Poimboeufd83a7cb2017-02-13 19:42:40 -0600901 /*
902 * Only unpatch the module if the patch is enabled or
903 * is in transition.
904 */
905 if (patch->enabled || patch == klp_transition_patch) {
Jessica Yu7e545d62016-03-16 20:55:39 -0400906 pr_notice("reverting patch '%s' on unloading module '%s'\n",
907 patch->mod->name, obj->mod->name);
Josh Poimboeuf0dade9f2017-02-13 19:42:35 -0600908 klp_unpatch_object(obj);
Jessica Yu7e545d62016-03-16 20:55:39 -0400909 }
910
911 klp_free_object_loaded(obj);
912 break;
913 }
914 }
915
916 mutex_unlock(&klp_mutex);
917}
Seth Jenningsb700e7f2014-12-16 11:58:19 -0600918
Minfei Huang26029d82015-05-22 22:26:29 +0800919static int __init klp_init(void)
Seth Jenningsb700e7f2014-12-16 11:58:19 -0600920{
921 int ret;
922
Jiri Kosinab9dfe0b2015-01-09 10:53:21 +0100923 ret = klp_check_compiler_support();
924 if (ret) {
925 pr_info("Your compiler is too old; turning off.\n");
926 return -EINVAL;
927 }
928
Seth Jenningsb700e7f2014-12-16 11:58:19 -0600929 klp_root_kobj = kobject_create_and_add("livepatch", kernel_kobj);
Jessica Yu7e545d62016-03-16 20:55:39 -0400930 if (!klp_root_kobj)
931 return -ENOMEM;
Seth Jenningsb700e7f2014-12-16 11:58:19 -0600932
933 return 0;
Seth Jenningsb700e7f2014-12-16 11:58:19 -0600934}
935
936module_init(klp_init);