?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Add support for ARM architecture.
[ksplice.git] / kmodsrc / ksplice.c
blob2ff26dd2097a6c68f9e2dd67ef67450a43247699
1 /* Copyright (C) 2007-2008 Jeffrey Brian Arnold <jbarnold@mit.edu>
2 * Copyright (C) 2008 Anders Kaseorg <andersk@mit.edu>,
3 * Tim Abbott <tabbott@mit.edu>
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License, version 2.
7 *
8 * This program is distributed in the hope that it will be useful,
9 * but WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 * GNU General Public License for more details.
12 *
13 * You should have received a copy of the GNU General Public License
14 * along with this program; if not, write to the Free Software
15 * Foundation, Inc., 51 Franklin Street - Fifth Floor, Boston, MA
16 * 02110-1301, USA.
17 */
18
19 #include <linux/module.h>
20 #ifdef CONFIG_DEBUG_FS
21 #include <linux/debugfs.h>
22 #endif /* CONFIG_DEBUG_FS */
23 #include <linux/errno.h>
24 #include <linux/kallsyms.h>
25 #include <linux/kobject.h>
26 #include <linux/kthread.h>
27 #include <linux/pagemap.h>
28 #include <linux/sched.h>
29 #include <linux/stop_machine.h>
30 #include <linux/sysfs.h>
31 #include <linux/time.h>
32 #include <linux/version.h>
33 #include <linux/vmalloc.h>
34 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,18)
35 #include <linux/uaccess.h>
36 #else /* LINUX_VERSION_CODE < */
37 /* linux/uaccess.h doesn't exist in kernels before 2.6.18 */
38 #include <asm/uaccess.h>
39 #endif /* LINUX_VERSION_CODE */
40 #ifdef KSPLICE_NEED_PARAINSTRUCTIONS
41 #include <asm/alternative.h>
42 #endif /* KSPLICE_NEED_PARAINSTRUCTIONS */
43 #ifdef KSPLICE_STANDALONE
44 #include "ksplice.h"
45 #else /* !KSPLICE_STANDALONE */
46 #include <linux/ksplice.h>
47 #endif /* KSPLICE_STANDALONE */
48
49 #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,19)
50 /* 6e21828743247270d09a86756a0c11702500dbfb was after 2.6.18 */
51 #define bool _Bool
52 #define false 0
53 #define true 1
54 #endif /* LINUX_VERSION_CODE */
55
56 #if BITS_PER_LONG == 32
57 #define ADDR "08lx"
58 #elif BITS_PER_LONG == 64
59 #define ADDR "016lx"
60 #endif /* BITS_PER_LONG */
61
62 enum ksplice_stage_enum {
63 PREPARING, APPLIED, REVERSED
64 };
65
66 #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,9)
67 /* 5d7b32de9935c65ca8285ac6ec2382afdbb5d479 was after 2.6.8 */
68 #define __bitwise__
69 #elif LINUX_VERSION_CODE < KERNEL_VERSION(2,6,15)
70 /* af4ca457eaf2d6682059c18463eb106e2ce58198 was after 2.6.14 */
71 #define __bitwise__ __bitwise
72 #endif
73
74 typedef int __bitwise__ abort_t;
75
76 #define OK ((__force abort_t) 0)
77 #define NO_MATCH ((__force abort_t) 1)
78 #define BAD_SYSTEM_MAP ((__force abort_t) 2)
79 #define CODE_BUSY ((__force abort_t) 3)
80 #define MODULE_BUSY ((__force abort_t) 4)
81 #define OUT_OF_MEMORY ((__force abort_t) 5)
82 #define FAILED_TO_FIND ((__force abort_t) 6)
83 #define ALREADY_REVERSED ((__force abort_t) 7)
84 #define MISSING_EXPORT ((__force abort_t) 8)
85 #define UNEXPECTED_RUNNING_TASK ((__force abort_t) 9)
86 #define UNEXPECTED ((__force abort_t) 10)
87
88 struct update_bundle {
89 const char *kid;
90 const char *name;
91 struct kobject kobj;
92 enum ksplice_stage_enum stage;
93 abort_t abort_cause;
94 int debug;
95 #ifdef CONFIG_DEBUG_FS
96 struct debugfs_blob_wrapper debug_blob;
97 struct dentry *debugfs_dentry;
98 #endif /* CONFIG_DEBUG_FS */
99 struct list_head packs;
100 struct list_head conflicts;
101 struct list_head list;
102 };
103
104 struct conflict {
105 const char *process_name;
106 pid_t pid;
107 struct list_head stack;
108 struct list_head list;
109 };
110
111 struct ksplice_frame {
112 unsigned long addr;
113 int has_conflict;
114 const char *label;
115 struct list_head list;
116 };
117
118 #if defined(CONFIG_DEBUG_FS) && LINUX_VERSION_CODE < KERNEL_VERSION(2,6,17)
119 /* Old kernels don't have debugfs_create_blob */
120 struct debugfs_blob_wrapper {
121 void *data;
122 unsigned long size;
123 };
124 #endif /* CONFIG_DEBUG_FS && LINUX_VERSION_CODE */
125
126 struct reloc_nameval {
127 struct list_head list;
128 const char *label;
129 unsigned long val;
130 enum { NOVAL, TEMP, VAL } status;
131 };
132
133 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,20)
134 static int virtual_address_mapped(unsigned long addr)
135 {
136 char retval;
137 return probe_kernel_address(addr, retval) != -EFAULT;
138 }
139 #else /* LINUX_VERSION_CODE < */
140 /* 2fff0a48416af891dce38fd425246e337831e0bb was after 2.6.19 */
141 static int virtual_address_mapped(unsigned long addr);
142 #endif /* LINUX_VERSION_CODE */
143
144 static struct reloc_nameval *find_nameval(struct module_pack *pack,
145 const char *label);
146 static abort_t create_nameval(struct module_pack *pack, const char *label,
147 unsigned long val, int status);
148 static abort_t lookup_reloc(struct module_pack *pack, unsigned long addr,
149 const struct ksplice_reloc **relocp);
150 static abort_t handle_reloc(struct module_pack *pack,
151 const struct ksplice_reloc *r,
152 unsigned long run_addr, int rerun);
153
154 struct safety_record {
155 struct list_head list;
156 const char *label;
157 unsigned long addr;
158 unsigned int size;
159 bool first_byte_safe;
160 };
161
162 struct candidate_val {
163 struct list_head list;
164 unsigned long val;
165 };
166
167 #define singular(list) (!list_empty(list) && (list)->next->next == (list))
168
169 #ifdef CONFIG_DEBUG_FS
170 static abort_t init_debug_buf(struct update_bundle *bundle);
171 static void clear_debug_buf(struct update_bundle *bundle);
172 static int __attribute__((format(printf, 2, 3)))
173 __ksdebug(struct update_bundle *bundle, const char *fmt, ...);
174 #else /* !CONFIG_DEBUG_FS */
175 static inline abort_t init_debug_buf(struct update_bundle *bundle)
176 {
177 return OK;
178 }
179
180 static inline void clear_debug_buf(struct update_bundle *bundle)
181 {
182 return;
183 }
184
185 #define __ksdebug(bundle, fmt, ...) printk(fmt, ## __VA_ARGS__)
186 #endif /* CONFIG_DEBUG_FS */
187
188 #define _ksdebug(bundle, level, fmt, ...) \
189 do { \
190 if ((bundle)->debug >= (level)) \
191 __ksdebug(bundle, fmt, ## __VA_ARGS__); \
192 } while (0)
193 #define ksdebug(pack, level, fmt, ...) \
194 do { _ksdebug((pack)->bundle, level, fmt, ## __VA_ARGS__); } while (0)
195 #define failed_to_find(pack, sym_name) \
196 ksdebug(pack, 0, KERN_ERR "ksplice: Failed to find symbol %s at " \
197 "%s:%d\n", sym_name, __FILE__, __LINE__)
198
199 static inline void print_abort(struct module_pack *pack, const char *str)
200 {
201 ksdebug(pack, 0, KERN_ERR "ksplice: Aborted. (%s)\n", str);
202 }
203
204 static LIST_HEAD(update_bundles);
205 #ifdef KSPLICE_STANDALONE
206 #if defined(CONFIG_KSPLICE) || defined(CONFIG_KSPLICE_MODULE)
207 extern struct list_head ksplice_module_list;
208 #else /* !CONFIG_KSPLICE */
209 LIST_HEAD(ksplice_module_list);
210 #endif /* CONFIG_KSPLICE */
211 #else /* !KSPLICE_STANDALONE */
212 LIST_HEAD(ksplice_module_list);
213 EXPORT_SYMBOL_GPL(ksplice_module_list);
214 #endif /* KSPLICE_STANDALONE */
215
216 #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,9)
217 /* Old kernels do not have kcalloc
218 * e629946abd0bb8266e9c3d0fd1bff2ef8dec5443 was after 2.6.8
219 */
220 static inline void *kcalloc(size_t n, size_t size, typeof(GFP_KERNEL) flags)
221 {
222 char *mem;
223 if (n != 0 && size > ULONG_MAX / n)
224 return NULL;
225 mem = kmalloc(n * size, flags);
226 if (mem)
227 memset(mem, 0, n * size);
228 return mem;
229 }
230 #endif /* LINUX_VERSION_CODE */
231
232 #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,13)
233 /* Old kernels do not have kstrdup
234 * 543537bd922692bc978e2e356fcd8bfc9c2ee7d5 was 2.6.13-rc4
235 */
236 static char *kstrdup(const char *s, typeof(GFP_KERNEL) gfp)
237 {
238 size_t len;
239 char *buf;
240
241 if (!s)
242 return NULL;
243
244 len = strlen(s) + 1;
245 buf = kmalloc(len, gfp);
246 if (buf)
247 memcpy(buf, s, len);
248 return buf;
249 }
250 #endif /* LINUX_VERSION_CODE */
251
252 #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,17)
253 /* Old kernels use semaphore instead of mutex
254 * 97d1f15b7ef52c1e9c28dc48b454024bb53a5fd2 was after 2.6.16
255 */
256 #define mutex semaphore
257 #define mutex_lock down
258 #define mutex_unlock up
259 #endif /* LINUX_VERSION_CODE */
260
261 #ifndef task_thread_info
262 #define task_thread_info(task) (task)->thread_info
263 #endif /* !task_thread_info */
264
265 #ifdef KSPLICE_STANDALONE
266
267 static int bootstrapped = 0;
268
269 #ifdef CONFIG_KALLSYMS
270 extern unsigned long kallsyms_addresses[], kallsyms_num_syms;
271 extern u8 kallsyms_names[];
272 #endif /* CONFIG_KALLSYMS */
273
274 /* defined by ksplice-create */
275 extern const struct ksplice_reloc ksplice_init_relocs[],
276 ksplice_init_relocs_end[];
277
278 /* Obtained via System.map */
279 extern struct list_head modules;
280 extern struct mutex module_mutex;
281 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,18) && defined(CONFIG_UNUSED_SYMBOLS)
282 /* f71d20e961474dde77e6558396efb93d6ac80a4b was after 2.6.17 */
283 #define KSPLICE_KSYMTAB_UNUSED_SUPPORT 1
284 #endif /* LINUX_VERSION_CODE */
285 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,17)
286 /* 9f28bb7e1d0188a993403ab39b774785892805e1 was after 2.6.16 */
287 #define KSPLICE_KSYMTAB_FUTURE_SUPPORT 1
288 #endif /* LINUX_VERSION_CODE */
289 extern const struct kernel_symbol __start___ksymtab[];
290 extern const struct kernel_symbol __stop___ksymtab[];
291 extern const unsigned long __start___kcrctab[];
292 extern const struct kernel_symbol __start___ksymtab_gpl[];
293 extern const struct kernel_symbol __stop___ksymtab_gpl[];
294 extern const unsigned long __start___kcrctab_gpl[];
295 #ifdef KSPLICE_KSYMTAB_UNUSED_SUPPORT
296 extern const struct kernel_symbol __start___ksymtab_unused[];
297 extern const struct kernel_symbol __stop___ksymtab_unused[];
298 extern const unsigned long __start___kcrctab_unused[];
299 extern const struct kernel_symbol __start___ksymtab_unused_gpl[];
300 extern const struct kernel_symbol __stop___ksymtab_unused_gpl[];
301 extern const unsigned long __start___kcrctab_unused_gpl[];
302 #endif /* KSPLICE_KSYMTAB_UNUSED_SUPPORT */
303 #ifdef KSPLICE_KSYMTAB_FUTURE_SUPPORT
304 extern const struct kernel_symbol __start___ksymtab_gpl_future[];
305 extern const struct kernel_symbol __stop___ksymtab_gpl_future[];
306 extern const unsigned long __start___kcrctab_gpl_future[];
307 #endif /* KSPLICE_KSYMTAB_FUTURE_SUPPORT */
308
309 #endif /* KSPLICE_STANDALONE */
310
311 static abort_t process_primary_relocs(struct module_pack *pack,
312 const struct ksplice_reloc *relocs,
313 const struct ksplice_reloc *relocs_end);
314 static abort_t process_primary_reloc(struct module_pack *pack,
315 const struct ksplice_reloc *r);
316 static abort_t apply_ksplice_reloc(struct module_pack *pack,
317 const struct ksplice_reloc *r,
318 unsigned long sym_addr);
319 static abort_t add_system_map_candidates(struct module_pack *pack,
320 const struct ksplice_symbol *symbol,
321 struct list_head *vals);
322 static abort_t compute_address(struct module_pack *pack,
323 const struct ksplice_symbol *ksym,
324 struct list_head *vals);
325
326 struct accumulate_struct {
327 const char *desired_name;
328 struct list_head *vals;
329 };
330
331 #ifdef CONFIG_KALLSYMS
332 static int accumulate_matching_names(void *data, const char *sym_name,
333 unsigned long sym_val);
334 static abort_t kernel_lookup(const char *name, struct list_head *vals);
335 static abort_t other_module_lookup(struct module_pack *pack, const char *name,
336 struct list_head *vals);
337 #ifdef KSPLICE_STANDALONE
338 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,10)
339 static unsigned long ksplice_kallsyms_expand_symbol(unsigned long off,
340 char *result);
341 #endif /* LINUX_VERSION_CODE */
342 #endif /* KSPLICE_STANDALONE */
343 #endif /* CONFIG_KALLSYMS */
344 static abort_t exported_symbol_lookup(const char *name, struct list_head *vals);
345 static abort_t new_export_lookup(struct update_bundle *bundle,
346 const char *name, struct list_head *vals);
347
348 #ifdef KSPLICE_STANDALONE
349 static abort_t brute_search_all(struct module_pack *pack,
350 const struct ksplice_size *s,
351 struct list_head *vals);
352 #endif /* KSPLICE_STANDALONE */
353
354 static abort_t add_candidate_val(struct list_head *vals, unsigned long val);
355 static void release_vals(struct list_head *vals);
356 static void set_temp_myst_relocs(struct module_pack *pack, int status_val);
357 static int contains_canary(struct module_pack *pack, unsigned long blank_addr,
358 int size, long dst_mask);
359 static int starts_with(const char *str, const char *prefix);
360 static int ends_with(const char *str, const char *suffix);
361
362 #define clear_list(head, type, member) \
363 do { \
364 struct list_head *_pos, *_n; \
365 list_for_each_safe(_pos, _n, head) { \
366 list_del(_pos); \
367 kfree(list_entry(_pos, type, member)); \
368 } \
369 } while (0)
370
371 /* primary */
372 static abort_t activate_primary(struct module_pack *pack);
373 static abort_t process_exports(struct module_pack *pack);
374 static abort_t found_all_patches(struct module_pack *pack);
375 static int __apply_patches(void *bundle);
376 static int __reverse_patches(void *bundle);
377 static abort_t check_each_task(struct update_bundle *bundle);
378 static abort_t check_task(struct update_bundle *bundle,
379 const struct task_struct *t);
380 static abort_t check_stack(struct update_bundle *bundle, struct conflict *conf,
381 const struct thread_info *tinfo,
382 const unsigned long *stack);
383 static abort_t check_address_for_conflict(struct update_bundle *bundle,
384 struct conflict *conf,
385 unsigned long addr);
386 static int valid_stack_ptr(const struct thread_info *tinfo, const void *p);
387 static int is_stop_machine(const struct task_struct *t);
388 static void cleanup_conflicts(struct update_bundle *bundle);
389 static void print_conflicts(struct update_bundle *bundle);
390 static void insert_trampoline(struct ksplice_patch *p);
391 static void remove_trampoline(const struct ksplice_patch *p);
392 /* Architecture-specific functions defined in ARCH/ksplice-arch.c */
393 static abort_t create_trampoline(struct ksplice_patch *p);
394 static unsigned long follow_trampolines(struct module_pack *pack,
395 unsigned long addr);
396 static abort_t handle_paravirt(struct module_pack *pack, unsigned long pre,
397 unsigned long run, int *matched);
398 static int virtual_address_mapped(unsigned long addr);
399
400 static abort_t add_dependency_on_address(struct module_pack *pack,
401 unsigned long addr);
402 static abort_t add_patch_dependencies(struct module_pack *pack);
403
404 #if defined(KSPLICE_STANDALONE) && \
405 !defined(CONFIG_KSPLICE) && !defined(CONFIG_KSPLICE_MODULE)
406 #define KSPLICE_NO_KERNEL_SUPPORT 1
407 #endif /* KSPLICE_STANDALONE && !CONFIG_KSPLICE && !CONFIG_KSPLICE_MODULE */
408
409 #ifdef KSPLICE_NO_KERNEL_SUPPORT
410 /* Functions defined here that will be exported in later kernels */
411 #ifdef CONFIG_KALLSYMS
412 static int module_kallsyms_on_each_symbol(const struct module *mod,
413 int (*fn)(void *, const char *,
414 unsigned long),
415 void *data);
416 #endif /* CONFIG_KALLSYMS */
417 static struct module *find_module(const char *name);
418 static int use_module(struct module *a, struct module *b);
419 static const struct kernel_symbol *find_symbol(const char *name,
420 struct module **owner,
421 const unsigned long **crc,
422 bool gplok, bool warn);
423 static struct module *__module_data_address(unsigned long addr);
424 #endif /* KSPLICE_NO_KERNEL_SUPPORT */
425
426 /* helper */
427 static abort_t activate_helper(struct module_pack *pack,
428 bool consider_data_sections);
429 static abort_t search_for_match(struct module_pack *pack,
430 const struct ksplice_size *s);
431 static abort_t try_addr(struct module_pack *pack, const struct ksplice_size *s,
432 unsigned long run_addr, int final);
433 static abort_t run_pre_cmp(struct module_pack *pack,
434 const struct ksplice_size *s,
435 unsigned long run_addr, unsigned long *run_size,
436 int rerun);
437 #ifndef FUNCTION_SECTIONS
438 /* defined in $ARCH/ksplice-arch.c */
439 static abort_t arch_run_pre_cmp(struct module_pack *pack,
440 const struct ksplice_size *s,
441 unsigned long run_addr,
442 unsigned long *run_size, int rerun);
443 #endif /* FUNCTION_SECTIONS */
444 static void print_bytes(struct module_pack *pack,
445 const unsigned char *run, int runc,
446 const unsigned char *pre, int prec);
447
448 static abort_t reverse_patches(struct update_bundle *bundle);
449 static abort_t apply_patches(struct update_bundle *bundle);
450 static abort_t apply_update(struct update_bundle *bundle);
451 static int register_ksplice_module(struct module_pack *pack);
452 static void unregister_ksplice_module(struct module_pack *pack);
453 static struct update_bundle *init_ksplice_bundle(const char *kid);
454 static void cleanup_ksplice_bundle(struct update_bundle *bundle);
455 static void add_to_bundle(struct module_pack *pack,
456 struct update_bundle *bundle);
457 static int ksplice_sysfs_init(struct update_bundle *bundle);
458
459 #ifndef KSPLICE_STANDALONE
460 #include "ksplice-arch.c"
461 #elif defined CONFIG_X86
462 #include "x86/ksplice-arch.c"
463 #elif defined CONFIG_ARM
464 #include "arm/ksplice-arch.c"
465 #endif /* KSPLICE_STANDALONE */
466
467 #ifndef KSPLICE_STANDALONE
468 static struct kobject *ksplice_kobj;
469 #endif /* !KSPLICE_STANDALONE */
470
471 struct ksplice_attribute {
472 struct attribute attr;
473 ssize_t (*show)(struct update_bundle *bundle, char *buf);
474 ssize_t (*store)(struct update_bundle *bundle, const char *buf,
475 size_t len);
476 };
477
478 static ssize_t ksplice_attr_show(struct kobject *kobj, struct attribute *attr,
479 char *buf)
480 {
481 struct ksplice_attribute *attribute =
482 container_of(attr, struct ksplice_attribute, attr);
483 struct update_bundle *bundle =
484 container_of(kobj, struct update_bundle, kobj);
485 if (attribute->show == NULL)
486 return -EIO;
487 return attribute->show(bundle, buf);
488 }
489
490 static ssize_t ksplice_attr_store(struct kobject *kobj, struct attribute *attr,
491 const char *buf, size_t len)
492 {
493 struct ksplice_attribute *attribute =
494 container_of(attr, struct ksplice_attribute, attr);
495 struct update_bundle *bundle =
496 container_of(kobj, struct update_bundle, kobj);
497 if (attribute->store == NULL)
498 return -EIO;
499 return attribute->store(bundle, buf, len);
500 }
501
502 static struct sysfs_ops ksplice_sysfs_ops = {
503 .show = ksplice_attr_show,
504 .store = ksplice_attr_store,
505 };
506
507 static void ksplice_release(struct kobject *kobj)
508 {
509 struct update_bundle *bundle;
510 bundle = container_of(kobj, struct update_bundle, kobj);
511 cleanup_ksplice_bundle(bundle);
512 }
513
514 static ssize_t stage_show(struct update_bundle *bundle, char *buf)
515 {
516 switch (bundle->stage) {
517 case PREPARING:
518 return snprintf(buf, PAGE_SIZE, "preparing\n");
519 case APPLIED:
520 return snprintf(buf, PAGE_SIZE, "applied\n");
521 case REVERSED:
522 return snprintf(buf, PAGE_SIZE, "reversed\n");
523 }
524 return 0;
525 }
526
527 static ssize_t abort_cause_show(struct update_bundle *bundle, char *buf)
528 {
529 switch (bundle->abort_cause) {
530 case OK:
531 return snprintf(buf, PAGE_SIZE, "ok\n");
532 case NO_MATCH:
533 return snprintf(buf, PAGE_SIZE, "no_match\n");
534 case BAD_SYSTEM_MAP:
535 return snprintf(buf, PAGE_SIZE, "bad_system_map\n");
536 case CODE_BUSY:
537 return snprintf(buf, PAGE_SIZE, "code_busy\n");
538 case MODULE_BUSY:
539 return snprintf(buf, PAGE_SIZE, "module_busy\n");
540 case OUT_OF_MEMORY:
541 return snprintf(buf, PAGE_SIZE, "out_of_memory\n");
542 case FAILED_TO_FIND:
543 return snprintf(buf, PAGE_SIZE, "failed_to_find\n");
544 case ALREADY_REVERSED:
545 return snprintf(buf, PAGE_SIZE, "already_reversed\n");
546 case MISSING_EXPORT:
547 return snprintf(buf, PAGE_SIZE, "missing_export\n");
548 case UNEXPECTED_RUNNING_TASK:
549 return snprintf(buf, PAGE_SIZE, "unexpected_running_task\n");
550 case UNEXPECTED:
551 return snprintf(buf, PAGE_SIZE, "unexpected\n");
552 }
553 return 0;
554 }
555
556 static ssize_t conflict_show(struct update_bundle *bundle, char *buf)
557 {
558 const struct conflict *conf;
559 const struct ksplice_frame *frame;
560 int used = 0;
561 list_for_each_entry(conf, &bundle->conflicts, list) {
562 used += snprintf(buf + used, PAGE_SIZE - used, "%s %d",
563 conf->process_name, conf->pid);
564 list_for_each_entry(frame, &conf->stack, list) {
565 if (!frame->has_conflict)
566 continue;
567 used += snprintf(buf + used, PAGE_SIZE - used, " %s",
568 frame->label);
569 }
570 used += snprintf(buf + used, PAGE_SIZE - used, "\n");
571 }
572 return used;
573 }
574
575 static ssize_t stage_store(struct update_bundle *bundle,
576 const char *buf, size_t len)
577 {
578 if (strncmp(buf, "applied\n", len) == 0 && bundle->stage == PREPARING)
579 bundle->abort_cause = apply_update(bundle);
580 else if (strncmp(buf, "reversed\n", len) == 0 &&
581 bundle->stage == APPLIED)
582 bundle->abort_cause = reverse_patches(bundle);
583 return len;
584 }
585
586 static ssize_t debug_show(struct update_bundle *bundle, char *buf)
587 {
588 return snprintf(buf, PAGE_SIZE, "%d\n", bundle->debug);
589 }
590
591 static ssize_t debug_store(struct update_bundle *bundle, const char *buf,
592 size_t len)
593 {
594 char *tmp;
595 int d = simple_strtoul(buf, &tmp, 10);
596 if (*buf && (*tmp == '\0' || *tmp == '\n')) {
597 bundle->debug = d;
598 return len;
599 }
600 return -EINVAL;
601 }
602
603 static struct ksplice_attribute stage_attribute =
604 __ATTR(stage, 0600, stage_show, stage_store);
605 static struct ksplice_attribute abort_cause_attribute =
606 __ATTR(abort_cause, 0400, abort_cause_show, NULL);
607 static struct ksplice_attribute debug_attribute =
608 __ATTR(debug, 0600, debug_show, debug_store);
609 static struct ksplice_attribute conflict_attribute =
610 __ATTR(conflicts, 0400, conflict_show, NULL);
611
612 static struct attribute *ksplice_attrs[] = {
613 &stage_attribute.attr,
614 &abort_cause_attribute.attr,
615 &debug_attribute.attr,
616 &conflict_attribute.attr,
617 NULL
618 };
619
620 static struct kobj_type ksplice_ktype = {
621 .sysfs_ops = &ksplice_sysfs_ops,
622 .release = ksplice_release,
623 .default_attrs = ksplice_attrs,
624 };
625
626 void cleanup_ksplice_module(struct module_pack *pack)
627 {
628 if (pack->bundle == NULL)
629 return;
630 if (pack->bundle->stage != APPLIED)
631 unregister_ksplice_module(pack);
632 }
633 EXPORT_SYMBOL_GPL(cleanup_ksplice_module);
634
635 static abort_t activate_primary(struct module_pack *pack)
636 {
637 abort_t ret;
638 ret = process_primary_relocs(pack, pack->primary_relocs,
639 pack->primary_relocs_end);
640 if (ret != OK)
641 return ret;
642
643 ret = process_exports(pack);
644 if (ret != OK)
645 return ret;
646
647 ret = add_patch_dependencies(pack);
648 if (ret != OK)
649 return ret;
650
651 ret = found_all_patches(pack);
652 if (ret != OK)
653 return ret;
654
655 return OK;
656 }
657
658 static abort_t found_all_patches(struct module_pack *pack)
659 {
660 const struct ksplice_patch *p;
661 struct safety_record *rec;
662
663 /* Check every patch has a safety_record */
664 for (p = pack->patches; p < pack->patches_end; p++) {
665 int found = 0;
666 list_for_each_entry(rec, &pack->safety_records, list) {
667 if (strcmp(rec->label, p->symbol->label) == 0) {
668 found = 1;
669 break;
670 }
671 }
672 if (!found)
673 return UNEXPECTED;
674 if (rec->size < p->size) {
675 ksdebug(pack, 0, KERN_DEBUG "Symbol %s is too short "
676 "for trampoline\n", p->symbol->label);
677 return UNEXPECTED;
678 }
679 }
680 return OK;
681 }
682
683 static abort_t process_exports(struct module_pack *pack)
684 {
685 struct ksplice_export *export;
686 struct module *m;
687 const struct kernel_symbol *sym;
688 const char *export_type;
689
690 for (export = pack->exports; export < pack->exports_end; export++) {
691 sym = find_symbol(export->name, &m, NULL, true, false);
692 if (sym == NULL) {
693 ksdebug(pack, 0, "Could not find kernel_symbol struct"
694 "for %s (%s)\n", export->name, export->type);
695 return MISSING_EXPORT;
696 }
697 /* Do we actually want the following check? */
698 export_type = export->type;
699 if (strcmp(export_type, export->type) != 0) {
700 ksdebug(pack, 0, "Nonmatching export type for %s "
701 "(%s/%s)\n", export->name, export->type,
702 export_type);
703 return MISSING_EXPORT;
704 }
705 /* Cast away const since we are planning to mutate the
706 * kernel_symbol structure. */
707 export->sym = (struct kernel_symbol *)sym;
708 export->saved_name = export->sym->name;
709 if (m != pack->primary && use_module(pack->primary, m) != 1)
710 return UNEXPECTED;
711 }
712 return OK;
713 }
714
715 static void insert_trampoline(struct ksplice_patch *p)
716 {
717 mm_segment_t old_fs = get_fs();
718 create_trampoline(p);
719 set_fs(KERNEL_DS);
720 memcpy((void *)p->saved, (void *)p->oldaddr, p->size);
721 memcpy((void *)p->oldaddr, (void *)p->trampoline, p->size);
722 flush_icache_range(p->oldaddr, p->oldaddr + p->size);
723 set_fs(old_fs);
724 }
725
726 static void remove_trampoline(const struct ksplice_patch *p)
727 {
728 mm_segment_t old_fs = get_fs();
729 set_fs(KERNEL_DS);
730 memcpy((void *)p->oldaddr, (void *)p->saved, p->size);
731 flush_icache_range(p->oldaddr, p->oldaddr + p->size);
732 set_fs(old_fs);
733 }
734
735 static abort_t apply_patches(struct update_bundle *bundle)
736 {
737 int i;
738 abort_t ret;
739
740 for (i = 0; i < 5; i++) {
741 cleanup_conflicts(bundle);
742 #ifdef KSPLICE_STANDALONE
743 bust_spinlocks(1);
744 #endif /* KSPLICE_STANDALONE */
745 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,27)
746 ret = (__force abort_t)stop_machine(__apply_patches, bundle,
747 NULL);
748 #else /* LINUX_VERSION_CODE < */
749 /* 9b1a4d38373a5581a4e01032a3ccdd94cd93477b was after 2.6.26 */
750 ret = (__force abort_t)stop_machine_run(__apply_patches, bundle,
751 NR_CPUS);
752 #endif /* LINUX_VERSION_CODE */
753 #ifdef KSPLICE_STANDALONE
754 bust_spinlocks(0);
755 #endif /* KSPLICE_STANDALONE */
756 if (ret != CODE_BUSY)
757 break;
758 set_current_state(TASK_INTERRUPTIBLE);
759 schedule_timeout(msecs_to_jiffies(1000));
760 }
761
762 if (ret == OK) {
763 struct module_pack *pack;
764 const struct ksplice_size *s;
765 struct safety_record *rec;
766 list_for_each_entry(pack, &bundle->packs, list) {
767 for (s = pack->primary_sizes;
768 s < pack->primary_sizes_end; s++) {
769 rec = kmalloc(sizeof(*rec), GFP_KERNEL);
770 if (rec == NULL)
771 return OUT_OF_MEMORY;
772 rec->addr = s->thismod_addr;
773 rec->size = s->size;
774 rec->label = s->symbol->label;
775 list_add(&rec->list, &pack->safety_records);
776 }
777 }
778 _ksdebug(bundle, 0, KERN_INFO "ksplice: Update %s applied "
779 "successfully\n", bundle->kid);
780 return 0;
781 } else if (ret == CODE_BUSY) {
782 print_conflicts(bundle);
783 _ksdebug(bundle, 0, KERN_ERR "ksplice: Aborted %s. stack "
784 "check: to-be-replaced code is busy\n", bundle->kid);
785 } else if (ret == ALREADY_REVERSED) {
786 _ksdebug(bundle, 0, KERN_ERR "ksplice: Aborted %s. Ksplice "
787 "update %s is already reversed.\n", bundle->kid,
788 bundle->kid);
789 }
790 return ret;
791 }
792
793 static abort_t reverse_patches(struct update_bundle *bundle)
794 {
795 int i;
796 abort_t ret;
797 struct module_pack *pack;
798
799 clear_debug_buf(bundle);
800 ret = init_debug_buf(bundle);
801 if (ret != OK)
802 return ret;
803
804 _ksdebug(bundle, 0, KERN_INFO "ksplice: Preparing to reverse %s\n",
805 bundle->kid);
806
807 for (i = 0; i < 5; i++) {
808 cleanup_conflicts(bundle);
809 clear_list(&bundle->conflicts, struct conflict, list);
810 #ifdef KSPLICE_STANDALONE
811 bust_spinlocks(1);
812 #endif /* KSPLICE_STANDALONE */
813 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,27)
814 ret = (__force abort_t)stop_machine(__reverse_patches, bundle,
815 NULL);
816 #else /* LINUX_VERSION_CODE < */
817 /* 9b1a4d38373a5581a4e01032a3ccdd94cd93477b was after 2.6.26 */
818 ret = (__force abort_t)stop_machine_run(__reverse_patches,
819 bundle, NR_CPUS);
820 #endif /* LINUX_VERSION_CODE */
821 #ifdef KSPLICE_STANDALONE
822 bust_spinlocks(0);
823 #endif /* KSPLICE_STANDALONE */
824 if (ret != CODE_BUSY)
825 break;
826 set_current_state(TASK_INTERRUPTIBLE);
827 schedule_timeout(msecs_to_jiffies(1000));
828 }
829 list_for_each_entry(pack, &bundle->packs, list)
830 clear_list(&pack->safety_records, struct safety_record, list);
831 if (ret == OK) {
832 _ksdebug(bundle, 0, KERN_INFO "ksplice: Update %s reversed"
833 " successfully\n", bundle->kid);
834 } else if (ret == CODE_BUSY) {
835 print_conflicts(bundle);
836 _ksdebug(bundle, 0, KERN_ERR "ksplice: Aborted %s. stack "
837 "check: to-be-reversed code is busy\n", bundle->kid);
838 } else if (ret == MODULE_BUSY) {
839 _ksdebug(bundle, 0, KERN_ERR "ksplice: Update %s is"
840 " in use by another module\n", bundle->kid);
841 }
842 return ret;
843 }
844
845 static int __apply_patches(void *bundleptr)
846 {
847 struct update_bundle *bundle = bundleptr;
848 struct module_pack *pack;
849 struct ksplice_patch *p;
850 struct ksplice_export *export;
851 abort_t ret;
852
853 if (bundle->stage == APPLIED)
854 return (__force int)OK;
855
856 if (bundle->stage != PREPARING)
857 return (__force int)UNEXPECTED;
858
859 ret = check_each_task(bundle);
860 if (ret != OK)
861 return (__force int)ret;
862
863 list_for_each_entry(pack, &bundle->packs, list) {
864 if (try_module_get(pack->primary) != 1) {
865 struct module_pack *pack1;
866 list_for_each_entry(pack1, &bundle->packs, list) {
867 if (pack1 == pack)
868 break;
869 module_put(pack1->primary);
870 }
871 return (__force int)UNEXPECTED;
872 }
873 }
874
875 bundle->stage = APPLIED;
876
877 list_for_each_entry(pack, &bundle->packs, list) {
878 for (export = pack->exports; export < pack->exports_end;
879 export++)
880 export->sym->name = export->new_name;
881 }
882
883 list_for_each_entry(pack, &bundle->packs, list) {
884 for (p = pack->patches; p < pack->patches_end; p++)
885 insert_trampoline(p);
886 }
887 return (__force int)OK;
888 }
889
890 static int __reverse_patches(void *bundleptr)
891 {
892 struct update_bundle *bundle = bundleptr;
893 struct module_pack *pack;
894 const struct ksplice_patch *p;
895 struct ksplice_export *export;
896 abort_t ret;
897
898 if (bundle->stage != APPLIED)
899 return (__force int)OK;
900
901 #ifdef CONFIG_MODULE_UNLOAD
902 /* primary's refcount isn't changed by accessing ksplice.ko's sysfs */
903 list_for_each_entry(pack, &bundle->packs, list) {
904 if (module_refcount(pack->primary) != 1)
905 return (__force int)MODULE_BUSY;
906 }
907 #endif /* CONFIG_MODULE_UNLOAD */
908
909 ret = check_each_task(bundle);
910 if (ret != OK)
911 return (__force int)ret;
912
913 bundle->stage = REVERSED;
914
915 list_for_each_entry(pack, &bundle->packs, list)
916 module_put(pack->primary);
917
918 list_for_each_entry(pack, &bundle->packs, list) {
919 for (export = pack->exports; export < pack->exports_end;
920 export++)
921 export->sym->name = export->saved_name;
922 }
923
924 list_for_each_entry(pack, &bundle->packs, list) {
925 for (p = pack->patches; p < pack->patches_end; p++)
926 remove_trampoline(p);
927 }
928 return (__force int)OK;
929 }
930
931 static abort_t check_each_task(struct update_bundle *bundle)
932 {
933 const struct task_struct *g, *p;
934 abort_t result, ret = OK;
935 #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,11)
936 /* 5d4564e68210e4b1edb3f013bc3e59982bb35737 was after 2.6.10 */
937 read_lock(&tasklist_lock);
938 #endif /* LINUX_VERSION_CODE */
939 do_each_thread(g, p) {
940 /* do_each_thread is a double loop! */
941 result = check_task(bundle, p);
942 if (result != OK)
943 ret = result;
944 }
945 while_each_thread(g, p);
946 #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,11)
947 /* 5d4564e68210e4b1edb3f013bc3e59982bb35737 was after 2.6.10 */
948 read_unlock(&tasklist_lock);
949 #endif /* LINUX_VERSION_CODE */
950 return ret;
951 }
952
953 static abort_t check_task(struct update_bundle *bundle,
954 const struct task_struct *t)
955 {
956 abort_t status, ret;
957 struct conflict *conf = kmalloc(sizeof(*conf), GFP_ATOMIC);
958 if (conf == NULL)
959 return OUT_OF_MEMORY;
960 conf->process_name = kstrdup(t->comm, GFP_ATOMIC);
961 if (conf->process_name == NULL) {
962 kfree(conf);
963 return OUT_OF_MEMORY;
964 }
965 conf->pid = t->pid;
966 INIT_LIST_HEAD(&conf->stack);
967 list_add(&conf->list, &bundle->conflicts);
968
969 status = check_address_for_conflict(bundle, conf, KSPLICE_IP(t));
970 if (t == current) {
971 ret = check_stack(bundle, conf, task_thread_info(t),
972 (unsigned long *)__builtin_frame_address(0));
973 if (status == OK)
974 status = ret;
975 } else if (!task_curr(t)) {
976 ret = check_stack(bundle, conf, task_thread_info(t),
977 (unsigned long *)KSPLICE_SP(t));
978 if (status == OK)
979 status = ret;
980 } else if (!is_stop_machine(t)) {
981 status = UNEXPECTED_RUNNING_TASK;
982 }
983 return status;
984 }
985
986 /* Modified version of Linux's print_context_stack */
987 static abort_t check_stack(struct update_bundle *bundle, struct conflict *conf,
988 const struct thread_info *tinfo,
989 const unsigned long *stack)
990 {
991 abort_t status = OK, ret;
992 unsigned long addr;
993
994 while (valid_stack_ptr(tinfo, stack)) {
995 addr = *stack++;
996 ret = check_address_for_conflict(bundle, conf, addr);
997 if (ret != OK)
998 status = ret;
999 }
1000 return status;
1001 }
1002
1003 static abort_t check_address_for_conflict(struct update_bundle *bundle,
1004 struct conflict *conf,
1005 unsigned long addr)
1006 {
1007 const struct safety_record *rec;
1008 struct module_pack *pack;
1009 struct ksplice_frame *frame = kmalloc(sizeof(*frame), GFP_ATOMIC);
1010 if (frame == NULL)
1011 return OUT_OF_MEMORY;
1012 frame->addr = addr;
1013 frame->has_conflict = 0;
1014 frame->label = NULL;
1015 list_add(&frame->list, &conf->stack);
1016
1017 list_for_each_entry(pack, &bundle->packs, list) {
1018 list_for_each_entry(rec, &pack->safety_records, list) {
1019 if ((addr > rec->addr && addr < rec->addr + rec->size)
1020 || (addr == rec->addr && !rec->first_byte_safe)) {
1021 frame->label = rec->label;
1022 frame->has_conflict = 1;
1023 return CODE_BUSY;
1024 }
1025 }
1026 }
1027 return OK;
1028 }
1029
1030 /* Modified version of Linux's valid_stack_ptr */
1031 static int valid_stack_ptr(const struct thread_info *tinfo, const void *p)
1032 {
1033 return p > (const void *)tinfo
1034 && p <= (const void *)tinfo + THREAD_SIZE - sizeof(long);
1035 }
1036
1037 static int is_stop_machine(const struct task_struct *t)
1038 {
1039 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,27)
1040 const char *num;
1041 if (!starts_with(t->comm, "kstop"))
1042 return 0;
1043 num = t->comm + strlen("kstop");
1044 return num[strspn(num, "0123456789")] == '\0';
1045 #else /* LINUX_VERSION_CODE < */
1046 return strcmp(t->comm, "kstopmachine") == 0;
1047 #endif /* LINUX_VERSION_CODE */
1048 }
1049
1050 static void cleanup_conflicts(struct update_bundle *bundle)
1051 {
1052 struct conflict *conf;
1053 list_for_each_entry(conf, &bundle->conflicts, list) {
1054 clear_list(&conf->stack, struct ksplice_frame, list);
1055 kfree(conf->process_name);
1056 }
1057 clear_list(&bundle->conflicts, struct conflict, list);
1058 }
1059
1060 static void print_conflicts(struct update_bundle *bundle)
1061 {
1062 const struct conflict *conf;
1063 const struct ksplice_frame *frame;
1064 list_for_each_entry(conf, &bundle->conflicts, list) {
1065 _ksdebug(bundle, 2, KERN_DEBUG "ksplice: stack check: pid %d "
1066 "(%s):", conf->pid, conf->process_name);
1067 list_for_each_entry(frame, &conf->stack, list) {
1068 _ksdebug(bundle, 2, " %" ADDR, frame->addr);
1069 if (frame->has_conflict)
1070 _ksdebug(bundle, 2, " [<-CONFLICT]");
1071 }
1072 _ksdebug(bundle, 2, "\n");
1073 }
1074 }
1075
1076 #ifdef KSPLICE_NO_KERNEL_SUPPORT
1077 static struct module *find_module(const char *name)
1078 {
1079 struct module *mod;
1080
1081 list_for_each_entry(mod, &modules, list) {
1082 if (strcmp(mod->name, name) == 0)
1083 return mod;
1084 }
1085 return NULL;
1086 }
1087 #endif /* KSPLICE_NO_KERNEL_SUPPORT */
1088
1089 static int register_ksplice_module(struct module_pack *pack)
1090 {
1091 struct update_bundle *bundle;
1092 int ret = 0;
1093
1094 INIT_LIST_HEAD(&pack->reloc_namevals);
1095 INIT_LIST_HEAD(&pack->safety_records);
1096
1097 mutex_lock(&module_mutex);
1098 if (strcmp(pack->target_name, "vmlinux") == 0) {
1099 pack->target = NULL;
1100 } else {
1101 pack->target = find_module(pack->target_name);
1102 if (pack->target == NULL || !module_is_live(pack->target)) {
1103 ret = -ENODEV;
1104 goto out;
1105 }
1106 }
1107 list_for_each_entry(bundle, &update_bundles, list) {
1108 if (strcmp(pack->kid, bundle->kid) == 0) {
1109 if (bundle->stage != PREPARING) {
1110 ret = -EPERM;
1111 goto out;
1112 }
1113 add_to_bundle(pack, bundle);
1114 goto out;
1115 }
1116 }
1117 bundle = init_ksplice_bundle(pack->kid);
1118 if (bundle == NULL) {
1119 ret = -ENOMEM;
1120 goto out;
1121 }
1122 ret = ksplice_sysfs_init(bundle);
1123 if (ret != 0) {
1124 cleanup_ksplice_bundle(bundle);
1125 goto out;
1126 }
1127 add_to_bundle(pack, bundle);
1128 out:
1129 mutex_unlock(&module_mutex);
1130 return ret;
1131 }
1132
1133 static void unregister_ksplice_module(struct module_pack *pack)
1134 {
1135 if (pack->bundle == NULL)
1136 return;
1137 if (pack->bundle->stage != APPLIED) {
1138 mutex_lock(&module_mutex);
1139 list_del(&pack->list);
1140 list_del(&pack->module_list_entry.list);
1141 mutex_unlock(&module_mutex);
1142 if (list_empty(&pack->bundle->packs))
1143 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,25)
1144 kobject_put(&pack->bundle->kobj);
1145 #else /* LINUX_VERSION_CODE < */
1146 /* 6d06adfaf82d154023141ddc0c9de18b6a49090b was after 2.6.24 */
1147 kobject_unregister(&pack->bundle->kobj);
1148 #endif /* LINUX_VERSION_CODE */
1149 pack->bundle = NULL;
1150 }
1151 }
1152
1153 static void add_to_bundle(struct module_pack *pack,
1154 struct update_bundle *bundle)
1155 {
1156 pack->bundle = bundle;
1157 list_add(&pack->list, &bundle->packs);
1158 list_add(&pack->module_list_entry.list, &ksplice_module_list);
1159 pack->module_list_entry.target = pack->target;
1160 pack->module_list_entry.primary = pack->primary;
1161 }
1162
1163 static void cleanup_ksplice_bundle(struct update_bundle *bundle)
1164 {
1165 mutex_lock(&module_mutex);
1166 list_del(&bundle->list);
1167 mutex_unlock(&module_mutex);
1168 cleanup_conflicts(bundle);
1169 clear_debug_buf(bundle);
1170 kfree(bundle->kid);
1171 kfree(bundle->name);
1172 kfree(bundle);
1173 }
1174
1175 static struct update_bundle *init_ksplice_bundle(const char *kid)
1176 {
1177 struct update_bundle *bundle;
1178 const char *str = "ksplice_";
1179 char *buf;
1180 bundle = kcalloc(1, sizeof(struct update_bundle), GFP_KERNEL);
1181 if (bundle == NULL)
1182 return NULL;
1183 buf = kmalloc(strlen(kid) + strlen(str) + 1, GFP_KERNEL);
1184 if (buf == NULL) {
1185 kfree(bundle);
1186 return NULL;
1187 }
1188 snprintf(buf, strlen(kid) + strlen(str) + 1, "%s%s", str, kid);
1189 bundle->name = buf;
1190 bundle->kid = kstrdup(kid, GFP_KERNEL);
1191 if (bundle->kid == NULL) {
1192 kfree(bundle->name);
1193 kfree(bundle);
1194 return NULL;
1195 }
1196 INIT_LIST_HEAD(&bundle->packs);
1197 if (init_debug_buf(bundle) != OK) {
1198 kfree(bundle->kid);
1199 kfree(bundle->name);
1200 kfree(bundle);
1201 return NULL;
1202 }
1203 list_add(&bundle->list, &update_bundles);
1204 bundle->stage = PREPARING;
1205 bundle->abort_cause = OK;
1206 INIT_LIST_HEAD(&bundle->conflicts);
1207 return bundle;
1208 }
1209
1210 static int ksplice_sysfs_init(struct update_bundle *bundle)
1211 {
1212 int ret = 0;
1213 memset(&bundle->kobj, 0, sizeof(bundle->kobj));
1214 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,25)
1215 #ifndef KSPLICE_STANDALONE
1216 ret = kobject_init_and_add(&bundle->kobj, &ksplice_ktype,
1217 ksplice_kobj, "%s", bundle->kid);
1218 #else /* KSPLICE_STANDALONE */
1219 /* 6d06adfaf82d154023141ddc0c9de18b6a49090b was after 2.6.24 */
1220 ret = kobject_init_and_add(&bundle->kobj, &ksplice_ktype,
1221 &THIS_MODULE->mkobj.kobj, "ksplice");
1222 #endif /* KSPLICE_STANDALONE */
1223 #else /* LINUX_VERSION_CODE < */
1224 ret = kobject_set_name(&bundle->kobj, "%s", "ksplice");
1225 if (ret != 0)
1226 return ret;
1227 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,11)
1228 bundle->kobj.parent = &THIS_MODULE->mkobj.kobj;
1229 #else /* LINUX_VERSION_CODE < */
1230 /* b86ab02803095190d6b72bcc18dcf620bf378df9 was after 2.6.10 */
1231 bundle->kobj.parent = &THIS_MODULE->mkobj->kobj;
1232 #endif /* LINUX_VERSION_CODE */
1233 bundle->kobj.ktype = &ksplice_ktype;
1234 ret = kobject_register(&bundle->kobj);
1235 #endif /* LINUX_VERSION_CODE */
1236 if (ret != 0)
1237 return ret;
1238 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,15)
1239 kobject_uevent(&bundle->kobj, KOBJ_ADD);
1240 #elif LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,10)
1241 /* 312c004d36ce6c739512bac83b452f4c20ab1f62 was after 2.6.14 */
1242 /* 12025235884570ba7f02a6f427f973ac6be7ec54 was after 2.6.9 */
1243 kobject_uevent(&bundle->kobj, KOBJ_ADD, NULL);
1244 #endif /* LINUX_VERSION_CODE */
1245 return 0;
1246 }
1247
1248 int init_ksplice_module(struct module_pack *pack)
1249 {
1250 #ifdef KSPLICE_STANDALONE
1251 if (bootstrapped == 0)
1252 return -1;
1253 #endif /* KSPLICE_STANDALONE */
1254 return register_ksplice_module(pack);
1255 }
1256 EXPORT_SYMBOL(init_ksplice_module);
1257
1258 static abort_t apply_update(struct update_bundle *bundle)
1259 {
1260 struct module_pack *pack;
1261 abort_t ret;
1262
1263 mutex_lock(&module_mutex);
1264 #ifdef KSPLICE_NEED_PARAINSTRUCTIONS
1265 list_for_each_entry(pack, &bundle->packs, list) {
1266 if (pack->target == NULL) {
1267 apply_paravirt(pack->primary_parainstructions,
1268 pack->primary_parainstructions_end);
1269 apply_paravirt(pack->helper_parainstructions,
1270 pack->helper_parainstructions_end);
1271 }
1272 }
1273 #endif /* KSPLICE_NEED_PARAINSTRUCTIONS */
1274
1275 list_for_each_entry(pack, &bundle->packs, list) {
1276 ksdebug(pack, 0, KERN_INFO "ksplice_h: Preparing and checking "
1277 "%s\n", pack->name);
1278 ret = activate_helper(pack, false);
1279 if (ret == NO_MATCH) {
1280 ksdebug(pack, 1, KERN_DEBUG "ksplice: "
1281 "Trying to continue without the unmatched "
1282 "sections; we will find them later.\n");
1283 ret = activate_primary(pack);
1284 if (ret != OK) {
1285 ksdebug(pack, 1, KERN_DEBUG "ksplice: "
1286 "Aborted. Unable to continue without "
1287 "the unmatched sections.\n");
1288 goto out;
1289 }
1290 ksdebug(pack, 1, KERN_DEBUG "ksplice: run-pre: "
1291 "Considering .data sections to find the "
1292 "unmatched sections\n");
1293 ret = activate_helper(pack, true);
1294 if (ret != OK)
1295 goto out;
1296 ksdebug(pack, 1, KERN_DEBUG "ksplice_h: run-pre: "
1297 "Found all previously unmatched sections\n");
1298 } else if (ret != OK) {
1299 goto out;
1300 } else {
1301 ret = activate_primary(pack);
1302 if (ret != OK)
1303 goto out;
1304 }
1305 }
1306 ret = apply_patches(bundle);
1307 out:
1308 list_for_each_entry(pack, &bundle->packs, list) {
1309 clear_list(&pack->reloc_namevals, struct reloc_nameval, list);
1310 if (bundle->stage == PREPARING)
1311 clear_list(&pack->safety_records, struct safety_record,
1312 list);
1313 }
1314 mutex_unlock(&module_mutex);
1315 return ret;
1316 }
1317
1318 static abort_t activate_helper(struct module_pack *pack,
1319 bool consider_data_sections)
1320 {
1321 const struct ksplice_size *s;
1322 abort_t ret;
1323 char *finished;
1324 int i, remaining = 0;
1325 bool progress;
1326
1327 finished = kcalloc(pack->helper_sizes_end - pack->helper_sizes,
1328 sizeof(char), GFP_KERNEL);
1329 if (finished == NULL)
1330 return OUT_OF_MEMORY;
1331 for (s = pack->helper_sizes; s < pack->helper_sizes_end; s++) {
1332 if ((s->flags & KSPLICE_SIZE_DATA) == 0)
1333 remaining++;
1334 }
1335
1336 while (remaining > 0) {
1337 progress = false;
1338 for (s = pack->helper_sizes; s < pack->helper_sizes_end; s++) {
1339 i = s - pack->helper_sizes;
1340 if (finished[i])
1341 continue;
1342 if (!consider_data_sections &&
1343 (s->flags & KSPLICE_SIZE_DATA) != 0)
1344 continue;
1345 ret = search_for_match(pack, s);
1346 if (ret == OK) {
1347 finished[i] = 1;
1348 if ((s->flags & KSPLICE_SIZE_DATA) == 0)
1349 remaining--;
1350 progress = true;
1351 } else if (ret != NO_MATCH) {
1352 kfree(finished);
1353 return ret;
1354 }
1355 }
1356
1357 if (progress)
1358 continue;
1359
1360 for (s = pack->helper_sizes; s < pack->helper_sizes_end; s++) {
1361 i = s - pack->helper_sizes;
1362 if (finished[i] == 0)
1363 ksdebug(pack, 2, KERN_DEBUG "ksplice: run-pre: "
1364 "could not match section %s\n",
1365 s->symbol->label);
1366 }
1367 print_abort(pack, "run-pre: could not match some sections");
1368 kfree(finished);
1369 return NO_MATCH;
1370 }
1371 return OK;
1372 }
1373
1374 static abort_t search_for_match(struct module_pack *pack,
1375 const struct ksplice_size *s)
1376 {
1377 int i;
1378 abort_t ret;
1379 unsigned long run_addr;
1380 LIST_HEAD(vals);
1381 struct candidate_val *v, *n;
1382
1383 ret = add_system_map_candidates(pack, s->symbol, &vals);
1384 if (ret != OK) {
1385 release_vals(&vals);
1386 return ret;
1387 }
1388 ret = compute_address(pack, s->symbol, &vals);
1389 if (ret != OK) {
1390 release_vals(&vals);
1391 return ret;
1392 }
1393
1394 ksdebug(pack, 3, KERN_DEBUG "ksplice_h: run-pre: starting sect search "
1395 "for %s\n", s->symbol->label);
1396
1397 list_for_each_entry_safe(v, n, &vals, list) {
1398 run_addr = v->val;
1399
1400 yield();
1401 ret = try_addr(pack, s, run_addr, 0);
1402 if (ret == NO_MATCH) {
1403 list_del(&v->list);
1404 kfree(v);
1405 } else if (ret != OK) {
1406 release_vals(&vals);
1407 return ret;
1408 }
1409 }
1410
1411 #ifdef KSPLICE_STANDALONE
1412 if (list_empty(&vals) && (s->flags & KSPLICE_SIZE_DATA) == 0) {
1413 ret = brute_search_all(pack, s, &vals);
1414 if (ret != OK)
1415 return ret;
1416 }
1417 #endif /* KSPLICE_STANDALONE */
1418
1419 if (singular(&vals)) {
1420 run_addr = list_entry(vals.next, struct candidate_val,
1421 list)->val;
1422 ret = try_addr(pack, s, run_addr, 1);
1423 release_vals(&vals);
1424 if (ret != OK)
1425 ksdebug(pack, 3, KERN_DEBUG "ksplice_h: run-pre: "
1426 "Final run failed for sect %s:\n",
1427 s->symbol->label);
1428 return ret;
1429 } else if (!list_empty(&vals)) {
1430 struct candidate_val *val;
1431 ksdebug(pack, 3, KERN_DEBUG "ksplice_h: run-pre: multiple "
1432 "candidates for sect %s:\n", s->symbol->label);
1433 i = 0;
1434 list_for_each_entry(val, &vals, list) {
1435 i++;
1436 ksdebug(pack, 3, KERN_DEBUG "%lx\n", val->val);
1437 if (i > 5) {
1438 ksdebug(pack, 3, KERN_DEBUG "...\n");
1439 break;
1440 }
1441 }
1442 release_vals(&vals);
1443 return NO_MATCH;
1444 }
1445 release_vals(&vals);
1446 return NO_MATCH;
1447 }
1448
1449 static void print_bytes(struct module_pack *pack,
1450 const unsigned char *run, int runc,
1451 const unsigned char *pre, int prec)
1452 {
1453 int o;
1454 int matched = min(runc, prec);
1455 for (o = 0; o < matched; o++) {
1456 if (run[o] == pre[o])
1457 ksdebug(pack, 0, "%02x ", run[o]);
1458 else
1459 ksdebug(pack, 0, "%02x/%02x ", run[o], pre[o]);
1460 }
1461 for (o = matched; o < runc; o++)
1462 ksdebug(pack, 0, "%02x/ ", run[o]);
1463 for (o = matched; o < prec; o++)
1464 ksdebug(pack, 0, "/%02x ", pre[o]);
1465 }
1466
1467 static abort_t run_pre_cmp(struct module_pack *pack,
1468 const struct ksplice_size *s,
1469 unsigned long run_addr, unsigned long *run_size,
1470 int rerun)
1471 {
1472 int matched = 0;
1473 abort_t ret;
1474 unsigned long pre_addr = s->thismod_addr;
1475 const struct ksplice_reloc *r;
1476 const unsigned char *pre, *run;
1477
1478 if ((s->flags & KSPLICE_SIZE_TEXT) != 0)
1479 run_addr = follow_trampolines(pack, run_addr);
1480
1481 pre = (const unsigned char *)pre_addr;
1482 run = (const unsigned char *)run_addr;
1483 while (pre < (const unsigned char *)pre_addr + s->size) {
1484 if (!virtual_address_mapped((unsigned long)run)) {
1485 if (!rerun)
1486 ksdebug(pack, 3, "sect unmapped after "
1487 "%lx/%lx bytes\n",
1488 (unsigned long)pre - pre_addr, s->size);
1489 return NO_MATCH;
1490 }
1491 ret = lookup_reloc(pack, (unsigned long)pre, &r);
1492 if (ret == OK) {
1493 if (!virtual_address_mapped((unsigned long)run +
1494 r->size - 1))
1495 return NO_MATCH;
1496 ret = handle_reloc(pack, r, (unsigned long)run, rerun);
1497 if (ret != OK) {
1498 if (!rerun)
1499 ksdebug(pack, 3, "reloc in sect does "
1500 "not match after %lx/%lx bytes"
1501 "\n", (unsigned long)pre -
1502 pre_addr, s->size);
1503 return ret;
1504 }
1505 if (rerun)
1506 print_bytes(pack, run, r->size, pre, r->size);
1507 pre += r->size;
1508 run += r->size;
1509 continue;
1510 } else if (ret != NO_MATCH) {
1511 return ret;
1512 }
1513
1514 if ((s->flags & KSPLICE_SIZE_TEXT) != 0) {
1515 ret = handle_paravirt(pack, (unsigned long)pre,
1516 (unsigned long)run, &matched);
1517 if (ret != OK)
1518 return ret;
1519 if (matched != 0) {
1520 if (rerun)
1521 print_bytes(pack, run, matched, pre,
1522 matched);
1523 pre += matched;
1524 run += matched;
1525 continue;
1526 }
1527 }
1528
1529 if (*run != *pre && (s->flags & KSPLICE_SIZE_DATA) == 0) {
1530 if (!rerun)
1531 ksdebug(pack, 3, "sect does not match after "
1532 "%lx/%lx bytes\n",
1533 (unsigned long)pre - pre_addr, s->size);
1534 if (rerun) {
1535 print_bytes(pack, run, 1, pre, 1);
1536 ksdebug(pack, 0, "[p_o=%lx] ! ",
1537 (unsigned long)pre - pre_addr);
1538 print_bytes(pack, run + 1, 2, pre + 1, 2);
1539 }
1540 return NO_MATCH;
1541 }
1542 if (rerun)
1543 print_bytes(pack, run, 1, pre, 1);
1544 pre++;
1545 run++;
1546 }
1547 *run_size = (unsigned long)run - run_addr;
1548 return OK;
1549 }
1550
1551 #ifdef KSPLICE_NO_KERNEL_SUPPORT
1552 static struct module *__module_data_address(unsigned long addr)
1553 {
1554 struct module *mod;
1555
1556 list_for_each_entry(mod, &modules, list) {
1557 if (addr >= (unsigned long)mod->module_core +
1558 mod->core_text_size &&
1559 addr < (unsigned long)mod->module_core + mod->core_size)
1560 return mod;
1561 }
1562 return NULL;
1563 }
1564 #endif /* KSPLICE_NO_KERNEL_SUPPORT */
1565
1566 static abort_t try_addr(struct module_pack *pack, const struct ksplice_size *s,
1567 unsigned long run_addr, int final)
1568 {
1569 struct safety_record *rec;
1570 struct ksplice_patch *p;
1571 abort_t ret;
1572 unsigned long run_size;
1573 const struct module *run_module;
1574
1575 if ((s->flags & KSPLICE_SIZE_RODATA) != 0 ||
1576 (s->flags & KSPLICE_SIZE_DATA) != 0)
1577 run_module = __module_data_address(run_addr);
1578 else
1579 run_module = __module_text_address(run_addr);
1580 if (run_module != pack->target) {
1581 ksdebug(pack, 1, KERN_DEBUG "ksplice_h: run-pre: ignoring "
1582 "address %" ADDR " in other module %s for sect %s\n",
1583 run_addr,
1584 run_module == NULL ? "vmlinux" : run_module->name,
1585 s->symbol->label);
1586 return NO_MATCH;
1587 }
1588
1589 ret = create_nameval(pack, s->symbol->label, run_addr, TEMP);
1590 if (ret != OK)
1591 return ret;
1592
1593 #ifdef FUNCTION_SECTIONS
1594 ret = run_pre_cmp(pack, s, run_addr, &run_size, 0);
1595 #else
1596 if ((s->flags & KSPLICE_SIZE_TEXT) != 0)
1597 ret = arch_run_pre_cmp(pack, s, run_addr, &run_size, 0);
1598 else
1599 ret = run_pre_cmp(pack, s, run_addr, &run_size, 0);
1600 #endif
1601 if (ret == NO_MATCH) {
1602 set_temp_myst_relocs(pack, NOVAL);
1603 ksdebug(pack, 1, KERN_DEBUG "ksplice_h: run-pre: %s sect %s "
1604 "does not match ",
1605 (s->flags & KSPLICE_SIZE_RODATA) != 0 ? "data" : "text",
1606 s->symbol->label);
1607 ksdebug(pack, 1, "(r_a=%" ADDR " p_a=%" ADDR " s=%ld)\n",
1608 run_addr, s->thismod_addr, s->size);
1609 ksdebug(pack, 1, KERN_DEBUG "ksplice_h: run-pre: ");
1610 if (pack->bundle->debug >= 1) {
1611 #ifdef FUNCTION_SECTIONS
1612 ret = run_pre_cmp(pack, s, run_addr, &run_size, 1);
1613 #else
1614 if ((s->flags & KSPLICE_SIZE_TEXT) != 0)
1615 ret = arch_run_pre_cmp(pack, s, run_addr,
1616 &run_size, 1);
1617 else
1618 ret = run_pre_cmp(pack, s, run_addr, &run_size,
1619 1);
1620 #endif
1621 set_temp_myst_relocs(pack, NOVAL);
1622 }
1623 ksdebug(pack, 1, "\n");
1624 return ret;
1625 } else if (ret != OK) {
1626 set_temp_myst_relocs(pack, NOVAL);
1627 return ret;
1628 } else if (!final) {
1629 set_temp_myst_relocs(pack, NOVAL);
1630 ksdebug(pack, 3, KERN_DEBUG "ksplice_h: run-pre: candidate "
1631 "for sect %s=%" ADDR "\n", s->symbol->label, run_addr);
1632 return OK;
1633 }
1634
1635 set_temp_myst_relocs(pack, VAL);
1636 ksdebug(pack, 3, KERN_DEBUG "ksplice_h: run-pre: found sect %s=%" ADDR
1637 "\n", s->symbol->label, run_addr);
1638
1639 for (p = pack->patches; p < pack->patches_end; p++) {
1640 if (strcmp(s->symbol->label, p->symbol->label) == 0)
1641 break;
1642 }
1643 if (p >= pack->patches_end && (s->flags & KSPLICE_SIZE_DELETED) == 0)
1644 return OK;
1645
1646 if ((s->flags & KSPLICE_SIZE_TEXT) == 0 &&
1647 (s->flags & KSPLICE_SIZE_DELETED) == 0) {
1648 ksdebug(pack, 3, KERN_DEBUG "ksplice_h: Error: ksplice_patch "
1649 "%s is matched to a non-deleted non-text section!\n",
1650 s->symbol->label);
1651 return UNEXPECTED;
1652 }
1653
1654 rec = kmalloc(sizeof(*rec), GFP_KERNEL);
1655 if (rec == NULL)
1656 return OUT_OF_MEMORY;
1657 rec->addr = run_addr;
1658 rec->size = run_size;
1659 rec->label = s->symbol->label;
1660 /* The beginning of a patched function is safe to
1661 because that location will be overwritten with a trampoline. */
1662 rec->first_byte_safe = (s->flags & KSPLICE_SIZE_DELETED) == 0 &&
1663 (s->flags & KSPLICE_SIZE_TEXT) != 0;
1664
1665 list_add(&rec->list, &pack->safety_records);
1666 return OK;
1667 }
1668
1669 static abort_t handle_reloc(struct module_pack *pack,
1670 const struct ksplice_reloc *r,
1671 unsigned long run_addr, int rerun)
1672 {
1673 long run_reloc_val, expected;
1674 abort_t ret;
1675 struct reloc_nameval *nv = find_nameval(pack, r->symbol->label);
1676
1677 switch (r->size) {
1678 case 1:
1679 run_reloc_val = *(int8_t *)run_addr & (int8_t)r->dst_mask;
1680 break;
1681 case 2:
1682 run_reloc_val = *(int16_t *)run_addr & (int16_t)r->dst_mask;
1683 break;
1684 case 4:
1685 run_reloc_val = *(int32_t *)run_addr & (int32_t)r->dst_mask;
1686 break;
1687 case 8:
1688 run_reloc_val = *(int64_t *)run_addr & r->dst_mask;
1689 break;
1690 default:
1691 print_abort(pack, "Invalid relocation size");
1692 return UNEXPECTED;
1693 }
1694
1695 if (r->signed_addend)
1696 run_reloc_val |=
1697 -(run_reloc_val & (r->dst_mask & ~(r->dst_mask >> 1)));
1698
1699 run_reloc_val <<= r->rightshift;
1700
1701 if (!rerun) {
1702 ksdebug(pack, 3, KERN_DEBUG "ksplice_h: run-pre: reloc at r_a=%"
1703 ADDR " p_a=%" ADDR ": ", run_addr, r->blank_addr);
1704 ksdebug(pack, 3, "%s=%" ADDR " (A=%" ADDR " *r=%" ADDR ")\n",
1705 r->symbol->label, nv != NULL ? nv->val : 0, r->addend,
1706 run_reloc_val);
1707 }
1708
1709 if (!starts_with(r->symbol->label, ".rodata.str")) {
1710 if (contains_canary(pack, run_addr, r->size, r->dst_mask) != 0)
1711 return UNEXPECTED;
1712
1713 expected = run_reloc_val - r->addend;
1714 if (r->pcrel)
1715 expected += run_addr;
1716
1717 ret = create_nameval(pack, r->symbol->label, expected, TEMP);
1718 if (ret == NO_MATCH && !rerun) {
1719 ksdebug(pack, 1, KERN_DEBUG "ksplice_h: run-pre reloc: "
1720 "Nameval address %" ADDR "(%d) does not match "
1721 "expected %" ADDR " for %s!\n", nv->val,
1722 nv->status, expected, r->symbol->label);
1723 ksdebug(pack, 1, KERN_DEBUG "ksplice_h: run-pre reloc: "
1724 "run_reloc: %" ADDR " %" ADDR " %" ADDR "\n",
1725 run_addr, run_reloc_val, r->addend);
1726 return ret;
1727 } else if (ret != OK) {
1728 return ret;
1729 }
1730 }
1731 return OK;
1732 }
1733
1734 static abort_t apply_ksplice_reloc(struct module_pack *pack,
1735 const struct ksplice_reloc *r,
1736 unsigned long sym_addr)
1737 {
1738 unsigned long val = sym_addr + r->addend;
1739 if (r->pcrel)
1740 val -= r->blank_addr;
1741 switch (r->size) {
1742 case 1:
1743 *(int8_t *)r->blank_addr =
1744 (*(int8_t *)r->blank_addr & ~(int8_t)r->dst_mask)
1745 | ((val >> r->rightshift) & (int8_t)r->dst_mask);
1746 break;
1747 case 2:
1748 *(int16_t *)r->blank_addr =
1749 (*(int16_t *)r->blank_addr & ~(int16_t)r->dst_mask)
1750 | ((val >> r->rightshift) & (int16_t)r->dst_mask);
1751 break;
1752 case 4:
1753 *(int32_t *)r->blank_addr =
1754 (*(int32_t *)r->blank_addr & ~(int32_t)r->dst_mask)
1755 | ((val >> r->rightshift) & (int32_t)r->dst_mask);
1756 break;
1757 case 8:
1758 *(int64_t *)r->blank_addr =
1759 (*(int64_t *)r->blank_addr & ~r->dst_mask) |
1760 ((val >> r->rightshift) & r->dst_mask);
1761 break;
1762 default:
1763 print_abort(pack, "Invalid relocation size");
1764 return UNEXPECTED;
1765 }
1766 return OK;
1767 }
1768
1769 static abort_t process_primary_relocs(struct module_pack *pack,
1770 const struct ksplice_reloc *relocs,
1771 const struct ksplice_reloc *relocs_end)
1772 {
1773 const struct ksplice_reloc *r;
1774 for (r = relocs; r < relocs_end; r++) {
1775 abort_t ret = process_primary_reloc(pack, r);
1776 if (ret != OK)
1777 return ret;
1778 }
1779 return OK;
1780 }
1781
1782 static abort_t process_primary_reloc(struct module_pack *pack,
1783 const struct ksplice_reloc *r)
1784 {
1785 abort_t ret;
1786 int canary_ret;
1787 unsigned long sym_addr;
1788 LIST_HEAD(vals);
1789
1790 canary_ret = contains_canary(pack, r->blank_addr, r->size, r->dst_mask);
1791 if (canary_ret < 0)
1792 return UNEXPECTED;
1793 if (canary_ret == 0) {
1794 ksdebug(pack, 4, KERN_DEBUG "ksplice: reloc: skipped %s:%"
1795 ADDR " (altinstr)\n", r->symbol->label,
1796 r->blank_offset);
1797 return OK;
1798 }
1799
1800 #ifdef KSPLICE_STANDALONE
1801 if (!bootstrapped) {
1802 ret = add_system_map_candidates(pack, r->symbol, &vals);
1803 if (ret != OK) {
1804 release_vals(&vals);
1805 return ret;
1806 }
1807 }
1808 #else /* !KSPLICE_STANDALONE */
1809 #ifdef CONFIG_KALLSYMS
1810 ret = add_system_map_candidates(pack, r->symbol, &vals);
1811 if (ret != OK) {
1812 release_vals(&vals);
1813 return ret;
1814 }
1815 #endif /* CONFIG_KALLSYMS */
1816 #endif /* KSPLICE_STANDALONE */
1817 ret = compute_address(pack, r->symbol, &vals);
1818 if (ret != OK) {
1819 release_vals(&vals);
1820 return ret;
1821 }
1822 if (!singular(&vals)) {
1823 release_vals(&vals);
1824 failed_to_find(pack, r->symbol->label);
1825 return FAILED_TO_FIND;
1826 }
1827 sym_addr = list_entry(vals.next, struct candidate_val, list)->val;
1828 release_vals(&vals);
1829
1830 ret = apply_ksplice_reloc(pack, r, sym_addr);
1831 if (ret != OK)
1832 return ret;
1833
1834 ksdebug(pack, 4, KERN_DEBUG "ksplice: reloc: %s:%" ADDR " ",
1835 r->symbol->label, r->blank_offset);
1836 ksdebug(pack, 4, "(S=%" ADDR " A=%" ADDR " ", sym_addr, r->addend);
1837 switch (r->size) {
1838 case 1:
1839 ksdebug(pack, 4, "aft=%02x)\n", *(int8_t *)r->blank_addr);
1840 break;
1841 case 2:
1842 ksdebug(pack, 4, "aft=%04x)\n", *(int16_t *)r->blank_addr);
1843 break;
1844 case 4:
1845 ksdebug(pack, 4, "aft=%08x)\n", *(int32_t *)r->blank_addr);
1846 break;
1847 case 8:
1848 ksdebug(pack, 4, "aft=%016llx)\n", *(int64_t *)r->blank_addr);
1849 break;
1850 default:
1851 print_abort(pack, "Invalid relocation size");
1852 return UNEXPECTED;
1853 }
1854 #ifdef KSPLICE_STANDALONE
1855 if (!bootstrapped)
1856 return OK;
1857 #endif /* KSPLICE_STANDALONE */
1858 /* Create namevals so that we can verify our choices in the second
1859 round of run-pre matching that considers data sections. */
1860 ret = create_nameval(pack, r->symbol->label, sym_addr, VAL);
1861 if (ret != OK)
1862 return ret;
1863 return add_dependency_on_address(pack, sym_addr);
1864 }
1865
1866 static abort_t add_system_map_candidates(struct module_pack *pack,
1867 const struct ksplice_symbol *symbol,
1868 struct list_head *vals)
1869 {
1870 abort_t ret;
1871 long off;
1872 int i;
1873
1874 /* Some Fedora kernel releases have System.map files whose symbol
1875 * addresses disagree with the running kernel by a constant address
1876 * offset because of the CONFIG_PHYSICAL_START and CONFIG_PHYSICAL_ALIGN
1877 * values used to compile these kernels. This constant address offset
1878 * is always a multiple of 0x100000.
1879 *
1880 * If we observe an offset that is NOT a multiple of 0x100000, then the
1881 * user provided us with an incorrect System.map file, and we should
1882 * abort.
1883 * If we observe an offset that is a multiple of 0x100000, then we can
1884 * adjust the System.map address values accordingly and proceed.
1885 */
1886 off = (unsigned long)printk - pack->map_printk;
1887 if (off & 0xfffff) {
1888 print_abort(pack, "System.map does not match kernel");
1889 return BAD_SYSTEM_MAP;
1890 }
1891 for (i = 0; i < symbol->nr_candidates; i++) {
1892 ret = add_candidate_val(vals, symbol->candidates[i] + off);
1893 if (ret != OK)
1894 return ret;
1895 }
1896 return OK;
1897 }
1898
1899 static abort_t add_dependency_on_address(struct module_pack *pack,
1900 unsigned long addr)
1901 {
1902 struct module *m =
1903 __module_text_address(follow_trampolines(pack, addr));
1904 if (m == NULL || starts_with(m->name, pack->name) ||
1905 ends_with(m->name, "_helper"))
1906 return OK;
1907 if (use_module(pack->primary, m) != 1)
1908 return MODULE_BUSY;
1909 return OK;
1910 }
1911
1912 static abort_t add_patch_dependencies(struct module_pack *pack)
1913 {
1914 abort_t ret;
1915 const struct ksplice_patch *p;
1916 for (p = pack->patches; p < pack->patches_end; p++) {
1917 ret = add_dependency_on_address(pack, p->oldaddr);
1918 if (ret != OK)
1919 return ret;
1920 }
1921 return 0;
1922 }
1923
1924 #ifdef KSPLICE_NO_KERNEL_SUPPORT
1925 #ifdef CONFIG_MODULE_UNLOAD
1926 struct module_use {
1927 struct list_head list;
1928 struct module *module_which_uses;
1929 };
1930
1931 /* I'm not yet certain whether we need the strong form of this. */
1932 static inline int strong_try_module_get(struct module *mod)
1933 {
1934 if (mod && mod->state != MODULE_STATE_LIVE)
1935 return -EBUSY;
1936 if (try_module_get(mod))
1937 return 0;
1938 return -ENOENT;
1939 }
1940
1941 /* Does a already use b? */
1942 static int already_uses(struct module *a, struct module *b)
1943 {
1944 struct module_use *use;
1945 list_for_each_entry(use, &b->modules_which_use_me, list) {
1946 if (use->module_which_uses == a)
1947 return 1;
1948 }
1949 return 0;
1950 }
1951
1952 /* Make it so module a uses b. Must be holding module_mutex */
1953 static int use_module(struct module *a, struct module *b)
1954 {
1955 struct module_use *use;
1956 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,21)
1957 /* 270a6c4cad809e92d7b81adde92d0b3d94eeb8ee was after 2.6.20 */
1958 int no_warn;
1959 #endif /* LINUX_VERSION_CODE */
1960 if (b == NULL || already_uses(a, b))
1961 return 1;
1962
1963 if (strong_try_module_get(b) < 0)
1964 return 0;
1965
1966 use = kmalloc(sizeof(*use), GFP_ATOMIC);
1967 if (!use) {
1968 module_put(b);
1969 return 0;
1970 }
1971 use->module_which_uses = a;
1972 list_add(&use->list, &b->modules_which_use_me);
1973 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,21)
1974 /* 270a6c4cad809e92d7b81adde92d0b3d94eeb8ee was after 2.6.20 */
1975 no_warn = sysfs_create_link(b->holders_dir, &a->mkobj.kobj, a->name);
1976 #endif /* LINUX_VERSION_CODE */
1977 return 1;
1978 }
1979 #else /* CONFIG_MODULE_UNLOAD */
1980 static int use_module(struct module *a, struct module *b)
1981 {
1982 return 1;
1983 }
1984 #endif /* CONFIG_MODULE_UNLOAD */
1985 #endif /* KSPLICE_NO_KERNEL_SUPPORT */
1986
1987 static abort_t compute_address(struct module_pack *pack,
1988 const struct ksplice_symbol *ksym,
1989 struct list_head *vals)
1990 {
1991 abort_t ret;
1992 struct reloc_nameval *nv;
1993
1994 #ifdef KSPLICE_STANDALONE
1995 if (!bootstrapped)
1996 return OK;
1997 #endif /* KSPLICE_STANDALONE */
1998
1999 nv = find_nameval(pack, ksym->label);
2000 if (nv != NULL) {
2001 release_vals(vals);
2002 ksdebug(pack, 1, KERN_DEBUG "ksplice: using detected "
2003 "sym %s=%" ADDR "\n", ksym->label, nv->val);
2004 return add_candidate_val(vals, nv->val);
2005 }
2006
2007 if (starts_with(ksym->label, ".rodata"))
2008 return OK;
2009
2010 #ifdef CONFIG_MODULE_UNLOAD
2011 if (strcmp(ksym->label, "cleanup_module") == 0 && pack->target != NULL
2012 && pack->target->exit != NULL) {
2013 ret = add_candidate_val(vals,
2014 (unsigned long)pack->target->exit);
2015 if (ret != OK)
2016 return ret;
2017 }
2018 #endif
2019
2020 ret = exported_symbol_lookup(ksym->name, vals);
2021 if (ret == OK)
2022 ret = new_export_lookup(pack->bundle, ksym->name, vals);
2023 #ifdef CONFIG_KALLSYMS
2024 if (ret == OK)
2025 ret = kernel_lookup(ksym->name, vals);
2026 if (ret == OK)
2027 ret = other_module_lookup(pack, ksym->name, vals);
2028 #endif /* CONFIG_KALLSYMS */
2029 if (ret != OK)
2030 return ret;
2031
2032 return OK;
2033 }
2034
2035 static abort_t new_export_lookup(struct update_bundle *bundle,
2036 const char *name, struct list_head *vals)
2037 {
2038 struct module_pack *pack;
2039 struct ksplice_export *exp;
2040 list_for_each_entry(pack, &bundle->packs, list) {
2041 for (exp = pack->exports; exp < pack->exports_end; exp++) {
2042 if (strcmp(exp->new_name, name) == 0 &&
2043 exp->sym != NULL &&
2044 contains_canary(pack,
2045 (unsigned long)&exp->sym->value,
2046 sizeof(unsigned long), -1) == 0)
2047 return add_candidate_val(vals, exp->sym->value);
2048 }
2049 }
2050 return OK;
2051 }
2052
2053 static abort_t exported_symbol_lookup(const char *name, struct list_head *vals)
2054 {
2055 const struct kernel_symbol *sym;
2056 sym = find_symbol(name, NULL, NULL, true, false);
2057 if (sym == NULL)
2058 return OK;
2059 return add_candidate_val(vals, sym->value);
2060 }
2061
2062 #ifdef KSPLICE_NO_KERNEL_SUPPORT
2063 #ifndef CONFIG_MODVERSIONS
2064 #define symversion(base, idx) NULL
2065 #else
2066 #define symversion(base, idx) ((base != NULL) ? ((base) + (idx)) : NULL)
2067 #endif
2068
2069 struct symsearch {
2070 const struct kernel_symbol *start, *stop;
2071 const unsigned long *crcs;
2072 enum {
2073 NOT_GPL_ONLY,
2074 GPL_ONLY,
2075 WILL_BE_GPL_ONLY,
2076 } licence;
2077 bool unused;
2078 };
2079
2080 static bool each_symbol_in_section(const struct symsearch *arr,
2081 unsigned int arrsize,
2082 struct module *owner,
2083 bool (*fn)(const struct symsearch *syms,
2084 struct module *owner,
2085 unsigned int symnum, void *data),
2086 void *data)
2087 {
2088 unsigned int i, j;
2089
2090 for (j = 0; j < arrsize; j++) {
2091 for (i = 0; i < arr[j].stop - arr[j].start; i++)
2092 if (fn(&arr[j], owner, i, data))
2093 return true;
2094 }
2095
2096 return false;
2097 }
2098
2099 /* Returns true as soon as fn returns true, otherwise false. */
2100 static bool each_symbol(bool (*fn)(const struct symsearch *arr,
2101 struct module *owner,
2102 unsigned int symnum, void *data),
2103 void *data)
2104 {
2105 struct module *mod;
2106 const struct symsearch arr[] = {
2107 { __start___ksymtab, __stop___ksymtab, __start___kcrctab,
2108 NOT_GPL_ONLY, false },
2109 { __start___ksymtab_gpl, __stop___ksymtab_gpl,
2110 __start___kcrctab_gpl,
2111 GPL_ONLY, false },
2112 #ifdef KSPLICE_KSYMTAB_FUTURE_SUPPORT
2113 { __start___ksymtab_gpl_future, __stop___ksymtab_gpl_future,
2114 __start___kcrctab_gpl_future,
2115 WILL_BE_GPL_ONLY, false },
2116 #endif /* KSPLICE_KSYMTAB_FUTURE_SUPPORT */
2117 #ifdef KSPLICE_KSYMTAB_UNUSED_SUPPORT
2118 { __start___ksymtab_unused, __stop___ksymtab_unused,
2119 __start___kcrctab_unused,
2120 NOT_GPL_ONLY, true },
2121 { __start___ksymtab_unused_gpl, __stop___ksymtab_unused_gpl,
2122 __start___kcrctab_unused_gpl,
2123 GPL_ONLY, true },
2124 #endif /* KSPLICE_KSYMTAB_UNUSED_SUPPORT */
2125 };
2126
2127 if (each_symbol_in_section(arr, ARRAY_SIZE(arr), NULL, fn, data))
2128 return 1;
2129
2130 list_for_each_entry(mod, &modules, list) {
2131 struct symsearch module_arr[] = {
2132 { mod->syms, mod->syms + mod->num_syms, mod->crcs,
2133 NOT_GPL_ONLY, false },
2134 { mod->gpl_syms, mod->gpl_syms + mod->num_gpl_syms,
2135 mod->gpl_crcs,
2136 GPL_ONLY, false },
2137 #ifdef KSPLICE_KSYMTAB_FUTURE_SUPPORT
2138 { mod->gpl_future_syms,
2139 mod->gpl_future_syms + mod->num_gpl_future_syms,
2140 mod->gpl_future_crcs,
2141 WILL_BE_GPL_ONLY, false },
2142 #endif /* KSPLICE_KSYMTAB_FUTURE_SUPPORT */
2143 #ifdef KSPLICE_KSYMTAB_UNUSED_SUPPORT
2144 { mod->unused_syms,
2145 mod->unused_syms + mod->num_unused_syms,
2146 mod->unused_crcs,
2147 NOT_GPL_ONLY, true },
2148 { mod->unused_gpl_syms,
2149 mod->unused_gpl_syms + mod->num_unused_gpl_syms,
2150 mod->unused_gpl_crcs,
2151 GPL_ONLY, true },
2152 #endif /* KSPLICE_KSYMTAB_UNUSED_SUPPORT */
2153 };
2154
2155 if (each_symbol_in_section(module_arr, ARRAY_SIZE(module_arr),
2156 mod, fn, data))
2157 return true;
2158 }
2159 return false;
2160 }
2161
2162 struct find_symbol_arg {
2163 /* Input */
2164 const char *name;
2165 bool gplok;
2166 bool warn;
2167
2168 /* Output */
2169 struct module *owner;
2170 const unsigned long *crc;
2171 const struct kernel_symbol *sym;
2172 };
2173
2174 static bool find_symbol_in_section(const struct symsearch *syms,
2175 struct module *owner,
2176 unsigned int symnum, void *data)
2177 {
2178 struct find_symbol_arg *fsa = data;
2179
2180 if (strcmp(syms->start[symnum].name, fsa->name) != 0)
2181 return false;
2182
2183 if (!fsa->gplok) {
2184 if (syms->licence == GPL_ONLY)
2185 return false;
2186 if (syms->licence == WILL_BE_GPL_ONLY && fsa->warn) {
2187 printk(KERN_WARNING "Symbol %s is being used "
2188 "by a non-GPL module, which will not "
2189 "be allowed in the future\n", fsa->name);
2190 printk(KERN_WARNING "Please see the file "
2191 "Documentation/feature-removal-schedule.txt "
2192 "in the kernel source tree for more details.\n");
2193 }
2194 }
2195
2196 #ifdef CONFIG_UNUSED_SYMBOLS
2197 if (syms->unused && fsa->warn) {
2198 printk(KERN_WARNING "Symbol %s is marked as UNUSED, "
2199 "however this module is using it.\n", fsa->name);
2200 printk(KERN_WARNING
2201 "This symbol will go away in the future.\n");
2202 printk(KERN_WARNING
2203 "Please evalute if this is the right api to use and if "
2204 "it really is, submit a report the linux kernel "
2205 "mailinglist together with submitting your code for "
2206 "inclusion.\n");
2207 }
2208 #endif
2209
2210 fsa->owner = owner;
2211 fsa->crc = symversion(syms->crcs, symnum);
2212 fsa->sym = &syms->start[symnum];
2213 return true;
2214 }
2215
2216 /* Find a symbol and return it, along with, (optional) crc and
2217 * (optional) module which owns it */
2218 static const struct kernel_symbol *find_symbol(const char *name,
2219 struct module **owner,
2220 const unsigned long **crc,
2221 bool gplok, bool warn)
2222 {
2223 struct find_symbol_arg fsa;
2224
2225 fsa.name = name;
2226 fsa.gplok = gplok;
2227 fsa.warn = warn;
2228
2229 if (each_symbol(find_symbol_in_section, &fsa)) {
2230 if (owner)
2231 *owner = fsa.owner;
2232 if (crc)
2233 *crc = fsa.crc;
2234 return fsa.sym;
2235 }
2236
2237 return NULL;
2238 }
2239 #endif /* KSPLICE_NO_KERNEL_SUPPORT */
2240
2241 #ifdef CONFIG_KALLSYMS
2242 #ifdef KSPLICE_NO_KERNEL_SUPPORT
2243 static abort_t other_module_lookup(struct module_pack *pack, const char *name,
2244 struct list_head *vals)
2245 {
2246 abort_t ret = OK;
2247 struct accumulate_struct acc = { name, vals };
2248 const struct module *m;
2249
2250 list_for_each_entry(m, &modules, list) {
2251 if (starts_with(m->name, pack->name) ||
2252 !ends_with(m->name, pack->target_name))
2253 continue;
2254 ret = (__force abort_t)
2255 module_kallsyms_on_each_symbol(m, accumulate_matching_names,
2256 &acc);
2257 if (ret != OK)
2258 break;
2259 }
2260 return ret;
2261 }
2262 #else /* !KSPLICE_NO_KERNEL_SUPPORT */
2263 static abort_t other_module_lookup(struct module_pack *pack, const char *name,
2264 struct list_head *vals)
2265 {
2266 struct accumulate_struct acc = { name, vals };
2267 struct ksplice_module_list_entry *entry;
2268 abort_t ret;
2269
2270 list_for_each_entry(entry, &ksplice_module_list, list) {
2271 if (entry->target != pack->target ||
2272 entry->primary == pack->primary)
2273 continue;
2274 ret = (__force abort_t)
2275 module_kallsyms_on_each_symbol(entry->primary,
2276 accumulate_matching_names,
2277 &acc);
2278 if (ret != OK)
2279 return ret;
2280 }
2281 if (pack->target == NULL)
2282 return OK;
2283 ret = (__force abort_t)
2284 module_kallsyms_on_each_symbol(pack->target,
2285 accumulate_matching_names, &acc);
2286 return ret;
2287 }
2288 #endif /* KSPLICE_NO_KERNEL_SUPPORT */
2289
2290 static int accumulate_matching_names(void *data, const char *sym_name,
2291 unsigned long sym_val)
2292 {
2293 abort_t ret = OK;
2294 struct accumulate_struct *acc = data;
2295
2296 if (strcmp(sym_name, acc->desired_name) == 0)
2297 ret = add_candidate_val(acc->vals, sym_val);
2298 return (__force int)ret;
2299 }
2300 #endif /* CONFIG_KALLSYMS */
2301
2302 #ifdef KSPLICE_STANDALONE
2303 static abort_t brute_search(struct module_pack *pack,
2304 const struct ksplice_size *s,
2305 const void *start, unsigned long len,
2306 struct list_head *vals)
2307 {
2308 unsigned long addr;
2309 char run, pre;
2310 abort_t ret;
2311
2312 for (addr = (unsigned long)start; addr < (unsigned long)start + len;
2313 addr++) {
2314 if (addr % 100000 == 0)
2315 yield();
2316
2317 if (!virtual_address_mapped(addr))
2318 return OK;
2319
2320 run = *(const unsigned char *)(addr);
2321 pre = *(const unsigned char *)(s->thismod_addr);
2322
2323 if (run != pre)
2324 continue;
2325
2326 ret = try_addr(pack, s, addr, 0);
2327 if (ret == OK) {
2328 ret = add_candidate_val(vals, addr);
2329 if (ret != OK)
2330 return ret;
2331 } else if (ret != NO_MATCH) {
2332 return ret;
2333 }
2334 }
2335
2336 return OK;
2337 }
2338
2339 static abort_t brute_search_all(struct module_pack *pack,
2340 const struct ksplice_size *s,
2341 struct list_head *vals)
2342 {
2343 struct module *m;
2344 abort_t ret = OK;
2345 int saved_debug;
2346
2347 ksdebug(pack, 2, KERN_DEBUG "ksplice: brute_search: searching for %s\n",
2348 s->symbol->label);
2349 saved_debug = pack->bundle->debug;
2350 pack->bundle->debug = 0;
2351
2352 list_for_each_entry(m, &modules, list) {
2353 if (starts_with(m->name, pack->name) ||
2354 ends_with(m->name, "_helper"))
2355 continue;
2356 ret = brute_search(pack, s, m->module_core, m->core_size, vals);
2357 if (ret != OK)
2358 break;
2359 ret = brute_search(pack, s, m->module_init, m->init_size, vals);
2360 if (ret != OK)
2361 break;
2362 }
2363 if (ret == OK)
2364 ret = brute_search(pack, s, (const void *)init_mm.start_code,
2365 init_mm.end_code - init_mm.start_code, vals);
2366 pack->bundle->debug = saved_debug;
2367
2368 return ret;
2369 }
2370
2371 #ifdef CONFIG_KALLSYMS
2372 /* Modified version of Linux's kallsyms_lookup_name */
2373 static abort_t kernel_lookup(const char *name, struct list_head *vals)
2374 {
2375 abort_t ret;
2376 char namebuf[KSYM_NAME_LEN + 1];
2377 unsigned long i;
2378 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,10)
2379 unsigned long off;
2380 #endif /* LINUX_VERSION_CODE */
2381
2382 /* kallsyms compression was added by 5648d78927ca65e74aadc88a2b1d6431e55e78ec
2383 * 2.6.10 was the first release after this commit
2384 */
2385 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,10)
2386 for (i = 0, off = 0; i < kallsyms_num_syms; i++) {
2387 off = ksplice_kallsyms_expand_symbol(off, namebuf);
2388
2389 if (strcmp(namebuf, name) == 0) {
2390 ret = add_candidate_val(vals, kallsyms_addresses[i]);
2391 if (ret != OK)
2392 return ret;
2393 }
2394 }
2395 #else /* LINUX_VERSION_CODE < */
2396 char *knames;
2397
2398 for (i = 0, knames = kallsyms_names; i < kallsyms_num_syms; i++) {
2399 unsigned prefix = *knames++;
2400
2401 strlcpy(namebuf + prefix, knames, KSYM_NAME_LEN - prefix);
2402
2403 if (strcmp(namebuf, name) == 0) {
2404 ret = add_candidate_val(vals, kallsyms_addresses[i]);
2405 if (ret != OK)
2406 return ret;
2407 }
2408
2409 knames += strlen(knames) + 1;
2410 }
2411 #endif /* LINUX_VERSION_CODE */
2412
2413 return OK;
2414 }
2415
2416 /* kallsyms compression was added by 5648d78927ca65e74aadc88a2b1d6431e55e78ec
2417 * 2.6.10 was the first release after this commit
2418 */
2419 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,10)
2420 extern u8 kallsyms_token_table[];
2421 extern u16 kallsyms_token_index[];
2422 /* Modified version of Linux's kallsyms_expand_symbol */
2423 static unsigned long ksplice_kallsyms_expand_symbol(unsigned long off,
2424 char *result)
2425 {
2426 long len, skipped_first = 0;
2427 const u8 *tptr, *data;
2428
2429 data = &kallsyms_names[off];
2430 len = *data;
2431 data++;
2432
2433 off += len + 1;
2434
2435 while (len) {
2436 tptr = &kallsyms_token_table[kallsyms_token_index[*data]];
2437 data++;
2438 len--;
2439
2440 while (*tptr) {
2441 if (skipped_first) {
2442 *result = *tptr;
2443 result++;
2444 } else
2445 skipped_first = 1;
2446 tptr++;
2447 }
2448 }
2449
2450 *result = '\0';
2451
2452 return off;
2453 }
2454 #endif /* LINUX_VERSION_CODE */
2455
2456 #ifdef KSPLICE_NO_KERNEL_SUPPORT
2457 static int module_kallsyms_on_each_symbol(const struct module *mod,
2458 int (*fn)(void *, const char *,
2459 unsigned long),
2460 void *data)
2461 {
2462 unsigned int i;
2463 int ret;
2464
2465 for (i = 0; i < mod->num_symtab; i++) {
2466 if ((ret =
2467 fn(data, mod->strtab + mod->symtab[i].st_name,
2468 mod->symtab[i].st_value) != 0))
2469 return ret;
2470 }
2471 return 0;
2472 }
2473 #endif /* KSPLICE_NO_KERNEL_SUPPORT */
2474 #endif /* CONFIG_KALLSYMS */
2475 #else /* !KSPLICE_STANDALONE */
2476
2477 static abort_t kernel_lookup(const char *name, struct list_head *vals)
2478 {
2479 struct accumulate_struct acc = { name, vals };
2480 return (__force abort_t)
2481 kernel_kallsyms_on_each_symbol(accumulate_matching_names, &acc);
2482 }
2483 #endif /* KSPLICE_STANDALONE */
2484
2485 static abort_t add_candidate_val(struct list_head *vals, unsigned long val)
2486 {
2487 struct candidate_val *tmp, *new;
2488
2489 list_for_each_entry(tmp, vals, list) {
2490 if (tmp->val == val)
2491 return OK;
2492 }
2493 new = kmalloc(sizeof(*new), GFP_KERNEL);
2494 if (new == NULL)
2495 return OUT_OF_MEMORY;
2496 new->val = val;
2497 list_add(&new->list, vals);
2498 return OK;
2499 }
2500
2501 static void release_vals(struct list_head *vals)
2502 {
2503 clear_list(vals, struct candidate_val, list);
2504 }
2505
2506 static struct reloc_nameval *find_nameval(struct module_pack *pack,
2507 const char *label)
2508 {
2509 struct reloc_nameval *nv;
2510 list_for_each_entry(nv, &pack->reloc_namevals, list) {
2511 if (strcmp(nv->label, label) == 0)
2512 return nv;
2513 }
2514 return NULL;
2515 }
2516
2517 static abort_t create_nameval(struct module_pack *pack, const char *label,
2518 unsigned long val, int status)
2519 {
2520 struct reloc_nameval *nv = find_nameval(pack, label);
2521 if (nv != NULL)
2522 return nv->val == val ? OK : NO_MATCH;
2523
2524 nv = kmalloc(sizeof(*nv), GFP_KERNEL);
2525 if (nv == NULL)
2526 return OUT_OF_MEMORY;
2527 nv->label = label;
2528 nv->val = val;
2529 nv->status = status;
2530 list_add(&nv->list, &pack->reloc_namevals);
2531 return OK;
2532 }
2533
2534 static abort_t lookup_reloc(struct module_pack *pack, unsigned long addr,
2535 const struct ksplice_reloc **relocp)
2536 {
2537 const struct ksplice_reloc *r;
2538 int canary_ret;
2539 for (r = pack->helper_relocs; r < pack->helper_relocs_end; r++) {
2540 if (addr >= r->blank_addr && addr < r->blank_addr + r->size) {
2541 canary_ret = contains_canary(pack, r->blank_addr,
2542 r->size, r->dst_mask);
2543 if (canary_ret < 0)
2544 return UNEXPECTED;
2545 if (canary_ret == 0) {
2546 ksdebug(pack, 4, KERN_DEBUG "ksplice_h: reloc: "
2547 "skipped %s:%" ADDR " (altinstr)\n",
2548 r->symbol->label, r->blank_offset);
2549 return NO_MATCH;
2550 }
2551 if (addr != r->blank_addr) {
2552 ksdebug(pack, 4, KERN_DEBUG "ksplice_h: Invalid"
2553 " nonzero relocation offset\n");
2554 return UNEXPECTED;
2555 }
2556 *relocp = r;
2557 return OK;
2558 }
2559 }
2560 return NO_MATCH;
2561 }
2562
2563 static void set_temp_myst_relocs(struct module_pack *pack, int status_val)
2564 {
2565 struct reloc_nameval *nv, *n;
2566 list_for_each_entry_safe(nv, n, &pack->reloc_namevals, list) {
2567 if (nv->status == TEMP) {
2568 if (status_val == NOVAL) {
2569 list_del(&nv->list);
2570 kfree(nv);
2571 } else {
2572 nv->status = status_val;
2573 }
2574 }
2575 }
2576 }
2577
2578 static int contains_canary(struct module_pack *pack, unsigned long blank_addr,
2579 int size, long dst_mask)
2580 {
2581 switch (size) {
2582 case 1:
2583 return (*(int8_t *)blank_addr & (int8_t)dst_mask) ==
2584 (0x77 & dst_mask);
2585 case 2:
2586 return (*(int16_t *)blank_addr & (int16_t)dst_mask) ==
2587 (0x7777 & dst_mask);
2588 case 4:
2589 return (*(int32_t *)blank_addr & (int32_t)dst_mask) ==
2590 (0x77777777 & dst_mask);
2591 case 8:
2592 return (*(int64_t *)blank_addr & dst_mask) ==
2593 (0x7777777777777777ll & dst_mask);
2594 default:
2595 print_abort(pack, "Invalid relocation size");
2596 return -1;
2597 }
2598 }
2599
2600 static int starts_with(const char *str, const char *prefix)
2601 {
2602 return strncmp(str, prefix, strlen(prefix)) == 0;
2603 }
2604
2605 static int ends_with(const char *str, const char *suffix)
2606 {
2607 return strlen(str) >= strlen(suffix) &&
2608 strcmp(&str[strlen(str) - strlen(suffix)], suffix) == 0;
2609 }
2610
2611 #ifdef CONFIG_DEBUG_FS
2612 #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,17)
2613 /* Old kernels don't have debugfs_create_blob */
2614 static ssize_t read_file_blob(struct file *file, char __user *user_buf,
2615 size_t count, loff_t *ppos)
2616 {
2617 struct debugfs_blob_wrapper *blob = file->private_data;
2618 return simple_read_from_buffer(user_buf, count, ppos, blob->data,
2619 blob->size);
2620 }
2621
2622 static int blob_open(struct inode *inode, struct file *file)
2623 {
2624 if (inode->i_private)
2625 file->private_data = inode->i_private;
2626 return 0;
2627 }
2628
2629 static struct file_operations fops_blob = {
2630 .read = read_file_blob,
2631 .open = blob_open,
2632 };
2633
2634 static struct dentry *debugfs_create_blob(const char *name, mode_t mode,
2635 struct dentry *parent,
2636 struct debugfs_blob_wrapper *blob)
2637 {
2638 return debugfs_create_file(name, mode, parent, blob, &fops_blob);
2639 }
2640 #endif /* LINUX_VERSION_CODE */
2641
2642 static void clear_debug_buf(struct update_bundle *bundle)
2643 {
2644 if (bundle->debugfs_dentry == NULL)
2645 return;
2646 debugfs_remove(bundle->debugfs_dentry);
2647 bundle->debugfs_dentry = NULL;
2648 bundle->debug_blob.size = 0;
2649 vfree(bundle->debug_blob.data);
2650 bundle->debug_blob.data = NULL;
2651 }
2652
2653 static abort_t init_debug_buf(struct update_bundle *bundle)
2654 {
2655 bundle->debug_blob.size = 0;
2656 bundle->debug_blob.data = NULL;
2657 bundle->debugfs_dentry =
2658 debugfs_create_blob(bundle->name, S_IFREG | S_IRUSR, NULL,
2659 &bundle->debug_blob);
2660 if (bundle->debugfs_dentry == NULL)
2661 return OUT_OF_MEMORY;
2662 return OK;
2663 }
2664
2665 static int __ksdebug(struct update_bundle *bundle, const char *fmt, ...)
2666 {
2667 va_list args;
2668 unsigned long size, old_size, new_size;
2669
2670 if ((bundle->debug_blob.data == NULL ||
2671 ((char *)bundle->debug_blob.data)[bundle->debug_blob.size - 1] ==
2672 '\n') && strlen(fmt) >= 3 && fmt[0] == '<' && fmt[1] >= '0' &&
2673 fmt[1] <= '7' && fmt[2] == '>')
2674 fmt += 3;
2675
2676 /* size includes the trailing '\0' */
2677 va_start(args, fmt);
2678 size = 1 + vsnprintf(bundle->debug_blob.data, 0, fmt, args);
2679 va_end(args);
2680 old_size = bundle->debug_blob.size == 0 ? 0 :
2681 max(PAGE_SIZE, roundup_pow_of_two(bundle->debug_blob.size));
2682 new_size = bundle->debug_blob.size + size == 0 ? 0 :
2683 max(PAGE_SIZE, roundup_pow_of_two(bundle->debug_blob.size + size));
2684 if (new_size > old_size) {
2685 char *buf = vmalloc(new_size);
2686 if (buf == NULL)
2687 return -ENOMEM;
2688 memcpy(buf, bundle->debug_blob.data, bundle->debug_blob.size);
2689 vfree(bundle->debug_blob.data);
2690 bundle->debug_blob.data = buf;
2691 }
2692 va_start(args, fmt);
2693 bundle->debug_blob.size += vsnprintf(bundle->debug_blob.data +
2694 bundle->debug_blob.size,
2695 size, fmt, args);
2696 va_end(args);
2697 return 0;
2698 }
2699 #endif /* CONFIG_DEBUG_FS */
2700
2701 #ifdef KSPLICE_STANDALONE
2702 static int debug;
2703 module_param(debug, int, 0600);
2704 MODULE_PARM_DESC(debug, "Debug level");
2705
2706 static struct module_pack ksplice_pack = {
2707 .name = "ksplice_" STR(KSPLICE_KID),
2708 .kid = "init_" STR(KSPLICE_KID),
2709 .target_name = NULL,
2710 .target = NULL,
2711 .map_printk = MAP_PRINTK,
2712 .primary = THIS_MODULE,
2713 .reloc_namevals = LIST_HEAD_INIT(ksplice_pack.reloc_namevals),
2714 };
2715 #endif /* KSPLICE_STANDALONE */
2716
2717 static int init_ksplice(void)
2718 {
2719 #ifdef KSPLICE_STANDALONE
2720 struct module_pack *pack = &ksplice_pack;
2721 pack->bundle = init_ksplice_bundle(pack->kid);
2722 if (pack->bundle == NULL)
2723 return -ENOMEM;
2724 add_to_bundle(pack, pack->bundle);
2725 pack->bundle->debug = debug;
2726 pack->bundle->abort_cause =
2727 process_primary_relocs(pack, ksplice_init_relocs,
2728 ksplice_init_relocs_end);
2729 if (pack->bundle->abort_cause == OK)
2730 bootstrapped = 1;
2731 #else /* !KSPLICE_STANDALONE */
2732 ksplice_kobj = kobject_create_and_add("ksplice", kernel_kobj);
2733 if (ksplice_kobj == NULL)
2734 return -ENOMEM;
2735 #endif /* KSPLICE_STANDALONE */
2736 return 0;
2737 }
2738
2739 static void cleanup_ksplice(void)
2740 {
2741 #ifdef KSPLICE_STANDALONE
2742 cleanup_ksplice_bundle(ksplice_pack.bundle);
2743 #else /* !KSPLICE_STANDALONE */
2744 kobject_put(ksplice_kobj);
2745 #endif /* KSPLICE_STANDALONE */
2746 }
2747
2748 module_init(init_ksplice);
2749 module_exit(cleanup_ksplice);
2750
2751 MODULE_AUTHOR("Jeffrey Brian Arnold <jbarnold@mit.edu>");
2752 MODULE_DESCRIPTION("Ksplice rebootless update system");
2753 #ifdef KSPLICE_VERSION
2754 MODULE_VERSION(KSPLICE_VERSION);
2755 #endif
2756 MODULE_LICENSE("GPL v2");
Mirror of ksplice's git repository