arch/x86/kernel/alternative.c

   1 #include <linux/module.h>
   2 #include <linux/sched.h>
   3 #include <linux/mutex.h>
   4 #include <linux/list.h>
   5 #include <linux/stringify.h>
   6 #include <linux/kprobes.h>
   7 #include <linux/mm.h>
   8 #include <linux/vmalloc.h>
   9 #include <linux/memory.h>
  10 #include <linux/stop_machine.h>
  11 #include <linux/slab.h>
  12 #include <asm/alternative.h>
  13 #include <asm/sections.h>
  14 #include <asm/pgtable.h>
  15 #include <asm/mce.h>
  16 #include <asm/nmi.h>
  17 #include <asm/vsyscall.h>
  18 #include <asm/cacheflush.h>
  19 #include <asm/tlbflush.h>
  20 #include <asm/io.h>
  21 #include <asm/fixmap.h>
  22
  23 #define MAX_PATCH_LEN (255-1)
  24
  25 #ifdef CONFIG_HOTPLUG_CPU
  26 static int smp_alt_once;
  27
  28 static int __init bootonly(char *str)
  29 {
  30         smp_alt_once = 1;
  31         return 1;
  32 }
  33 __setup("smp-alt-boot", bootonly);
  34 #else
  35 #define smp_alt_once 1
  36 #endif
  37
  38 static int __initdata_or_module debug_alternative;
  39
  40 static int __init debug_alt(char *str)
  41 {
  42         debug_alternative = 1;
  43         return 1;
  44 }
  45 __setup("debug-alternative", debug_alt);
  46
  47 static int noreplace_smp;
  48
  49 static int __init setup_noreplace_smp(char *str)
  50 {
  51         noreplace_smp = 1;
  52         return 1;
  53 }
  54 __setup("noreplace-smp", setup_noreplace_smp);
  55
  56 #ifdef CONFIG_PARAVIRT
  57 static int __initdata_or_module noreplace_paravirt = 0;
  58
  59 static int __init setup_noreplace_paravirt(char *str)
  60 {
  61         noreplace_paravirt = 1;
  62         return 1;
  63 }
  64 __setup("noreplace-paravirt", setup_noreplace_paravirt);
  65 #endif
  66
  67 #define DPRINTK(fmt, args...) if (debug_alternative) \
  68         printk(KERN_DEBUG fmt, args)
  69
  70 #if defined(GENERIC_NOP1) && !defined(CONFIG_X86_64)
  71 /* Use inline assembly to define this because the nops are defined
  72    as inline assembly strings in the include files and we cannot
  73    get them easily into strings. */
  74 asm("\t" __stringify(__INITRODATA_OR_MODULE) "\nintelnops: "
  75         GENERIC_NOP1 GENERIC_NOP2 GENERIC_NOP3 GENERIC_NOP4 GENERIC_NOP5 GENERIC_NOP6
  76         GENERIC_NOP7 GENERIC_NOP8
  77     "\t.previous");
  78 extern const unsigned char intelnops[];
  79 static const unsigned char *const __initconst_or_module
  80 intel_nops[ASM_NOP_MAX+1] = {
  81         NULL,
  82         intelnops,
  83         intelnops + 1,
  84         intelnops + 1 + 2,
  85         intelnops + 1 + 2 + 3,
  86         intelnops + 1 + 2 + 3 + 4,
  87         intelnops + 1 + 2 + 3 + 4 + 5,
  88         intelnops + 1 + 2 + 3 + 4 + 5 + 6,
  89         intelnops + 1 + 2 + 3 + 4 + 5 + 6 + 7,
  90 };
  91 #endif
  92
  93 #ifdef K8_NOP1
  94 asm("\t" __stringify(__INITRODATA_OR_MODULE) "\nk8nops: "
  95         K8_NOP1 K8_NOP2 K8_NOP3 K8_NOP4 K8_NOP5 K8_NOP6
  96         K8_NOP7 K8_NOP8
  97     "\t.previous");
  98 extern const unsigned char k8nops[];
  99 static const unsigned char *const __initconst_or_module
 100 k8_nops[ASM_NOP_MAX+1] = {
 101         NULL,
 102         k8nops,
 103         k8nops + 1,
 104         k8nops + 1 + 2,
 105         k8nops + 1 + 2 + 3,
 106         k8nops + 1 + 2 + 3 + 4,
 107         k8nops + 1 + 2 + 3 + 4 + 5,
 108         k8nops + 1 + 2 + 3 + 4 + 5 + 6,
 109         k8nops + 1 + 2 + 3 + 4 + 5 + 6 + 7,
 110 };
 111 #endif
 112
 113 #if defined(K7_NOP1) && !defined(CONFIG_X86_64)
 114 asm("\t" __stringify(__INITRODATA_OR_MODULE) "\nk7nops: "
 115         K7_NOP1 K7_NOP2 K7_NOP3 K7_NOP4 K7_NOP5 K7_NOP6
 116         K7_NOP7 K7_NOP8
 117     "\t.previous");
 118 extern const unsigned char k7nops[];
 119 static const unsigned char *const __initconst_or_module
 120 k7_nops[ASM_NOP_MAX+1] = {
 121         NULL,
 122         k7nops,
 123         k7nops + 1,
 124         k7nops + 1 + 2,
 125         k7nops + 1 + 2 + 3,
 126         k7nops + 1 + 2 + 3 + 4,
 127         k7nops + 1 + 2 + 3 + 4 + 5,
 128         k7nops + 1 + 2 + 3 + 4 + 5 + 6,
 129         k7nops + 1 + 2 + 3 + 4 + 5 + 6 + 7,
 130 };
 131 #endif
 132
 133 #ifdef P6_NOP1
 134 asm("\t" __stringify(__INITRODATA_OR_MODULE) "\np6nops: "
 135         P6_NOP1 P6_NOP2 P6_NOP3 P6_NOP4 P6_NOP5 P6_NOP6
 136         P6_NOP7 P6_NOP8
 137     "\t.previous");
 138 extern const unsigned char p6nops[];
 139 static const unsigned char *const __initconst_or_module
 140 p6_nops[ASM_NOP_MAX+1] = {
 141         NULL,
 142         p6nops,
 143         p6nops + 1,
 144         p6nops + 1 + 2,
 145         p6nops + 1 + 2 + 3,
 146         p6nops + 1 + 2 + 3 + 4,
 147         p6nops + 1 + 2 + 3 + 4 + 5,
 148         p6nops + 1 + 2 + 3 + 4 + 5 + 6,
 149         p6nops + 1 + 2 + 3 + 4 + 5 + 6 + 7,
 150 };
 151 #endif
 152
 153 #ifdef CONFIG_X86_64
 154
 155 extern char __vsyscall_0;
 156 static const unsigned char *const *__init_or_module find_nop_table(void)
 157 {
 158         if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL &&
 159             boot_cpu_has(X86_FEATURE_NOPL))
 160                 return p6_nops;
 161         else
 162                 return k8_nops;
 163 }
 164
 165 #else /* CONFIG_X86_64 */
 166
 167 static const unsigned char *const *__init_or_module find_nop_table(void)
 168 {
 169         if (boot_cpu_has(X86_FEATURE_K8))
 170                 return k8_nops;
 171         else if (boot_cpu_has(X86_FEATURE_K7))
 172                 return k7_nops;
 173         else if (boot_cpu_has(X86_FEATURE_NOPL))
 174                 return p6_nops;
 175         else
 176                 return intel_nops;
 177 }
 178
 179 #endif /* CONFIG_X86_64 */
 180
 181 /* Use this to add nops to a buffer, then text_poke the whole buffer. */
 182 static void __init_or_module add_nops(void *insns, unsigned int len)
 183 {
 184         const unsigned char *const *noptable = find_nop_table();
 185
 186         while (len > 0) {
 187                 unsigned int noplen = len;
 188                 if (noplen > ASM_NOP_MAX)
 189                         noplen = ASM_NOP_MAX;
 190                 memcpy(insns, noptable[noplen], noplen);
 191                 insns += noplen;
 192                 len -= noplen;
 193         }
 194 }
 195
 196 extern struct alt_instr __alt_instructions[], __alt_instructions_end[];
 197 extern s32 __smp_locks[], __smp_locks_end[];
 198 void *text_poke_early(void *addr, const void *opcode, size_t len);
 199
 200 /* Replace instructions with better alternatives for this CPU type.
 201    This runs before SMP is initialized to avoid SMP problems with
 202    self modifying code. This implies that assymetric systems where
 203    APs have less capabilities than the boot processor are not handled.
 204    Tough. Make sure you disable such features by hand. */
 205
 206 void __init_or_module apply_alternatives(struct alt_instr *start,
 207                                          struct alt_instr *end)
 208 {
 209         struct alt_instr *a;
 210         u8 insnbuf[MAX_PATCH_LEN];
 211
 212         DPRINTK("%s: alt table %p -> %p\n", __func__, start, end);
 213         for (a = start; a < end; a++) {
 214                 u8 *instr = a->instr;
 215                 BUG_ON(a->replacementlen > a->instrlen);
 216                 BUG_ON(a->instrlen > sizeof(insnbuf));
 217                 BUG_ON(a->cpuid >= NCAPINTS*32);
 218                 if (!boot_cpu_has(a->cpuid))
 219                         continue;
 220 #ifdef CONFIG_X86_64
 221                 /* vsyscall code is not mapped yet. resolve it manually. */
 222                 if (instr >= (u8 *)VSYSCALL_START && instr < (u8*)VSYSCALL_END) {
 223                         instr = __va(instr - (u8*)VSYSCALL_START + (u8*)__pa_symbol(&__vsyscall_0));
 224                         DPRINTK("%s: vsyscall fixup: %p => %p\n",
 225                                 __func__, a->instr, instr);
 226                 }
 227 #endif
 228                 memcpy(insnbuf, a->replacement, a->replacementlen);
 229                 if (*insnbuf == 0xe8 && a->replacementlen == 5)
 230                     *(s32 *)(insnbuf + 1) += a->replacement - a->instr;
 231                 add_nops(insnbuf + a->replacementlen,
 232                          a->instrlen - a->replacementlen);
 233                 text_poke_early(instr, insnbuf, a->instrlen);
 234         }
 235 }
 236
 237 #ifdef CONFIG_SMP
 238
 239 static void alternatives_smp_lock(const s32 *start, const s32 *end,
 240                                   u8 *text, u8 *text_end)
 241 {
 242         const s32 *poff;
 243
 244         mutex_lock(&text_mutex);
 245         for (poff = start; poff < end; poff++) {
 246                 u8 *ptr = (u8 *)poff + *poff;
 247
 248                 if (!*poff || ptr < text || ptr >= text_end)
 249                         continue;
 250                 /* turn DS segment override prefix into lock prefix */
 251                 if (*ptr == 0x3e)
 252                         text_poke(ptr, ((unsigned char []){0xf0}), 1);
 253         };
 254         mutex_unlock(&text_mutex);
 255 }
 256
 257 static void alternatives_smp_unlock(const s32 *start, const s32 *end,
 258                                     u8 *text, u8 *text_end)
 259 {
 260         const s32 *poff;
 261
 262         if (noreplace_smp)
 263                 return;
 264
 265         mutex_lock(&text_mutex);
 266         for (poff = start; poff < end; poff++) {
 267                 u8 *ptr = (u8 *)poff + *poff;
 268
 269                 if (!*poff || ptr < text || ptr >= text_end)
 270                         continue;
 271                 /* turn lock prefix into DS segment override prefix */
 272                 if (*ptr == 0xf0)
 273                         text_poke(ptr, ((unsigned char []){0x3E}), 1);
 274         };
 275         mutex_unlock(&text_mutex);
 276 }
 277
 278 struct smp_alt_module {
 279         /* what is this ??? */
 280         struct module   *mod;
 281         char            *name;
 282
 283         /* ptrs to lock prefixes */
 284         const s32       *locks;
 285         const s32       *locks_end;
 286
 287         /* .text segment, needed to avoid patching init code ;) */
 288         u8              *text;
 289         u8              *text_end;
 290
 291         struct list_head next;
 292 };
 293 static LIST_HEAD(smp_alt_modules);
 294 static DEFINE_MUTEX(smp_alt);
 295 static int smp_mode = 1;        /* protected by smp_alt */
 296
 297 void __init_or_module alternatives_smp_module_add(struct module *mod,
 298                                                   char *name,
 299                                                   void *locks, void *locks_end,
 300                                                   void *text,  void *text_end)
 301 {
 302         struct smp_alt_module *smp;
 303
 304         if (noreplace_smp)
 305                 return;
 306
 307         if (smp_alt_once) {
 308                 if (boot_cpu_has(X86_FEATURE_UP))
 309                         alternatives_smp_unlock(locks, locks_end,
 310                                                 text, text_end);
 311                 return;
 312         }
 313
 314         smp = kzalloc(sizeof(*smp), GFP_KERNEL);
 315         if (NULL == smp)
 316                 return; /* we'll run the (safe but slow) SMP code then ... */
 317
 318         smp->mod        = mod;
 319         smp->name       = name;
 320         smp->locks      = locks;
 321         smp->locks_end  = locks_end;
 322         smp->text       = text;
 323         smp->text_end   = text_end;
 324         DPRINTK("%s: locks %p -> %p, text %p -> %p, name %s\n",
 325                 __func__, smp->locks, smp->locks_end,
 326                 smp->text, smp->text_end, smp->name);
 327
 328         mutex_lock(&smp_alt);
 329         list_add_tail(&smp->next, &smp_alt_modules);
 330         if (boot_cpu_has(X86_FEATURE_UP))
 331                 alternatives_smp_unlock(smp->locks, smp->locks_end,
 332                                         smp->text, smp->text_end);
 333         mutex_unlock(&smp_alt);
 334 }
 335
 336 void __init_or_module alternatives_smp_module_del(struct module *mod)
 337 {
 338         struct smp_alt_module *item;
 339
 340         if (smp_alt_once || noreplace_smp)
 341                 return;
 342
 343         mutex_lock(&smp_alt);
 344         list_for_each_entry(item, &smp_alt_modules, next) {
 345                 if (mod != item->mod)
 346                         continue;
 347                 list_del(&item->next);
 348                 mutex_unlock(&smp_alt);
 349                 DPRINTK("%s: %s\n", __func__, item->name);
 350                 kfree(item);
 351                 return;
 352         }
 353         mutex_unlock(&smp_alt);
 354 }
 355
 356 bool skip_smp_alternatives;
 357 void alternatives_smp_switch(int smp)
 358 {
 359         struct smp_alt_module *mod;
 360
 361 #ifdef CONFIG_LOCKDEP
 362         /*
 363          * Older binutils section handling bug prevented
 364          * alternatives-replacement from working reliably.
 365          *
 366          * If this still occurs then you should see a hang
 367          * or crash shortly after this line:
 368          */
 369         printk("lockdep: fixing up alternatives.\n");
 370 #endif
 371
 372         if (noreplace_smp || smp_alt_once || skip_smp_alternatives)
 373                 return;
 374         BUG_ON(!smp && (num_online_cpus() > 1));
 375
 376         mutex_lock(&smp_alt);
 377
 378         /*
 379          * Avoid unnecessary switches because it forces JIT based VMs to
 380          * throw away all cached translations, which can be quite costly.
 381          */
 382         if (smp == smp_mode) {
 383                 /* nothing */
 384         } else if (smp) {
 385                 printk(KERN_INFO "SMP alternatives: switching to SMP code\n");
 386                 clear_cpu_cap(&boot_cpu_data, X86_FEATURE_UP);
 387                 clear_cpu_cap(&cpu_data(0), X86_FEATURE_UP);
 388                 list_for_each_entry(mod, &smp_alt_modules, next)
 389                         alternatives_smp_lock(mod->locks, mod->locks_end,
 390                                               mod->text, mod->text_end);
 391         } else {
 392                 printk(KERN_INFO "SMP alternatives: switching to UP code\n");
 393                 set_cpu_cap(&boot_cpu_data, X86_FEATURE_UP);
 394                 set_cpu_cap(&cpu_data(0), X86_FEATURE_UP);
 395                 list_for_each_entry(mod, &smp_alt_modules, next)
 396                         alternatives_smp_unlock(mod->locks, mod->locks_end,
 397                                                 mod->text, mod->text_end);
 398         }
 399         smp_mode = smp;
 400         mutex_unlock(&smp_alt);
 401 }
 402
 403 /* Return 1 if the address range is reserved for smp-alternatives */
 404 int alternatives_text_reserved(void *start, void *end)
 405 {
 406         struct smp_alt_module *mod;
 407         const s32 *poff;
 408         u8 *text_start = start;
 409         u8 *text_end = end;
 410
 411         list_for_each_entry(mod, &smp_alt_modules, next) {
 412                 if (mod->text > text_end || mod->text_end < text_start)
 413                         continue;
 414                 for (poff = mod->locks; poff < mod->locks_end; poff++) {
 415                         const u8 *ptr = (const u8 *)poff + *poff;
 416
 417                         if (text_start <= ptr && text_end > ptr)
 418                                 return 1;
 419                 }
 420         }
 421
 422         return 0;
 423 }
 424 #endif
 425
 426 #ifdef CONFIG_PARAVIRT
 427 void __init_or_module apply_paravirt(struct paravirt_patch_site *start,
 428                                      struct paravirt_patch_site *end)
 429 {
 430         struct paravirt_patch_site *p;
 431         char insnbuf[MAX_PATCH_LEN];
 432
 433         if (noreplace_paravirt)
 434                 return;
 435
 436         for (p = start; p < end; p++) {
 437                 unsigned int used;
 438
 439                 BUG_ON(p->len > MAX_PATCH_LEN);
 440                 /* prep the buffer with the original instructions */
 441                 memcpy(insnbuf, p->instr, p->len);
 442                 used = pv_init_ops.patch(p->instrtype, p->clobbers, insnbuf,
 443                                          (unsigned long)p->instr, p->len);
 444
 445                 BUG_ON(used > p->len);
 446
 447                 /* Pad the rest with nops */
 448                 add_nops(insnbuf + used, p->len - used);
 449                 text_poke_early(p->instr, insnbuf, p->len);
 450         }
 451 }
 452 extern struct paravirt_patch_site __start_parainstructions[],
 453         __stop_parainstructions[];
 454 #endif  /* CONFIG_PARAVIRT */
 455
 456 void __init alternative_instructions(void)
 457 {
 458         /* The patching is not fully atomic, so try to avoid local interruptions
 459            that might execute the to be patched code.
 460            Other CPUs are not running. */
 461         stop_nmi();
 462
 463         /*
 464          * Don't stop machine check exceptions while patching.
 465          * MCEs only happen when something got corrupted and in this
 466          * case we must do something about the corruption.
 467          * Ignoring it is worse than a unlikely patching race.
 468          * Also machine checks tend to be broadcast and if one CPU
 469          * goes into machine check the others follow quickly, so we don't
 470          * expect a machine check to cause undue problems during to code
 471          * patching.
 472          */
 473
 474         apply_alternatives(__alt_instructions, __alt_instructions_end);
 475
 476         /* switch to patch-once-at-boottime-only mode and free the
 477          * tables in case we know the number of CPUs will never ever
 478          * change */
 479 #ifdef CONFIG_HOTPLUG_CPU
 480         if (num_possible_cpus() < 2)
 481                 smp_alt_once = 1;
 482 #endif
 483
 484 #ifdef CONFIG_SMP
 485         if (smp_alt_once) {
 486                 if (1 == num_possible_cpus()) {
 487                         printk(KERN_INFO "SMP alternatives: switching to UP code\n");
 488                         set_cpu_cap(&boot_cpu_data, X86_FEATURE_UP);
 489                         set_cpu_cap(&cpu_data(0), X86_FEATURE_UP);
 490
 491                         alternatives_smp_unlock(__smp_locks, __smp_locks_end,
 492                                                 _text, _etext);
 493                 }
 494         } else {
 495                 alternatives_smp_module_add(NULL, "core kernel",
 496                                             __smp_locks, __smp_locks_end,
 497                                             _text, _etext);
 498
 499                 /* Only switch to UP mode if we don't immediately boot others */
 500                 if (num_present_cpus() == 1 || setup_max_cpus <= 1)
 501                         alternatives_smp_switch(0);
 502         }
 503 #endif
 504         apply_paravirt(__parainstructions, __parainstructions_end);
 505
 506         if (smp_alt_once)
 507                 free_init_pages("SMP alternatives",
 508                                 (unsigned long)__smp_locks,
 509                                 (unsigned long)__smp_locks_end);
 510
 511         restart_nmi();
 512 }
 513
 514 /**
 515  * text_poke_early - Update instructions on a live kernel at boot time
 516  * @addr: address to modify
 517  * @opcode: source of the copy
 518  * @len: length to copy
 519  *
 520  * When you use this code to patch more than one byte of an instruction
 521  * you need to make sure that other CPUs cannot execute this code in parallel.
 522  * Also no thread must be currently preempted in the middle of these
 523  * instructions. And on the local CPU you need to be protected again NMI or MCE
 524  * handlers seeing an inconsistent instruction while you patch.
 525  */
 526 void *__init_or_module text_poke_early(void *addr, const void *opcode,
 527                                               size_t len)
 528 {
 529         unsigned long flags;
 530         local_irq_save(flags);
 531         memcpy(addr, opcode, len);
 532         sync_core();
 533         local_irq_restore(flags);
 534         /* Could also do a CLFLUSH here to speed up CPU recovery; but
 535            that causes hangs on some VIA CPUs. */
 536         return addr;
 537 }
 538
 539 /**
 540  * text_poke - Update instructions on a live kernel
 541  * @addr: address to modify
 542  * @opcode: source of the copy
 543  * @len: length to copy
 544  *
 545  * Only atomic text poke/set should be allowed when not doing early patching.
 546  * It means the size must be writable atomically and the address must be aligned
 547  * in a way that permits an atomic write. It also makes sure we fit on a single
 548  * page.
 549  *
 550  * Note: Must be called under text_mutex.
 551  */
 552 void *__kprobes text_poke(void *addr, const void *opcode, size_t len)
 553 {
 554         unsigned long flags;
 555         char *vaddr;
 556         struct page *pages[2];
 557         int i;
 558
 559         if (!core_kernel_text((unsigned long)addr)) {
 560                 pages[0] = vmalloc_to_page(addr);
 561                 pages[1] = vmalloc_to_page(addr + PAGE_SIZE);
 562         } else {
 563                 pages[0] = virt_to_page(addr);
 564                 WARN_ON(!PageReserved(pages[0]));
 565                 pages[1] = virt_to_page(addr + PAGE_SIZE);
 566         }
 567         BUG_ON(!pages[0]);
 568         local_irq_save(flags);
 569         set_fixmap(FIX_TEXT_POKE0, page_to_phys(pages[0]));
 570         if (pages[1])
 571                 set_fixmap(FIX_TEXT_POKE1, page_to_phys(pages[1]));
 572         vaddr = (char *)fix_to_virt(FIX_TEXT_POKE0);
 573         memcpy(&vaddr[(unsigned long)addr & ~PAGE_MASK], opcode, len);
 574         clear_fixmap(FIX_TEXT_POKE0);
 575         if (pages[1])
 576                 clear_fixmap(FIX_TEXT_POKE1);
 577         local_flush_tlb();
 578         sync_core();
 579         /* Could also do a CLFLUSH here to speed up CPU recovery; but
 580            that causes hangs on some VIA CPUs. */
 581         for (i = 0; i < len; i++)
 582                 BUG_ON(((char *)addr)[i] != ((char *)opcode)[i]);
 583         local_irq_restore(flags);
 584         return addr;
 585 }
 586
 587 /*
 588  * Cross-modifying kernel text with stop_machine().
 589  * This code originally comes from immediate value.
 590  */
 591 static atomic_t stop_machine_first;
 592 static int wrote_text;
 593
 594 struct text_poke_params {
 595         struct text_poke_param *params;
 596         int nparams;
 597 };
 598
 599 static int __kprobes stop_machine_text_poke(void *data)
 600 {
 601         struct text_poke_params *tpp = data;
 602         struct text_poke_param *p;
 603         int i;
 604
 605         if (atomic_dec_and_test(&stop_machine_first)) {
 606                 for (i = 0; i < tpp->nparams; i++) {
 607                         p = &tpp->params[i];
 608                         text_poke(p->addr, p->opcode, p->len);
 609                 }
 610                 smp_wmb();      /* Make sure other cpus see that this has run */
 611                 wrote_text = 1;
 612         } else {
 613                 while (!wrote_text)
 614                         cpu_relax();
 615                 smp_mb();       /* Load wrote_text before following execution */
 616         }
 617
 618         for (i = 0; i < tpp->nparams; i++) {
 619                 p = &tpp->params[i];
 620                 flush_icache_range((unsigned long)p->addr,
 621                                    (unsigned long)p->addr + p->len);
 622         }
 623
 624         return 0;
 625 }
 626
 627 /**
 628  * text_poke_smp - Update instructions on a live kernel on SMP
 629  * @addr: address to modify
 630  * @opcode: source of the copy
 631  * @len: length to copy
 632  *
 633  * Modify multi-byte instruction by using stop_machine() on SMP. This allows
 634  * user to poke/set multi-byte text on SMP. Only non-NMI/MCE code modifying
 635  * should be allowed, since stop_machine() does _not_ protect code against
 636  * NMI and MCE.
 637  *
 638  * Note: Must be called under get_online_cpus() and text_mutex.
 639  */
 640 void *__kprobes text_poke_smp(void *addr, const void *opcode, size_t len)
 641 {
 642         struct text_poke_params tpp;
 643         struct text_poke_param p;
 644
 645         p.addr = addr;
 646         p.opcode = opcode;
 647         p.len = len;
 648         tpp.params = &p;
 649         tpp.nparams = 1;
 650         atomic_set(&stop_machine_first, 1);
 651         wrote_text = 0;
 652         /* Use __stop_machine() because the caller already got online_cpus. */
 653         __stop_machine(stop_machine_text_poke, (void *)&tpp, cpu_online_mask);
 654         return addr;
 655 }
 656
 657 /**
 658  * text_poke_smp_batch - Update instructions on a live kernel on SMP
 659  * @params: an array of text_poke parameters
 660  * @n: the number of elements in params.
 661  *
 662  * Modify multi-byte instruction by using stop_machine() on SMP. Since the
 663  * stop_machine() is heavy task, it is better to aggregate text_poke requests
 664  * and do it once if possible.
 665  *
 666  * Note: Must be called under get_online_cpus() and text_mutex.
 667  */
 668 void __kprobes text_poke_smp_batch(struct text_poke_param *params, int n)
 669 {
 670         struct text_poke_params tpp = {.params = params, .nparams = n};
 671
 672         atomic_set(&stop_machine_first, 1);
 673         wrote_text = 0;
 674         stop_machine(stop_machine_text_poke, (void *)&tpp, NULL);
 675 }
 676
 677 #if defined(CONFIG_DYNAMIC_FTRACE) || defined(HAVE_JUMP_LABEL)
 678
 679 #ifdef CONFIG_X86_64
 680 unsigned char ideal_nop5[5] = { 0x66, 0x66, 0x66, 0x66, 0x90 };
 681 #else
 682 unsigned char ideal_nop5[5] = { 0x3e, 0x8d, 0x74, 0x26, 0x00 };
 683 #endif
 684
 685 void __init arch_init_ideal_nop5(void)
 686 {
 687         /*
 688          * There is no good nop for all x86 archs.  This selection
 689          * algorithm should be unified with the one in find_nop_table(),
 690          * but this should be good enough for now.
 691          *
 692          * For cases other than the ones below, use the safe (as in
 693          * always functional) defaults above.
 694          */
 695 #ifdef CONFIG_X86_64
 696         /* Don't use these on 32 bits due to broken virtualizers */
 697         if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL)
 698                 memcpy(ideal_nop5, p6_nops[5], 5);
 699 #endif
 700 }
 701 #endif