arch/x86/include/asm/paravirt.h

   1 #ifndef _ASM_X86_PARAVIRT_H
   2 #define _ASM_X86_PARAVIRT_H
   3 /* Various instructions on x86 need to be replaced for
   4  * para-virtualization: those hooks are defined here. */
   5
   6 #ifdef CONFIG_PARAVIRT
   7 #include <asm/pgtable_types.h>
   8 #include <asm/asm.h>
   9
  10 #include <asm/paravirt_types.h>
  11
  12 #ifndef __ASSEMBLY__
  13 #include <linux/types.h>
  14 #include <linux/cpumask.h>
  15
  16 static inline int paravirt_enabled(void)
  17 {
  18         return pv_info.paravirt_enabled;
  19 }
  20
  21 static inline void load_sp0(struct tss_struct *tss,
  22                              struct thread_struct *thread)
  23 {
  24         PVOP_VCALL2(pv_cpu_ops.load_sp0, tss, thread);
  25 }
  26
  27 static inline unsigned long get_wallclock(void)
  28 {
  29         return PVOP_CALL0(unsigned long, pv_time_ops.get_wallclock);
  30 }
  31
  32 static inline int set_wallclock(unsigned long nowtime)
  33 {
  34         return PVOP_CALL1(int, pv_time_ops.set_wallclock, nowtime);
  35 }
  36
  37 static inline void (*choose_time_init(void))(void)
  38 {
  39         return pv_time_ops.time_init;
  40 }
  41
  42 /* The paravirtualized CPUID instruction. */
  43 static inline void __cpuid(unsigned int *eax, unsigned int *ebx,
  44                            unsigned int *ecx, unsigned int *edx)
  45 {
  46         PVOP_VCALL4(pv_cpu_ops.cpuid, eax, ebx, ecx, edx);
  47 }
  48
  49 /*
  50  * These special macros can be used to get or set a debugging register
  51  */
  52 static inline unsigned long paravirt_get_debugreg(int reg)
  53 {
  54         return PVOP_CALL1(unsigned long, pv_cpu_ops.get_debugreg, reg);
  55 }
  56 #define get_debugreg(var, reg) var = paravirt_get_debugreg(reg)
  57 static inline void set_debugreg(unsigned long val, int reg)
  58 {
  59         PVOP_VCALL2(pv_cpu_ops.set_debugreg, reg, val);
  60 }
  61
  62 static inline void clts(void)
  63 {
  64         PVOP_VCALL0(pv_cpu_ops.clts);
  65 }
  66
  67 static inline unsigned long read_cr0(void)
  68 {
  69         return PVOP_CALL0(unsigned long, pv_cpu_ops.read_cr0);
  70 }
  71
  72 static inline void write_cr0(unsigned long x)
  73 {
  74         PVOP_VCALL1(pv_cpu_ops.write_cr0, x);
  75 }
  76
  77 static inline unsigned long read_cr2(void)
  78 {
  79         return PVOP_CALL0(unsigned long, pv_mmu_ops.read_cr2);
  80 }
  81
  82 static inline void write_cr2(unsigned long x)
  83 {
  84         PVOP_VCALL1(pv_mmu_ops.write_cr2, x);
  85 }
  86
  87 static inline unsigned long read_cr3(void)
  88 {
  89         return PVOP_CALL0(unsigned long, pv_mmu_ops.read_cr3);
  90 }
  91
  92 static inline void write_cr3(unsigned long x)
  93 {
  94         PVOP_VCALL1(pv_mmu_ops.write_cr3, x);
  95 }
  96
  97 static inline unsigned long read_cr4(void)
  98 {
  99         return PVOP_CALL0(unsigned long, pv_cpu_ops.read_cr4);
 100 }
 101 static inline unsigned long read_cr4_safe(void)
 102 {
 103         return PVOP_CALL0(unsigned long, pv_cpu_ops.read_cr4_safe);
 104 }
 105
 106 static inline void write_cr4(unsigned long x)
 107 {
 108         PVOP_VCALL1(pv_cpu_ops.write_cr4, x);
 109 }
 110
 111 #ifdef CONFIG_X86_64
 112 static inline unsigned long read_cr8(void)
 113 {
 114         return PVOP_CALL0(unsigned long, pv_cpu_ops.read_cr8);
 115 }
 116
 117 static inline void write_cr8(unsigned long x)
 118 {
 119         PVOP_VCALL1(pv_cpu_ops.write_cr8, x);
 120 }
 121 #endif
 122
 123 static inline void raw_safe_halt(void)
 124 {
 125         PVOP_VCALL0(pv_irq_ops.safe_halt);
 126 }
 127
 128 static inline void halt(void)
 129 {
 130         PVOP_VCALL0(pv_irq_ops.safe_halt);
 131 }
 132
 133 static inline void wbinvd(void)
 134 {
 135         PVOP_VCALL0(pv_cpu_ops.wbinvd);
 136 }
 137
 138 #define get_kernel_rpl()  (pv_info.kernel_rpl)
 139
 140 static inline u64 paravirt_read_msr(unsigned msr, int *err)
 141 {
 142         return PVOP_CALL2(u64, pv_cpu_ops.read_msr, msr, err);
 143 }
 144 static inline u64 paravirt_read_msr_amd(unsigned msr, int *err)
 145 {
 146         return PVOP_CALL2(u64, pv_cpu_ops.read_msr_amd, msr, err);
 147 }
 148 static inline int paravirt_write_msr(unsigned msr, unsigned low, unsigned high)
 149 {
 150         return PVOP_CALL3(int, pv_cpu_ops.write_msr, msr, low, high);
 151 }
 152
 153 /* These should all do BUG_ON(_err), but our headers are too tangled. */
 154 #define rdmsr(msr, val1, val2)                  \
 155 do {                                            \
 156         int _err;                               \
 157         u64 _l = paravirt_read_msr(msr, &_err); \
 158         val1 = (u32)_l;                         \
 159         val2 = _l >> 32;                        \
 160 } while (0)
 161
 162 #define wrmsr(msr, val1, val2)                  \
 163 do {                                            \
 164         paravirt_write_msr(msr, val1, val2);    \
 165 } while (0)
 166
 167 #define rdmsrl(msr, val)                        \
 168 do {                                            \
 169         int _err;                               \
 170         val = paravirt_read_msr(msr, &_err);    \
 171 } while (0)
 172
 173 #define wrmsrl(msr, val)        wrmsr(msr, (u32)((u64)(val)), ((u64)(val))>>32)
 174 #define wrmsr_safe(msr, a, b)   paravirt_write_msr(msr, a, b)
 175
 176 /* rdmsr with exception handling */
 177 #define rdmsr_safe(msr, a, b)                   \
 178 ({                                              \
 179         int _err;                               \
 180         u64 _l = paravirt_read_msr(msr, &_err); \
 181         (*a) = (u32)_l;                         \
 182         (*b) = _l >> 32;                        \
 183         _err;                                   \
 184 })
 185
 186 static inline int rdmsrl_safe(unsigned msr, unsigned long long *p)
 187 {
 188         int err;
 189
 190         *p = paravirt_read_msr(msr, &err);
 191         return err;
 192 }
 193 static inline int rdmsrl_amd_safe(unsigned msr, unsigned long long *p)
 194 {
 195         int err;
 196
 197         *p = paravirt_read_msr_amd(msr, &err);
 198         return err;
 199 }
 200
 201 static inline u64 paravirt_read_tsc(void)
 202 {
 203         return PVOP_CALL0(u64, pv_cpu_ops.read_tsc);
 204 }
 205
 206 #define rdtscl(low)                             \
 207 do {                                            \
 208         u64 _l = paravirt_read_tsc();           \
 209         low = (int)_l;                          \
 210 } while (0)
 211
 212 #define rdtscll(val) (val = paravirt_read_tsc())
 213
 214 static inline unsigned long long paravirt_sched_clock(void)
 215 {
 216         return PVOP_CALL0(unsigned long long, pv_time_ops.sched_clock);
 217 }
 218 #define calibrate_tsc() (pv_time_ops.get_tsc_khz())
 219
 220 static inline unsigned long long paravirt_read_pmc(int counter)
 221 {
 222         return PVOP_CALL1(u64, pv_cpu_ops.read_pmc, counter);
 223 }
 224
 225 #define rdpmc(counter, low, high)               \
 226 do {                                            \
 227         u64 _l = paravirt_read_pmc(counter);    \
 228         low = (u32)_l;                          \
 229         high = _l >> 32;                        \
 230 } while (0)
 231
 232 static inline unsigned long long paravirt_rdtscp(unsigned int *aux)
 233 {
 234         return PVOP_CALL1(u64, pv_cpu_ops.read_tscp, aux);
 235 }
 236
 237 #define rdtscp(low, high, aux)                          \
 238 do {                                                    \
 239         int __aux;                                      \
 240         unsigned long __val = paravirt_rdtscp(&__aux);  \
 241         (low) = (u32)__val;                             \
 242         (high) = (u32)(__val >> 32);                    \
 243         (aux) = __aux;                                  \
 244 } while (0)
 245
 246 #define rdtscpll(val, aux)                              \
 247 do {                                                    \
 248         unsigned long __aux;                            \
 249         val = paravirt_rdtscp(&__aux);                  \
 250         (aux) = __aux;                                  \
 251 } while (0)
 252
 253 static inline void paravirt_alloc_ldt(struct desc_struct *ldt, unsigned entries)
 254 {
 255         PVOP_VCALL2(pv_cpu_ops.alloc_ldt, ldt, entries);
 256 }
 257
 258 static inline void paravirt_free_ldt(struct desc_struct *ldt, unsigned entries)
 259 {
 260         PVOP_VCALL2(pv_cpu_ops.free_ldt, ldt, entries);
 261 }
 262
 263 static inline void load_TR_desc(void)
 264 {
 265         PVOP_VCALL0(pv_cpu_ops.load_tr_desc);
 266 }
 267 static inline void load_gdt(const struct desc_ptr *dtr)
 268 {
 269         PVOP_VCALL1(pv_cpu_ops.load_gdt, dtr);
 270 }
 271 static inline void load_idt(const struct desc_ptr *dtr)
 272 {
 273         PVOP_VCALL1(pv_cpu_ops.load_idt, dtr);
 274 }
 275 static inline void set_ldt(const void *addr, unsigned entries)
 276 {
 277         PVOP_VCALL2(pv_cpu_ops.set_ldt, addr, entries);
 278 }
 279 static inline void store_gdt(struct desc_ptr *dtr)
 280 {
 281         PVOP_VCALL1(pv_cpu_ops.store_gdt, dtr);
 282 }
 283 static inline void store_idt(struct desc_ptr *dtr)
 284 {
 285         PVOP_VCALL1(pv_cpu_ops.store_idt, dtr);
 286 }
 287 static inline unsigned long paravirt_store_tr(void)
 288 {
 289         return PVOP_CALL0(unsigned long, pv_cpu_ops.store_tr);
 290 }
 291 #define store_tr(tr)    ((tr) = paravirt_store_tr())
 292 static inline void load_TLS(struct thread_struct *t, unsigned cpu)
 293 {
 294         PVOP_VCALL2(pv_cpu_ops.load_tls, t, cpu);
 295 }
 296
 297 #ifdef CONFIG_X86_64
 298 static inline void load_gs_index(unsigned int gs)
 299 {
 300         PVOP_VCALL1(pv_cpu_ops.load_gs_index, gs);
 301 }
 302 #endif
 303
 304 static inline void write_ldt_entry(struct desc_struct *dt, int entry,
 305                                    const void *desc)
 306 {
 307         PVOP_VCALL3(pv_cpu_ops.write_ldt_entry, dt, entry, desc);
 308 }
 309
 310 static inline void write_gdt_entry(struct desc_struct *dt, int entry,
 311                                    void *desc, int type)
 312 {
 313         PVOP_VCALL4(pv_cpu_ops.write_gdt_entry, dt, entry, desc, type);
 314 }
 315
 316 static inline void write_idt_entry(gate_desc *dt, int entry, const gate_desc *g)
 317 {
 318         PVOP_VCALL3(pv_cpu_ops.write_idt_entry, dt, entry, g);
 319 }
 320 static inline void set_iopl_mask(unsigned mask)
 321 {
 322         PVOP_VCALL1(pv_cpu_ops.set_iopl_mask, mask);
 323 }
 324
 325 /* The paravirtualized I/O functions */
 326 static inline void slow_down_io(void)
 327 {
 328         pv_cpu_ops.io_delay();
 329 #ifdef REALLY_SLOW_IO
 330         pv_cpu_ops.io_delay();
 331         pv_cpu_ops.io_delay();
 332         pv_cpu_ops.io_delay();
 333 #endif
 334 }
 335
 336 #ifdef CONFIG_X86_LOCAL_APIC
 337 static inline void setup_boot_clock(void)
 338 {
 339         PVOP_VCALL0(pv_apic_ops.setup_boot_clock);
 340 }
 341
 342 static inline void setup_secondary_clock(void)
 343 {
 344         PVOP_VCALL0(pv_apic_ops.setup_secondary_clock);
 345 }
 346 #endif
 347
 348 static inline void paravirt_post_allocator_init(void)
 349 {
 350         if (pv_init_ops.post_allocator_init)
 351                 (*pv_init_ops.post_allocator_init)();
 352 }
 353
 354 static inline void paravirt_pagetable_setup_start(pgd_t *base)
 355 {
 356         (*pv_mmu_ops.pagetable_setup_start)(base);
 357 }
 358
 359 static inline void paravirt_pagetable_setup_done(pgd_t *base)
 360 {
 361         (*pv_mmu_ops.pagetable_setup_done)(base);
 362 }
 363
 364 #ifdef CONFIG_SMP
 365 static inline void startup_ipi_hook(int phys_apicid, unsigned long start_eip,
 366                                     unsigned long start_esp)
 367 {
 368         PVOP_VCALL3(pv_apic_ops.startup_ipi_hook,
 369                     phys_apicid, start_eip, start_esp);
 370 }
 371 #endif
 372
 373 static inline void paravirt_activate_mm(struct mm_struct *prev,
 374                                         struct mm_struct *next)
 375 {
 376         PVOP_VCALL2(pv_mmu_ops.activate_mm, prev, next);
 377 }
 378
 379 static inline void arch_dup_mmap(struct mm_struct *oldmm,
 380                                  struct mm_struct *mm)
 381 {
 382         PVOP_VCALL2(pv_mmu_ops.dup_mmap, oldmm, mm);
 383 }
 384
 385 static inline void arch_exit_mmap(struct mm_struct *mm)
 386 {
 387         PVOP_VCALL1(pv_mmu_ops.exit_mmap, mm);
 388 }
 389
 390 static inline void __flush_tlb(void)
 391 {
 392         PVOP_VCALL0(pv_mmu_ops.flush_tlb_user);
 393 }
 394 static inline void __flush_tlb_global(void)
 395 {
 396         PVOP_VCALL0(pv_mmu_ops.flush_tlb_kernel);
 397 }
 398 static inline void __flush_tlb_single(unsigned long addr)
 399 {
 400         PVOP_VCALL1(pv_mmu_ops.flush_tlb_single, addr);
 401 }
 402
 403 static inline void flush_tlb_others(const struct cpumask *cpumask,
 404                                     struct mm_struct *mm,
 405                                     unsigned long va)
 406 {
 407         PVOP_VCALL3(pv_mmu_ops.flush_tlb_others, cpumask, mm, va);
 408 }
 409
 410 static inline int paravirt_pgd_alloc(struct mm_struct *mm)
 411 {
 412         return PVOP_CALL1(int, pv_mmu_ops.pgd_alloc, mm);
 413 }
 414
 415 static inline void paravirt_pgd_free(struct mm_struct *mm, pgd_t *pgd)
 416 {
 417         PVOP_VCALL2(pv_mmu_ops.pgd_free, mm, pgd);
 418 }
 419
 420 static inline void paravirt_alloc_pte(struct mm_struct *mm, unsigned long pfn)
 421 {
 422         PVOP_VCALL2(pv_mmu_ops.alloc_pte, mm, pfn);
 423 }
 424 static inline void paravirt_release_pte(unsigned long pfn)
 425 {
 426         PVOP_VCALL1(pv_mmu_ops.release_pte, pfn);
 427 }
 428
 429 static inline void paravirt_alloc_pmd(struct mm_struct *mm, unsigned long pfn)
 430 {
 431         PVOP_VCALL2(pv_mmu_ops.alloc_pmd, mm, pfn);
 432 }
 433
 434 static inline void paravirt_alloc_pmd_clone(unsigned long pfn, unsigned long clonepfn,
 435                                             unsigned long start, unsigned long count)
 436 {
 437         PVOP_VCALL4(pv_mmu_ops.alloc_pmd_clone, pfn, clonepfn, start, count);
 438 }
 439 static inline void paravirt_release_pmd(unsigned long pfn)
 440 {
 441         PVOP_VCALL1(pv_mmu_ops.release_pmd, pfn);
 442 }
 443
 444 static inline void paravirt_alloc_pud(struct mm_struct *mm, unsigned long pfn)
 445 {
 446         PVOP_VCALL2(pv_mmu_ops.alloc_pud, mm, pfn);
 447 }
 448 static inline void paravirt_release_pud(unsigned long pfn)
 449 {
 450         PVOP_VCALL1(pv_mmu_ops.release_pud, pfn);
 451 }
 452
 453 #ifdef CONFIG_HIGHPTE
 454 static inline void *kmap_atomic_pte(struct page *page, enum km_type type)
 455 {
 456         unsigned long ret;
 457         ret = PVOP_CALL2(unsigned long, pv_mmu_ops.kmap_atomic_pte, page, type);
 458         return (void *)ret;
 459 }
 460 #endif
 461
 462 static inline void pte_update(struct mm_struct *mm, unsigned long addr,
 463                               pte_t *ptep)
 464 {
 465         PVOP_VCALL3(pv_mmu_ops.pte_update, mm, addr, ptep);
 466 }
 467
 468 static inline void pte_update_defer(struct mm_struct *mm, unsigned long addr,
 469                                     pte_t *ptep)
 470 {
 471         PVOP_VCALL3(pv_mmu_ops.pte_update_defer, mm, addr, ptep);
 472 }
 473
 474 static inline pte_t __pte(pteval_t val)
 475 {
 476         pteval_t ret;
 477
 478         if (sizeof(pteval_t) > sizeof(long))
 479                 ret = PVOP_CALLEE2(pteval_t,
 480                                    pv_mmu_ops.make_pte,
 481                                    val, (u64)val >> 32);
 482         else
 483                 ret = PVOP_CALLEE1(pteval_t,
 484                                    pv_mmu_ops.make_pte,
 485                                    val);
 486
 487         return (pte_t) { .pte = ret };
 488 }
 489
 490 static inline pteval_t pte_val(pte_t pte)
 491 {
 492         pteval_t ret;
 493
 494         if (sizeof(pteval_t) > sizeof(long))
 495                 ret = PVOP_CALLEE2(pteval_t, pv_mmu_ops.pte_val,
 496                                    pte.pte, (u64)pte.pte >> 32);
 497         else
 498                 ret = PVOP_CALLEE1(pteval_t, pv_mmu_ops.pte_val,
 499                                    pte.pte);
 500
 501         return ret;
 502 }
 503
 504 static inline pgd_t __pgd(pgdval_t val)
 505 {
 506         pgdval_t ret;
 507
 508         if (sizeof(pgdval_t) > sizeof(long))
 509                 ret = PVOP_CALLEE2(pgdval_t, pv_mmu_ops.make_pgd,
 510                                    val, (u64)val >> 32);
 511         else
 512                 ret = PVOP_CALLEE1(pgdval_t, pv_mmu_ops.make_pgd,
 513                                    val);
 514
 515         return (pgd_t) { ret };
 516 }
 517
 518 static inline pgdval_t pgd_val(pgd_t pgd)
 519 {
 520         pgdval_t ret;
 521
 522         if (sizeof(pgdval_t) > sizeof(long))
 523                 ret =  PVOP_CALLEE2(pgdval_t, pv_mmu_ops.pgd_val,
 524                                     pgd.pgd, (u64)pgd.pgd >> 32);
 525         else
 526                 ret =  PVOP_CALLEE1(pgdval_t, pv_mmu_ops.pgd_val,
 527                                     pgd.pgd);
 528
 529         return ret;
 530 }
 531
 532 #define  __HAVE_ARCH_PTEP_MODIFY_PROT_TRANSACTION
 533 static inline pte_t ptep_modify_prot_start(struct mm_struct *mm, unsigned long addr,
 534                                            pte_t *ptep)
 535 {
 536         pteval_t ret;
 537
 538         ret = PVOP_CALL3(pteval_t, pv_mmu_ops.ptep_modify_prot_start,
 539                          mm, addr, ptep);
 540
 541         return (pte_t) { .pte = ret };
 542 }
 543
 544 static inline void ptep_modify_prot_commit(struct mm_struct *mm, unsigned long addr,
 545                                            pte_t *ptep, pte_t pte)
 546 {
 547         if (sizeof(pteval_t) > sizeof(long))
 548                 /* 5 arg words */
 549                 pv_mmu_ops.ptep_modify_prot_commit(mm, addr, ptep, pte);
 550         else
 551                 PVOP_VCALL4(pv_mmu_ops.ptep_modify_prot_commit,
 552                             mm, addr, ptep, pte.pte);
 553 }
 554
 555 static inline void set_pte(pte_t *ptep, pte_t pte)
 556 {
 557         if (sizeof(pteval_t) > sizeof(long))
 558                 PVOP_VCALL3(pv_mmu_ops.set_pte, ptep,
 559                             pte.pte, (u64)pte.pte >> 32);
 560         else
 561                 PVOP_VCALL2(pv_mmu_ops.set_pte, ptep,
 562                             pte.pte);
 563 }
 564
 565 static inline void set_pte_at(struct mm_struct *mm, unsigned long addr,
 566                               pte_t *ptep, pte_t pte)
 567 {
 568         if (sizeof(pteval_t) > sizeof(long))
 569                 /* 5 arg words */
 570                 pv_mmu_ops.set_pte_at(mm, addr, ptep, pte);
 571         else
 572                 PVOP_VCALL4(pv_mmu_ops.set_pte_at, mm, addr, ptep, pte.pte);
 573 }
 574
 575 static inline void set_pmd(pmd_t *pmdp, pmd_t pmd)
 576 {
 577         pmdval_t val = native_pmd_val(pmd);
 578
 579         if (sizeof(pmdval_t) > sizeof(long))
 580                 PVOP_VCALL3(pv_mmu_ops.set_pmd, pmdp, val, (u64)val >> 32);
 581         else
 582                 PVOP_VCALL2(pv_mmu_ops.set_pmd, pmdp, val);
 583 }
 584
 585 #if PAGETABLE_LEVELS >= 3
 586 static inline pmd_t __pmd(pmdval_t val)
 587 {
 588         pmdval_t ret;
 589
 590         if (sizeof(pmdval_t) > sizeof(long))
 591                 ret = PVOP_CALLEE2(pmdval_t, pv_mmu_ops.make_pmd,
 592                                    val, (u64)val >> 32);
 593         else
 594                 ret = PVOP_CALLEE1(pmdval_t, pv_mmu_ops.make_pmd,
 595                                    val);
 596
 597         return (pmd_t) { ret };
 598 }
 599
 600 static inline pmdval_t pmd_val(pmd_t pmd)
 601 {
 602         pmdval_t ret;
 603
 604         if (sizeof(pmdval_t) > sizeof(long))
 605                 ret =  PVOP_CALLEE2(pmdval_t, pv_mmu_ops.pmd_val,
 606                                     pmd.pmd, (u64)pmd.pmd >> 32);
 607         else
 608                 ret =  PVOP_CALLEE1(pmdval_t, pv_mmu_ops.pmd_val,
 609                                     pmd.pmd);
 610
 611         return ret;
 612 }
 613
 614 static inline void set_pud(pud_t *pudp, pud_t pud)
 615 {
 616         pudval_t val = native_pud_val(pud);
 617
 618         if (sizeof(pudval_t) > sizeof(long))
 619                 PVOP_VCALL3(pv_mmu_ops.set_pud, pudp,
 620                             val, (u64)val >> 32);
 621         else
 622                 PVOP_VCALL2(pv_mmu_ops.set_pud, pudp,
 623                             val);
 624 }
 625 #if PAGETABLE_LEVELS == 4
 626 static inline pud_t __pud(pudval_t val)
 627 {
 628         pudval_t ret;
 629
 630         if (sizeof(pudval_t) > sizeof(long))
 631                 ret = PVOP_CALLEE2(pudval_t, pv_mmu_ops.make_pud,
 632                                    val, (u64)val >> 32);
 633         else
 634                 ret = PVOP_CALLEE1(pudval_t, pv_mmu_ops.make_pud,
 635                                    val);
 636
 637         return (pud_t) { ret };
 638 }
 639
 640 static inline pudval_t pud_val(pud_t pud)
 641 {
 642         pudval_t ret;
 643
 644         if (sizeof(pudval_t) > sizeof(long))
 645                 ret =  PVOP_CALLEE2(pudval_t, pv_mmu_ops.pud_val,
 646                                     pud.pud, (u64)pud.pud >> 32);
 647         else
 648                 ret =  PVOP_CALLEE1(pudval_t, pv_mmu_ops.pud_val,
 649                                     pud.pud);
 650
 651         return ret;
 652 }
 653
 654 static inline void set_pgd(pgd_t *pgdp, pgd_t pgd)
 655 {
 656         pgdval_t val = native_pgd_val(pgd);
 657
 658         if (sizeof(pgdval_t) > sizeof(long))
 659                 PVOP_VCALL3(pv_mmu_ops.set_pgd, pgdp,
 660                             val, (u64)val >> 32);
 661         else
 662                 PVOP_VCALL2(pv_mmu_ops.set_pgd, pgdp,
 663                             val);
 664 }
 665
 666 static inline void pgd_clear(pgd_t *pgdp)
 667 {
 668         set_pgd(pgdp, __pgd(0));
 669 }
 670
 671 static inline void pud_clear(pud_t *pudp)
 672 {
 673         set_pud(pudp, __pud(0));
 674 }
 675
 676 #endif  /* PAGETABLE_LEVELS == 4 */
 677
 678 #endif  /* PAGETABLE_LEVELS >= 3 */
 679
 680 #ifdef CONFIG_X86_PAE
 681 /* Special-case pte-setting operations for PAE, which can't update a
 682    64-bit pte atomically */
 683 static inline void set_pte_atomic(pte_t *ptep, pte_t pte)
 684 {
 685         PVOP_VCALL3(pv_mmu_ops.set_pte_atomic, ptep,
 686                     pte.pte, pte.pte >> 32);
 687 }
 688
 689 static inline void pte_clear(struct mm_struct *mm, unsigned long addr,
 690                              pte_t *ptep)
 691 {
 692         PVOP_VCALL3(pv_mmu_ops.pte_clear, mm, addr, ptep);
 693 }
 694
 695 static inline void pmd_clear(pmd_t *pmdp)
 696 {
 697         PVOP_VCALL1(pv_mmu_ops.pmd_clear, pmdp);
 698 }
 699 #else  /* !CONFIG_X86_PAE */
 700 static inline void set_pte_atomic(pte_t *ptep, pte_t pte)
 701 {
 702         set_pte(ptep, pte);
 703 }
 704
 705 static inline void pte_clear(struct mm_struct *mm, unsigned long addr,
 706                              pte_t *ptep)
 707 {
 708         set_pte_at(mm, addr, ptep, __pte(0));
 709 }
 710
 711 static inline void pmd_clear(pmd_t *pmdp)
 712 {
 713         set_pmd(pmdp, __pmd(0));
 714 }
 715 #endif  /* CONFIG_X86_PAE */
 716
 717 #define  __HAVE_ARCH_START_CONTEXT_SWITCH
 718 static inline void arch_start_context_switch(struct task_struct *prev)
 719 {
 720         PVOP_VCALL1(pv_cpu_ops.start_context_switch, prev);
 721 }
 722
 723 static inline void arch_end_context_switch(struct task_struct *next)
 724 {
 725         PVOP_VCALL1(pv_cpu_ops.end_context_switch, next);
 726 }
 727
 728 #define  __HAVE_ARCH_ENTER_LAZY_MMU_MODE
 729 static inline void arch_enter_lazy_mmu_mode(void)
 730 {
 731         PVOP_VCALL0(pv_mmu_ops.lazy_mode.enter);
 732 }
 733
 734 static inline void arch_leave_lazy_mmu_mode(void)
 735 {
 736         PVOP_VCALL0(pv_mmu_ops.lazy_mode.leave);
 737 }
 738
 739 void arch_flush_lazy_mmu_mode(void);
 740
 741 static inline void __set_fixmap(unsigned /* enum fixed_addresses */ idx,
 742                                 phys_addr_t phys, pgprot_t flags)
 743 {
 744         pv_mmu_ops.set_fixmap(idx, phys, flags);
 745 }
 746
 747 #if defined(CONFIG_SMP) && defined(CONFIG_PARAVIRT_SPINLOCKS)
 748
 749 static inline int __raw_spin_is_locked(struct raw_spinlock *lock)
 750 {
 751         return PVOP_CALL1(int, pv_lock_ops.spin_is_locked, lock);
 752 }
 753
 754 static inline int __raw_spin_is_contended(struct raw_spinlock *lock)
 755 {
 756         return PVOP_CALL1(int, pv_lock_ops.spin_is_contended, lock);
 757 }
 758 #define __raw_spin_is_contended __raw_spin_is_contended
 759
 760 static __always_inline void __raw_spin_lock(struct raw_spinlock *lock)
 761 {
 762         PVOP_VCALL1(pv_lock_ops.spin_lock, lock);
 763 }
 764
 765 static __always_inline void __raw_spin_lock_flags(struct raw_spinlock *lock,
 766                                                   unsigned long flags)
 767 {
 768         PVOP_VCALL2(pv_lock_ops.spin_lock_flags, lock, flags);
 769 }
 770
 771 static __always_inline int __raw_spin_trylock(struct raw_spinlock *lock)
 772 {
 773         return PVOP_CALL1(int, pv_lock_ops.spin_trylock, lock);
 774 }
 775
 776 static __always_inline void __raw_spin_unlock(struct raw_spinlock *lock)
 777 {
 778         PVOP_VCALL1(pv_lock_ops.spin_unlock, lock);
 779 }
 780
 781 #endif
 782
 783 #ifdef CONFIG_X86_32
 784 #define PV_SAVE_REGS "pushl %ecx; pushl %edx;"
 785 #define PV_RESTORE_REGS "popl %edx; popl %ecx;"
 786
 787 /* save and restore all caller-save registers, except return value */
 788 #define PV_SAVE_ALL_CALLER_REGS         "pushl %ecx;"
 789 #define PV_RESTORE_ALL_CALLER_REGS      "popl  %ecx;"
 790
 791 #define PV_FLAGS_ARG "0"
 792 #define PV_EXTRA_CLOBBERS
 793 #define PV_VEXTRA_CLOBBERS
 794 #else
 795 /* save and restore all caller-save registers, except return value */
 796 #define PV_SAVE_ALL_CALLER_REGS                                         \
 797         "push %rcx;"                                                    \
 798         "push %rdx;"                                                    \
 799         "push %rsi;"                                                    \
 800         "push %rdi;"                                                    \
 801         "push %r8;"                                                     \
 802         "push %r9;"                                                     \
 803         "push %r10;"                                                    \
 804         "push %r11;"
 805 #define PV_RESTORE_ALL_CALLER_REGS                                      \
 806         "pop %r11;"                                                     \
 807         "pop %r10;"                                                     \
 808         "pop %r9;"                                                      \
 809         "pop %r8;"                                                      \
 810         "pop %rdi;"                                                     \
 811         "pop %rsi;"                                                     \
 812         "pop %rdx;"                                                     \
 813         "pop %rcx;"
 814
 815 /* We save some registers, but all of them, that's too much. We clobber all
 816  * caller saved registers but the argument parameter */
 817 #define PV_SAVE_REGS "pushq %%rdi;"
 818 #define PV_RESTORE_REGS "popq %%rdi;"
 819 #define PV_EXTRA_CLOBBERS EXTRA_CLOBBERS, "rcx" , "rdx", "rsi"
 820 #define PV_VEXTRA_CLOBBERS EXTRA_CLOBBERS, "rdi", "rcx" , "rdx", "rsi"
 821 #define PV_FLAGS_ARG "D"
 822 #endif
 823
 824 /*
 825  * Generate a thunk around a function which saves all caller-save
 826  * registers except for the return value.  This allows C functions to
 827  * be called from assembler code where fewer than normal registers are
 828  * available.  It may also help code generation around calls from C
 829  * code if the common case doesn't use many registers.
 830  *
 831  * When a callee is wrapped in a thunk, the caller can assume that all
 832  * arg regs and all scratch registers are preserved across the
 833  * call. The return value in rax/eax will not be saved, even for void
 834  * functions.
 835  */
 836 #define PV_CALLEE_SAVE_REGS_THUNK(func)                                 \
 837         extern typeof(func) __raw_callee_save_##func;                   \
 838         static void *__##func##__ __used = func;                        \
 839                                                                         \
 840         asm(".pushsection .text;"                                       \
 841             "__raw_callee_save_" #func ": "                             \
 842             PV_SAVE_ALL_CALLER_REGS                                     \
 843             "call " #func ";"                                           \
 844             PV_RESTORE_ALL_CALLER_REGS                                  \
 845             "ret;"                                                      \
 846             ".popsection")
 847
 848 /* Get a reference to a callee-save function */
 849 #define PV_CALLEE_SAVE(func)                                            \
 850         ((struct paravirt_callee_save) { __raw_callee_save_##func })
 851
 852 /* Promise that "func" already uses the right calling convention */
 853 #define __PV_IS_CALLEE_SAVE(func)                       \
 854         ((struct paravirt_callee_save) { func })
 855
 856 static inline unsigned long __raw_local_save_flags(void)
 857 {
 858         unsigned long f;
 859
 860         asm volatile(paravirt_alt(PARAVIRT_CALL)
 861                      : "=a"(f)
 862                      : paravirt_type(pv_irq_ops.save_fl),
 863                        paravirt_clobber(CLBR_EAX)
 864                      : "memory", "cc");
 865         return f;
 866 }
 867
 868 static inline void raw_local_irq_restore(unsigned long f)
 869 {
 870         asm volatile(paravirt_alt(PARAVIRT_CALL)
 871                      : "=a"(f)
 872                      : PV_FLAGS_ARG(f),
 873                        paravirt_type(pv_irq_ops.restore_fl),
 874                        paravirt_clobber(CLBR_EAX)
 875                      : "memory", "cc");
 876 }
 877
 878 static inline void raw_local_irq_disable(void)
 879 {
 880         asm volatile(paravirt_alt(PARAVIRT_CALL)
 881                      :
 882                      : paravirt_type(pv_irq_ops.irq_disable),
 883                        paravirt_clobber(CLBR_EAX)
 884                      : "memory", "eax", "cc");
 885 }
 886
 887 static inline void raw_local_irq_enable(void)
 888 {
 889         asm volatile(paravirt_alt(PARAVIRT_CALL)
 890                      :
 891                      : paravirt_type(pv_irq_ops.irq_enable),
 892                        paravirt_clobber(CLBR_EAX)
 893                      : "memory", "eax", "cc");
 894 }
 895
 896 static inline unsigned long __raw_local_irq_save(void)
 897 {
 898         unsigned long f;
 899
 900         f = __raw_local_save_flags();
 901         raw_local_irq_disable();
 902         return f;
 903 }
 904
 905
 906 /* Make sure as little as possible of this mess escapes. */
 907 #undef PARAVIRT_CALL
 908 #undef __PVOP_CALL
 909 #undef __PVOP_VCALL
 910 #undef PVOP_VCALL0
 911 #undef PVOP_CALL0
 912 #undef PVOP_VCALL1
 913 #undef PVOP_CALL1
 914 #undef PVOP_VCALL2
 915 #undef PVOP_CALL2
 916 #undef PVOP_VCALL3
 917 #undef PVOP_CALL3
 918 #undef PVOP_VCALL4
 919 #undef PVOP_CALL4
 920
 921 #else  /* __ASSEMBLY__ */
 922
 923 #define _PVSITE(ptype, clobbers, ops, word, algn)       \
 924 771:;                                           \
 925         ops;                                    \
 926 772:;                                           \
 927         .pushsection .parainstructions,"a";     \
 928          .align algn;                           \
 929          word 771b;                             \
 930          .byte ptype;                           \
 931          .byte 772b-771b;                       \
 932          .short clobbers;                       \
 933         .popsection
 934
 935
 936 #define COND_PUSH(set, mask, reg)                       \
 937         .if ((~(set)) & mask); push %reg; .endif
 938 #define COND_POP(set, mask, reg)                        \
 939         .if ((~(set)) & mask); pop %reg; .endif
 940
 941 #ifdef CONFIG_X86_64
 942
 943 #define PV_SAVE_REGS(set)                       \
 944         COND_PUSH(set, CLBR_RAX, rax);          \
 945         COND_PUSH(set, CLBR_RCX, rcx);          \
 946         COND_PUSH(set, CLBR_RDX, rdx);          \
 947         COND_PUSH(set, CLBR_RSI, rsi);          \
 948         COND_PUSH(set, CLBR_RDI, rdi);          \
 949         COND_PUSH(set, CLBR_R8, r8);            \
 950         COND_PUSH(set, CLBR_R9, r9);            \
 951         COND_PUSH(set, CLBR_R10, r10);          \
 952         COND_PUSH(set, CLBR_R11, r11)
 953 #define PV_RESTORE_REGS(set)                    \
 954         COND_POP(set, CLBR_R11, r11);           \
 955         COND_POP(set, CLBR_R10, r10);           \
 956         COND_POP(set, CLBR_R9, r9);             \
 957         COND_POP(set, CLBR_R8, r8);             \
 958         COND_POP(set, CLBR_RDI, rdi);           \
 959         COND_POP(set, CLBR_RSI, rsi);           \
 960         COND_POP(set, CLBR_RDX, rdx);           \
 961         COND_POP(set, CLBR_RCX, rcx);           \
 962         COND_POP(set, CLBR_RAX, rax)
 963
 964 #define PARA_PATCH(struct, off)        ((PARAVIRT_PATCH_##struct + (off)) / 8)
 965 #define PARA_SITE(ptype, clobbers, ops) _PVSITE(ptype, clobbers, ops, .quad, 8)
 966 #define PARA_INDIRECT(addr)     *addr(%rip)
 967 #else
 968 #define PV_SAVE_REGS(set)                       \
 969         COND_PUSH(set, CLBR_EAX, eax);          \
 970         COND_PUSH(set, CLBR_EDI, edi);          \
 971         COND_PUSH(set, CLBR_ECX, ecx);          \
 972         COND_PUSH(set, CLBR_EDX, edx)
 973 #define PV_RESTORE_REGS(set)                    \
 974         COND_POP(set, CLBR_EDX, edx);           \
 975         COND_POP(set, CLBR_ECX, ecx);           \
 976         COND_POP(set, CLBR_EDI, edi);           \
 977         COND_POP(set, CLBR_EAX, eax)
 978
 979 #define PARA_PATCH(struct, off)        ((PARAVIRT_PATCH_##struct + (off)) / 4)
 980 #define PARA_SITE(ptype, clobbers, ops) _PVSITE(ptype, clobbers, ops, .long, 4)
 981 #define PARA_INDIRECT(addr)     *%cs:addr
 982 #endif
 983
 984 #define INTERRUPT_RETURN                                                \
 985         PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_iret), CLBR_NONE,       \
 986                   jmp PARA_INDIRECT(pv_cpu_ops+PV_CPU_iret))
 987
 988 #define DISABLE_INTERRUPTS(clobbers)                                    \
 989         PARA_SITE(PARA_PATCH(pv_irq_ops, PV_IRQ_irq_disable), clobbers, \
 990                   PV_SAVE_REGS(clobbers | CLBR_CALLEE_SAVE);            \
 991                   call PARA_INDIRECT(pv_irq_ops+PV_IRQ_irq_disable);    \
 992                   PV_RESTORE_REGS(clobbers | CLBR_CALLEE_SAVE);)
 993
 994 #define ENABLE_INTERRUPTS(clobbers)                                     \
 995         PARA_SITE(PARA_PATCH(pv_irq_ops, PV_IRQ_irq_enable), clobbers,  \
 996                   PV_SAVE_REGS(clobbers | CLBR_CALLEE_SAVE);            \
 997                   call PARA_INDIRECT(pv_irq_ops+PV_IRQ_irq_enable);     \
 998                   PV_RESTORE_REGS(clobbers | CLBR_CALLEE_SAVE);)
 999
1000 #define USERGS_SYSRET32                                                 \
1001         PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_usergs_sysret32),       \
1002                   CLBR_NONE,                                            \
1003                   jmp PARA_INDIRECT(pv_cpu_ops+PV_CPU_usergs_sysret32))
1004
1005 #ifdef CONFIG_X86_32
1006 #define GET_CR0_INTO_EAX                                \
1007         push %ecx; push %edx;                           \
1008         call PARA_INDIRECT(pv_cpu_ops+PV_CPU_read_cr0); \
1009         pop %edx; pop %ecx
1010
1011 #define ENABLE_INTERRUPTS_SYSEXIT                                       \
1012         PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_irq_enable_sysexit),    \
1013                   CLBR_NONE,                                            \
1014                   jmp PARA_INDIRECT(pv_cpu_ops+PV_CPU_irq_enable_sysexit))
1015
1016
1017 #else   /* !CONFIG_X86_32 */
1018
1019 /*
1020  * If swapgs is used while the userspace stack is still current,
1021  * there's no way to call a pvop.  The PV replacement *must* be
1022  * inlined, or the swapgs instruction must be trapped and emulated.
1023  */
1024 #define SWAPGS_UNSAFE_STACK                                             \
1025         PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_swapgs), CLBR_NONE,     \
1026                   swapgs)
1027
1028 /*
1029  * Note: swapgs is very special, and in practise is either going to be
1030  * implemented with a single "swapgs" instruction or something very
1031  * special.  Either way, we don't need to save any registers for
1032  * it.
1033  */
1034 #define SWAPGS                                                          \
1035         PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_swapgs), CLBR_NONE,     \
1036                   call PARA_INDIRECT(pv_cpu_ops+PV_CPU_swapgs)          \
1037                  )
1038
1039 #define GET_CR2_INTO_RCX                                \
1040         call PARA_INDIRECT(pv_mmu_ops+PV_MMU_read_cr2); \
1041         movq %rax, %rcx;                                \
1042         xorq %rax, %rax;
1043
1044 #define PARAVIRT_ADJUST_EXCEPTION_FRAME                                 \
1045         PARA_SITE(PARA_PATCH(pv_irq_ops, PV_IRQ_adjust_exception_frame), \
1046                   CLBR_NONE,                                            \
1047                   call PARA_INDIRECT(pv_irq_ops+PV_IRQ_adjust_exception_frame))
1048
1049 #define USERGS_SYSRET64                                                 \
1050         PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_usergs_sysret64),       \
1051                   CLBR_NONE,                                            \
1052                   jmp PARA_INDIRECT(pv_cpu_ops+PV_CPU_usergs_sysret64))
1053
1054 #define ENABLE_INTERRUPTS_SYSEXIT32                                     \
1055         PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_irq_enable_sysexit),    \
1056                   CLBR_NONE,                                            \
1057                   jmp PARA_INDIRECT(pv_cpu_ops+PV_CPU_irq_enable_sysexit))
1058 #endif  /* CONFIG_X86_32 */
1059
1060 #endif /* __ASSEMBLY__ */
1061 #endif /* CONFIG_PARAVIRT */
1062 #endif /* _ASM_X86_PARAVIRT_H */