include/asm-ia64/processor.h

   1 #ifndef _ASM_IA64_PROCESSOR_H
   2 #define _ASM_IA64_PROCESSOR_H
   3
   4 /*
   5  * Copyright (C) 1998-2003 Hewlett-Packard Co
   6  *      David Mosberger-Tang <davidm@hpl.hp.com>
   7  *      Stephane Eranian <eranian@hpl.hp.com>
   8  * Copyright (C) 1999 Asit Mallick <asit.k.mallick@intel.com>
   9  * Copyright (C) 1999 Don Dugger <don.dugger@intel.com>
  10  *
  11  * 11/24/98     S.Eranian       added ia64_set_iva()
  12  * 12/03/99     D. Mosberger    implement thread_saved_pc() via kernel unwind API
  13  * 06/16/00     A. Mallick      added csd/ssd/tssd for ia32 support
  14  */
  15
  16 #include <linux/config.h>
  17
  18 #include <asm/ptrace.h>
  19 #include <asm/kregs.h>
  20 #include <asm/ustack.h>
  21
  22 #define IA64_NUM_DBG_REGS       8
  23 /*
  24  * Limits for PMC and PMD are set to less than maximum architected values
  25  * but should be sufficient for a while
  26  */
  27 #define IA64_NUM_PMC_REGS       32
  28 #define IA64_NUM_PMD_REGS       32
  29
  30 #define DEFAULT_MAP_BASE        0x2000000000000000
  31 #define DEFAULT_TASK_SIZE       0xa000000000000000
  32
  33 /*
  34  * TASK_SIZE really is a mis-named.  It really is the maximum user
  35  * space address (plus one).  On IA-64, there are five regions of 2TB
  36  * each (assuming 8KB page size), for a total of 8TB of user virtual
  37  * address space.
  38  */
  39 #define TASK_SIZE               (current->thread.task_size)
  40
  41 /*
  42  * MM_VM_SIZE(mm) gives the maximum address (plus 1) which may contain a mapping for
  43  * address-space MM.  Note that with 32-bit tasks, this is still DEFAULT_TASK_SIZE,
  44  * because the kernel may have installed helper-mappings above TASK_SIZE.  For example,
  45  * for x86 emulation, the LDT and GDT are mapped above TASK_SIZE.
  46  */
  47 #define MM_VM_SIZE(mm)          DEFAULT_TASK_SIZE
  48
  49 /*
  50  * This decides where the kernel will search for a free chunk of vm
  51  * space during mmap's.
  52  */
  53 #define TASK_UNMAPPED_BASE      (current->thread.map_base)
  54
  55 /*
  56  * Bus types
  57  */
  58 #define EISA_bus 0
  59 #define MCA_bus 0
  60 #define MCA_bus__is_a_macro /* for versions in ksyms.c */
  61
  62 #define IA64_THREAD_FPH_VALID   (__IA64_UL(1) << 0)     /* floating-point high state valid? */
  63 #define IA64_THREAD_DBG_VALID   (__IA64_UL(1) << 1)     /* debug registers valid? */
  64 #define IA64_THREAD_PM_VALID    (__IA64_UL(1) << 2)     /* performance registers valid? */
  65 #define IA64_THREAD_UAC_NOPRINT (__IA64_UL(1) << 3)     /* don't log unaligned accesses */
  66 #define IA64_THREAD_UAC_SIGBUS  (__IA64_UL(1) << 4)     /* generate SIGBUS on unaligned acc. */
  67 #define IA64_THREAD_KRBS_SYNCED (__IA64_UL(1) << 5)     /* krbs synced with process vm? */
  68 #define IA64_THREAD_FPEMU_NOPRINT (__IA64_UL(1) << 6)   /* don't log any fpswa faults */
  69 #define IA64_THREAD_FPEMU_SIGFPE  (__IA64_UL(1) << 7)   /* send a SIGFPE for fpswa faults */
  70 #define IA64_THREAD_XSTACK      (__IA64_UL(1) << 8)     /* stack executable by default? */
  71
  72 #define IA64_THREAD_UAC_SHIFT   3
  73 #define IA64_THREAD_UAC_MASK    (IA64_THREAD_UAC_NOPRINT | IA64_THREAD_UAC_SIGBUS)
  74 #define IA64_THREAD_FPEMU_SHIFT 6
  75 #define IA64_THREAD_FPEMU_MASK  (IA64_THREAD_FPEMU_NOPRINT | IA64_THREAD_FPEMU_SIGFPE)
  76
  77
  78 /*
  79  * This shift should be large enough to be able to represent 1000000000/itc_freq with good
  80  * accuracy while being small enough to fit 10*1000000000<<IA64_NSEC_PER_CYC_SHIFT in 64 bits
  81  * (this will give enough slack to represent 10 seconds worth of time as a scaled number).
  82  */
  83 #define IA64_NSEC_PER_CYC_SHIFT 30
  84
  85 #ifndef __ASSEMBLY__
  86
  87 #include <linux/cache.h>
  88 #include <linux/compiler.h>
  89 #include <linux/threads.h>
  90 #include <linux/types.h>
  91
  92 #include <asm/fpu.h>
  93 #include <asm/page.h>
  94 #include <asm/percpu.h>
  95 #include <asm/rse.h>
  96 #include <asm/unwind.h>
  97 #include <asm/atomic.h>
  98 #ifdef CONFIG_NUMA
  99 #include <asm/nodedata.h>
 100 #endif
 101
 102 /* like above but expressed as bitfields for more efficient access: */
 103 struct ia64_psr {
 104         __u64 reserved0 : 1;
 105         __u64 be : 1;
 106         __u64 up : 1;
 107         __u64 ac : 1;
 108         __u64 mfl : 1;
 109         __u64 mfh : 1;
 110         __u64 reserved1 : 7;
 111         __u64 ic : 1;
 112         __u64 i : 1;
 113         __u64 pk : 1;
 114         __u64 reserved2 : 1;
 115         __u64 dt : 1;
 116         __u64 dfl : 1;
 117         __u64 dfh : 1;
 118         __u64 sp : 1;
 119         __u64 pp : 1;
 120         __u64 di : 1;
 121         __u64 si : 1;
 122         __u64 db : 1;
 123         __u64 lp : 1;
 124         __u64 tb : 1;
 125         __u64 rt : 1;
 126         __u64 reserved3 : 4;
 127         __u64 cpl : 2;
 128         __u64 is : 1;
 129         __u64 mc : 1;
 130         __u64 it : 1;
 131         __u64 id : 1;
 132         __u64 da : 1;
 133         __u64 dd : 1;
 134         __u64 ss : 1;
 135         __u64 ri : 2;
 136         __u64 ed : 1;
 137         __u64 bn : 1;
 138         __u64 reserved4 : 19;
 139 };
 140
 141 /*
 142  * CPU type, hardware bug flags, and per-CPU state.  Frequently used
 143  * state comes earlier:
 144  */
 145 struct cpuinfo_ia64 {
 146         /* irq_stat must be 64-bit aligned */
 147         union {
 148                 struct {
 149                         __u32 irq_count;
 150                         __u32 bh_count;
 151                 } f;
 152                 __u64 irq_and_bh_counts;
 153         } irq_stat;
 154         __u32 softirq_pending;
 155         __u64 itm_delta;        /* # of clock cycles between clock ticks */
 156         __u64 itm_next;         /* interval timer mask value to use for next clock tick */
 157         __u64 *pgd_quick;
 158         __u64 *pmd_quick;
 159         __u64 pgtable_cache_sz;
 160         /* CPUID-derived information: */
 161         __u64 ppn;
 162         __u64 features;
 163         __u8 number;
 164         __u8 revision;
 165         __u8 model;
 166         __u8 family;
 167         __u8 archrev;
 168         char vendor[16];
 169         __u64 itc_freq;         /* frequency of ITC counter */
 170         __u64 proc_freq;        /* frequency of processor */
 171         __u64 cyc_per_usec;     /* itc_freq/1000000 */
 172         __u64 nsec_per_cyc;     /* (1000000000<<IA64_NSEC_PER_CYC_SHIFT)/itc_freq */
 173         __u64 unimpl_va_mask;   /* mask of unimplemented virtual address bits (from PAL) */
 174         __u64 unimpl_pa_mask;   /* mask of unimplemented physical address bits (from PAL) */
 175         __u64 ptce_base;
 176         __u32 ptce_count[2];
 177         __u32 ptce_stride[2];
 178         struct task_struct *ksoftirqd;  /* kernel softirq daemon for this CPU */
 179 #ifdef CONFIG_SMP
 180         int cpu;
 181         __u64 loops_per_jiffy;
 182         __u64 ipi_count;
 183         __u64 prof_counter;
 184         __u64 prof_multiplier;
 185 #endif
 186 #ifdef CONFIG_NUMA
 187         struct ia64_node_data *node_data;
 188 #endif
 189 };
 190
 191 DECLARE_PER_CPU(struct cpuinfo_ia64, cpu_info);
 192
 193 /*
 194  * The "local" data pointer.  It points to the per-CPU data of the currently executing
 195  * CPU, much like "current" points to the per-task data of the currently executing task.
 196  */
 197 #define local_cpu_data          (&__get_cpu_var(cpu_info))
 198 #define cpu_data(cpu)           (&per_cpu(cpu_info, cpu))
 199
 200 extern void identify_cpu (struct cpuinfo_ia64 *);
 201 extern void print_cpu_info (struct cpuinfo_ia64 *);
 202
 203 typedef struct {
 204         unsigned long seg;
 205 } mm_segment_t;
 206
 207 #define SET_UNALIGN_CTL(task,value)                                                             \
 208 ({                                                                                              \
 209         (task)->thread.flags = (((task)->thread.flags & ~IA64_THREAD_UAC_MASK)                  \
 210                                 | (((value) << IA64_THREAD_UAC_SHIFT) & IA64_THREAD_UAC_MASK)); \
 211         0;                                                                                      \
 212 })
 213 #define GET_UNALIGN_CTL(task,addr)                                                              \
 214 ({                                                                                              \
 215         put_user(((task)->thread.flags & IA64_THREAD_UAC_MASK) >> IA64_THREAD_UAC_SHIFT,        \
 216                  (int *) (addr));                                                               \
 217 })
 218
 219 #define SET_FPEMU_CTL(task,value)                                                               \
 220 ({                                                                                              \
 221         (task)->thread.flags = (((task)->thread.flags & ~IA64_THREAD_FPEMU_MASK)                \
 222                           | (((value) << IA64_THREAD_FPEMU_SHIFT) & IA64_THREAD_FPEMU_MASK));   \
 223         0;                                                                                      \
 224 })
 225 #define GET_FPEMU_CTL(task,addr)                                                                \
 226 ({                                                                                              \
 227         put_user(((task)->thread.flags & IA64_THREAD_FPEMU_MASK) >> IA64_THREAD_FPEMU_SHIFT,    \
 228                  (int *) (addr));                                                               \
 229 })
 230
 231 struct thread_struct {
 232         __u32 flags;                    /* various thread flags (see IA64_THREAD_*) */
 233         /* writing on_ustack is performance-critical, so it's worth spending 8 bits on it... */
 234         __u8 on_ustack;                 /* executing on user-stacks? */
 235         __u8 pad[3];
 236         __u64 ksp;                      /* kernel stack pointer */
 237         __u64 map_base;                 /* base address for get_unmapped_area() */
 238         __u64 task_size;                /* limit for task size */
 239         __u64 rbs_bot;                  /* the base address for the RBS */
 240         int last_fph_cpu;               /* CPU that may hold the contents of f32-f127 */
 241
 242 #ifdef CONFIG_IA32_SUPPORT
 243         __u64 eflag;                    /* IA32 EFLAGS reg */
 244         __u64 fsr;                      /* IA32 floating pt status reg */
 245         __u64 fcr;                      /* IA32 floating pt control reg */
 246         __u64 fir;                      /* IA32 fp except. instr. reg */
 247         __u64 fdr;                      /* IA32 fp except. data reg */
 248         __u64 old_k1;                   /* old value of ar.k1 */
 249         __u64 old_iob;                  /* old IOBase value */
 250 # define INIT_THREAD_IA32       .eflag =        0,                      \
 251                                 .fsr =          0,                      \
 252                                 .fcr =          0x17800000037fULL,      \
 253                                 .fir =          0,                      \
 254                                 .fdr =          0,                      \
 255                                 .old_k1 =       0,                      \
 256                                 .old_iob =      0,
 257 #else
 258 # define INIT_THREAD_IA32
 259 #endif /* CONFIG_IA32_SUPPORT */
 260 #ifdef CONFIG_PERFMON
 261         __u64 pmcs[IA64_NUM_PMC_REGS];
 262         __u64 pmds[IA64_NUM_PMD_REGS];
 263         void *pfm_context;                   /* pointer to detailed PMU context */
 264         unsigned long pfm_needs_checking;    /* when >0, pending perfmon work on kernel exit */
 265 # define INIT_THREAD_PM         .pmcs =                 {0UL, },  \
 266                                 .pmds =                 {0UL, },  \
 267                                 .pfm_context =          NULL,     \
 268                                 .pfm_needs_checking =   0UL,
 269 #else
 270 # define INIT_THREAD_PM
 271 #endif
 272         __u64 dbr[IA64_NUM_DBG_REGS];
 273         __u64 ibr[IA64_NUM_DBG_REGS];
 274         struct ia64_fpreg fph[96];      /* saved/loaded on demand */
 275 };
 276
 277 #define INIT_THREAD {                           \
 278         .flags =        0,                      \
 279         .on_ustack =    0,                      \
 280         .ksp =          0,                      \
 281         .map_base =     DEFAULT_MAP_BASE,       \
 282         .rbs_bot =      DEFAULT_USER_STACK_SIZE,        \
 283         .task_size =    DEFAULT_TASK_SIZE,      \
 284         .last_fph_cpu =  -1,                    \
 285         INIT_THREAD_IA32                        \
 286         INIT_THREAD_PM                          \
 287         .dbr =          {0, },                  \
 288         .ibr =          {0, },                  \
 289         .fph =          {{{{0}}}, }             \
 290 }
 291
 292 #define start_thread(regs,new_ip,new_sp) do {                                                   \
 293         set_fs(USER_DS);                                                                        \
 294         regs->cr_ipsr = ((regs->cr_ipsr | (IA64_PSR_BITS_TO_SET | IA64_PSR_CPL))                \
 295                          & ~(IA64_PSR_BITS_TO_CLEAR | IA64_PSR_RI | IA64_PSR_IS));              \
 296         regs->cr_iip = new_ip;                                                                  \
 297         regs->ar_rsc = 0xf;             /* eager mode, privilege level 3 */                     \
 298         regs->ar_rnat = 0;                                                                      \
 299         regs->ar_bspstore = current->thread.rbs_bot;                                            \
 300         regs->ar_fpsr = FPSR_DEFAULT;                                                           \
 301         regs->loadrs = 0;                                                                       \
 302         regs->r8 = current->mm->dumpable;       /* set "don't zap registers" flag */            \
 303         regs->r12 = new_sp - 16;        /* allocate 16 byte scratch area */                     \
 304         if (unlikely(!current->mm->dumpable)) {                                                 \
 305                 /*                                                                              \
 306                  * Zap scratch regs to avoid leaking bits between processes with different      \
 307                  * uid/privileges.                                                              \
 308                  */                                                                             \
 309                 regs->ar_pfs = 0; regs->b0 = 0; regs->pr = 0;                                   \
 310                 regs->r1 = 0; regs->r9  = 0; regs->r11 = 0; regs->r13 = 0; regs->r15 = 0;       \
 311         }                                                                                       \
 312 } while (0)
 313
 314 /* Forward declarations, a strange C thing... */
 315 struct mm_struct;
 316 struct task_struct;
 317
 318 /*
 319  * Free all resources held by a thread. This is called after the
 320  * parent of DEAD_TASK has collected the exit status of the task via
 321  * wait().
 322  */
 323 #define release_thread(dead_task)
 324
 325 /* Prepare to copy thread state - unlazy all lazy status */
 326 #define prepare_to_copy(tsk)    do { } while (0)
 327
 328 /*
 329  * This is the mechanism for creating a new kernel thread.
 330  *
 331  * NOTE 1: Only a kernel-only process (ie the swapper or direct
 332  * descendants who haven't done an "execve()") should use this: it
 333  * will work within a system call from a "real" process, but the
 334  * process memory space will not be free'd until both the parent and
 335  * the child have exited.
 336  *
 337  * NOTE 2: This MUST NOT be an inlined function.  Otherwise, we get
 338  * into trouble in init/main.c when the child thread returns to
 339  * do_basic_setup() and the timing is such that free_initmem() has
 340  * been called already.
 341  */
 342 extern pid_t kernel_thread (int (*fn)(void *), void *arg, unsigned long flags);
 343
 344 /* Get wait channel for task P.  */
 345 extern unsigned long get_wchan (struct task_struct *p);
 346
 347 /* Return instruction pointer of blocked task TSK.  */
 348 #define KSTK_EIP(tsk)                                   \
 349   ({                                                    \
 350         struct pt_regs *_regs = ia64_task_regs(tsk);    \
 351         _regs->cr_iip + ia64_psr(_regs)->ri;            \
 352   })
 353
 354 /* Return stack pointer of blocked task TSK.  */
 355 #define KSTK_ESP(tsk)  ((tsk)->thread.ksp)
 356
 357 static inline unsigned long
 358 ia64_get_kr (unsigned long regnum)
 359 {
 360         unsigned long r = 0;
 361
 362         switch (regnum) {
 363               case 0: asm volatile ("mov %0=ar.k0" : "=r"(r)); break;
 364               case 1: asm volatile ("mov %0=ar.k1" : "=r"(r)); break;
 365               case 2: asm volatile ("mov %0=ar.k2" : "=r"(r)); break;
 366               case 3: asm volatile ("mov %0=ar.k3" : "=r"(r)); break;
 367               case 4: asm volatile ("mov %0=ar.k4" : "=r"(r)); break;
 368               case 5: asm volatile ("mov %0=ar.k5" : "=r"(r)); break;
 369               case 6: asm volatile ("mov %0=ar.k6" : "=r"(r)); break;
 370               case 7: asm volatile ("mov %0=ar.k7" : "=r"(r)); break;
 371         }
 372         return r;
 373 }
 374
 375 static inline void
 376 ia64_set_kr (unsigned long regnum, unsigned long r)
 377 {
 378         switch (regnum) {
 379               case 0: asm volatile ("mov ar.k0=%0" :: "r"(r)); break;
 380               case 1: asm volatile ("mov ar.k1=%0" :: "r"(r)); break;
 381               case 2: asm volatile ("mov ar.k2=%0" :: "r"(r)); break;
 382               case 3: asm volatile ("mov ar.k3=%0" :: "r"(r)); break;
 383               case 4: asm volatile ("mov ar.k4=%0" :: "r"(r)); break;
 384               case 5: asm volatile ("mov ar.k5=%0" :: "r"(r)); break;
 385               case 6: asm volatile ("mov ar.k6=%0" :: "r"(r)); break;
 386               case 7: asm volatile ("mov ar.k7=%0" :: "r"(r)); break;
 387         }
 388 }
 389
 390 /*
 391  * The following three macros can't be inline functions because we don't have struct
 392  * task_struct at this point.
 393  */
 394
 395 /* Return TRUE if task T owns the fph partition of the CPU we're running on. */
 396 #define ia64_is_local_fpu_owner(t)                                                              \
 397 ({                                                                                              \
 398         struct task_struct *__ia64_islfo_task = (t);                                            \
 399         (__ia64_islfo_task->thread.last_fph_cpu == smp_processor_id()                           \
 400          && __ia64_islfo_task == (struct task_struct *) ia64_get_kr(IA64_KR_FPU_OWNER));        \
 401 })
 402
 403 /* Mark task T as owning the fph partition of the CPU we're running on. */
 404 #define ia64_set_local_fpu_owner(t) do {                                                \
 405         struct task_struct *__ia64_slfo_task = (t);                                     \
 406         __ia64_slfo_task->thread.last_fph_cpu = smp_processor_id();                     \
 407         ia64_set_kr(IA64_KR_FPU_OWNER, (unsigned long) __ia64_slfo_task);               \
 408 } while (0)
 409
 410 /* Mark the fph partition of task T as being invalid on all CPUs.  */
 411 #define ia64_drop_fpu(t)        ((t)->thread.last_fph_cpu = -1)
 412
 413 extern void __ia64_init_fpu (void);
 414 extern void __ia64_save_fpu (struct ia64_fpreg *fph);
 415 extern void __ia64_load_fpu (struct ia64_fpreg *fph);
 416 extern void ia64_save_debug_regs (unsigned long *save_area);
 417 extern void ia64_load_debug_regs (unsigned long *save_area);
 418
 419 #ifdef CONFIG_IA32_SUPPORT
 420 extern void ia32_save_state (struct task_struct *task);
 421 extern void ia32_load_state (struct task_struct *task);
 422 #endif
 423
 424 #define ia64_fph_enable()       asm volatile (";; rsm psr.dfh;; srlz.d;;" ::: "memory");
 425 #define ia64_fph_disable()      asm volatile (";; ssm psr.dfh;; srlz.d;;" ::: "memory");
 426
 427 /* load fp 0.0 into fph */
 428 static inline void
 429 ia64_init_fpu (void) {
 430         ia64_fph_enable();
 431         __ia64_init_fpu();
 432         ia64_fph_disable();
 433 }
 434
 435 /* save f32-f127 at FPH */
 436 static inline void
 437 ia64_save_fpu (struct ia64_fpreg *fph) {
 438         ia64_fph_enable();
 439         __ia64_save_fpu(fph);
 440         ia64_fph_disable();
 441 }
 442
 443 /* load f32-f127 from FPH */
 444 static inline void
 445 ia64_load_fpu (struct ia64_fpreg *fph) {
 446         ia64_fph_enable();
 447         __ia64_load_fpu(fph);
 448         ia64_fph_disable();
 449 }
 450
 451 static inline void
 452 ia64_fc (void *addr)
 453 {
 454         asm volatile ("fc %0" :: "r"(addr) : "memory");
 455 }
 456
 457 static inline void
 458 ia64_sync_i (void)
 459 {
 460         asm volatile (";; sync.i" ::: "memory");
 461 }
 462
 463 static inline void
 464 ia64_srlz_i (void)
 465 {
 466         asm volatile (";; srlz.i ;;" ::: "memory");
 467 }
 468
 469 static inline void
 470 ia64_srlz_d (void)
 471 {
 472         asm volatile (";; srlz.d" ::: "memory");
 473 }
 474
 475 static inline __u64
 476 ia64_get_rr (__u64 reg_bits)
 477 {
 478         __u64 r;
 479         asm volatile ("mov %0=rr[%1]" : "=r"(r) : "r"(reg_bits) : "memory");
 480         return r;
 481 }
 482
 483 static inline void
 484 ia64_set_rr (__u64 reg_bits, __u64 rr_val)
 485 {
 486         asm volatile ("mov rr[%0]=%1" :: "r"(reg_bits), "r"(rr_val) : "memory");
 487 }
 488
 489 static inline __u64
 490 ia64_get_dcr (void)
 491 {
 492         __u64 r;
 493         asm volatile ("mov %0=cr.dcr" : "=r"(r));
 494         return r;
 495 }
 496
 497 static inline void
 498 ia64_set_dcr (__u64 val)
 499 {
 500         asm volatile ("mov cr.dcr=%0;;" :: "r"(val) : "memory");
 501         ia64_srlz_d();
 502 }
 503
 504 static inline __u64
 505 ia64_get_lid (void)
 506 {
 507         __u64 r;
 508         asm volatile ("mov %0=cr.lid" : "=r"(r));
 509         return r;
 510 }
 511
 512 static inline void
 513 ia64_invala (void)
 514 {
 515         asm volatile ("invala" ::: "memory");
 516 }
 517
 518 static inline __u64
 519 ia64_clear_ic (void)
 520 {
 521         __u64 psr;
 522         asm volatile ("mov %0=psr;; rsm psr.i | psr.ic;; srlz.i;;" : "=r"(psr) :: "memory");
 523         return psr;
 524 }
 525
 526 /*
 527  * Restore the psr.
 528  */
 529 static inline void
 530 ia64_set_psr (__u64 psr)
 531 {
 532         asm volatile (";; mov psr.l=%0;; srlz.d" :: "r" (psr) : "memory");
 533 }
 534
 535 /*
 536  * Insert a translation into an instruction and/or data translation
 537  * register.
 538  */
 539 static inline void
 540 ia64_itr (__u64 target_mask, __u64 tr_num,
 541           __u64 vmaddr, __u64 pte,
 542           __u64 log_page_size)
 543 {
 544         asm volatile ("mov cr.itir=%0" :: "r"(log_page_size << 2) : "memory");
 545         asm volatile ("mov cr.ifa=%0;;" :: "r"(vmaddr) : "memory");
 546         if (target_mask & 0x1)
 547                 asm volatile ("itr.i itr[%0]=%1"
 548                                       :: "r"(tr_num), "r"(pte) : "memory");
 549         if (target_mask & 0x2)
 550                 asm volatile (";;itr.d dtr[%0]=%1"
 551                                       :: "r"(tr_num), "r"(pte) : "memory");
 552 }
 553
 554 /*
 555  * Insert a translation into the instruction and/or data translation
 556  * cache.
 557  */
 558 static inline void
 559 ia64_itc (__u64 target_mask, __u64 vmaddr, __u64 pte,
 560           __u64 log_page_size)
 561 {
 562         asm volatile ("mov cr.itir=%0" :: "r"(log_page_size << 2) : "memory");
 563         asm volatile ("mov cr.ifa=%0;;" :: "r"(vmaddr) : "memory");
 564         /* as per EAS2.6, itc must be the last instruction in an instruction group */
 565         if (target_mask & 0x1)
 566                 asm volatile ("itc.i %0;;" :: "r"(pte) : "memory");
 567         if (target_mask & 0x2)
 568                 asm volatile (";;itc.d %0;;" :: "r"(pte) : "memory");
 569 }
 570
 571 /*
 572  * Purge a range of addresses from instruction and/or data translation
 573  * register(s).
 574  */
 575 static inline void
 576 ia64_ptr (__u64 target_mask, __u64 vmaddr, __u64 log_size)
 577 {
 578         if (target_mask & 0x1)
 579                 asm volatile ("ptr.i %0,%1" :: "r"(vmaddr), "r"(log_size << 2));
 580         if (target_mask & 0x2)
 581                 asm volatile ("ptr.d %0,%1" :: "r"(vmaddr), "r"(log_size << 2));
 582 }
 583
 584 /* Set the interrupt vector address.  The address must be suitably aligned (32KB).  */
 585 static inline void
 586 ia64_set_iva (void *ivt_addr)
 587 {
 588         asm volatile ("mov cr.iva=%0;; srlz.i;;" :: "r"(ivt_addr) : "memory");
 589 }
 590
 591 /* Set the page table address and control bits.  */
 592 static inline void
 593 ia64_set_pta (__u64 pta)
 594 {
 595         /* Note: srlz.i implies srlz.d */
 596         asm volatile ("mov cr.pta=%0;; srlz.i;;" :: "r"(pta) : "memory");
 597 }
 598
 599 static inline __u64
 600 ia64_get_cpuid (__u64 regnum)
 601 {
 602         __u64 r;
 603
 604         asm ("mov %0=cpuid[%r1]" : "=r"(r) : "rO"(regnum));
 605         return r;
 606 }
 607
 608 static inline void
 609 ia64_eoi (void)
 610 {
 611         asm ("mov cr.eoi=r0;; srlz.d;;" ::: "memory");
 612 }
 613
 614 static inline void
 615 ia64_set_lrr0 (unsigned long val)
 616 {
 617         asm volatile ("mov cr.lrr0=%0;; srlz.d" :: "r"(val) : "memory");
 618 }
 619
 620 static inline void
 621 ia64_hint_pause (void)
 622 {
 623         asm volatile ("hint @pause" ::: "memory");
 624 }
 625
 626 #define cpu_relax()     ia64_hint_pause()
 627
 628 static inline void
 629 ia64_set_lrr1 (unsigned long val)
 630 {
 631         asm volatile ("mov cr.lrr1=%0;; srlz.d" :: "r"(val) : "memory");
 632 }
 633
 634 static inline void
 635 ia64_set_pmv (__u64 val)
 636 {
 637         asm volatile ("mov cr.pmv=%0" :: "r"(val) : "memory");
 638 }
 639
 640 static inline __u64
 641 ia64_get_pmc (__u64 regnum)
 642 {
 643         __u64 retval;
 644
 645         asm volatile ("mov %0=pmc[%1]" : "=r"(retval) : "r"(regnum));
 646         return retval;
 647 }
 648
 649 static inline void
 650 ia64_set_pmc (__u64 regnum, __u64 value)
 651 {
 652         asm volatile ("mov pmc[%0]=%1" :: "r"(regnum), "r"(value));
 653 }
 654
 655 static inline __u64
 656 ia64_get_pmd (__u64 regnum)
 657 {
 658         __u64 retval;
 659
 660         asm volatile ("mov %0=pmd[%1]" : "=r"(retval) : "r"(regnum));
 661         return retval;
 662 }
 663
 664 static inline void
 665 ia64_set_pmd (__u64 regnum, __u64 value)
 666 {
 667         asm volatile ("mov pmd[%0]=%1" :: "r"(regnum), "r"(value));
 668 }
 669
 670 /*
 671  * Given the address to which a spill occurred, return the unat bit
 672  * number that corresponds to this address.
 673  */
 674 static inline __u64
 675 ia64_unat_pos (void *spill_addr)
 676 {
 677         return ((__u64) spill_addr >> 3) & 0x3f;
 678 }
 679
 680 /*
 681  * Set the NaT bit of an integer register which was spilled at address
 682  * SPILL_ADDR.  UNAT is the mask to be updated.
 683  */
 684 static inline void
 685 ia64_set_unat (__u64 *unat, void *spill_addr, unsigned long nat)
 686 {
 687         __u64 bit = ia64_unat_pos(spill_addr);
 688         __u64 mask = 1UL << bit;
 689
 690         *unat = (*unat & ~mask) | (nat << bit);
 691 }
 692
 693 /*
 694  * Return saved PC of a blocked thread.
 695  * Note that the only way T can block is through a call to schedule() -> switch_to().
 696  */
 697 static inline unsigned long
 698 thread_saved_pc (struct task_struct *t)
 699 {
 700         struct unw_frame_info info;
 701         unsigned long ip;
 702
 703         unw_init_from_blocked_task(&info, t);
 704         if (unw_unwind(&info) < 0)
 705                 return 0;
 706         unw_get_ip(&info, &ip);
 707         return ip;
 708 }
 709
 710 /*
 711  * Get the current instruction/program counter value.
 712  */
 713 #define current_text_addr() \
 714         ({ void *_pc; asm volatile ("mov %0=ip" : "=r" (_pc)); _pc; })
 715
 716 /*
 717  * Set the correctable machine check vector register
 718  */
 719 static inline void
 720 ia64_set_cmcv (__u64 val)
 721 {
 722         asm volatile ("mov cr.cmcv=%0" :: "r"(val) : "memory");
 723 }
 724
 725 /*
 726  * Read the correctable machine check vector register
 727  */
 728 static inline __u64
 729 ia64_get_cmcv (void)
 730 {
 731         __u64 val;
 732
 733         asm volatile ("mov %0=cr.cmcv" : "=r"(val) :: "memory");
 734         return val;
 735 }
 736
 737 static inline __u64
 738 ia64_get_ivr (void)
 739 {
 740         __u64 r;
 741         asm volatile ("srlz.d;; mov %0=cr.ivr;; srlz.d;;" : "=r"(r));
 742         return r;
 743 }
 744
 745 static inline void
 746 ia64_set_tpr (__u64 val)
 747 {
 748         asm volatile ("mov cr.tpr=%0" :: "r"(val));
 749 }
 750
 751 static inline __u64
 752 ia64_get_tpr (void)
 753 {
 754         __u64 r;
 755         asm volatile ("mov %0=cr.tpr" : "=r"(r));
 756         return r;
 757 }
 758
 759 static inline void
 760 ia64_set_irr0 (__u64 val)
 761 {
 762         asm volatile("mov cr.irr0=%0;;" :: "r"(val) : "memory");
 763         ia64_srlz_d();
 764 }
 765
 766 static inline __u64
 767 ia64_get_irr0 (void)
 768 {
 769         __u64 val;
 770
 771         /* this is volatile because irr may change unbeknownst to gcc... */
 772         asm volatile("mov %0=cr.irr0" : "=r"(val));
 773         return val;
 774 }
 775
 776 static inline void
 777 ia64_set_irr1 (__u64 val)
 778 {
 779         asm volatile("mov cr.irr1=%0;;" :: "r"(val) : "memory");
 780         ia64_srlz_d();
 781 }
 782
 783 static inline __u64
 784 ia64_get_irr1 (void)
 785 {
 786         __u64 val;
 787
 788         /* this is volatile because irr may change unbeknownst to gcc... */
 789         asm volatile("mov %0=cr.irr1" : "=r"(val));
 790         return val;
 791 }
 792
 793 static inline void
 794 ia64_set_irr2 (__u64 val)
 795 {
 796         asm volatile("mov cr.irr2=%0;;" :: "r"(val) : "memory");
 797         ia64_srlz_d();
 798 }
 799
 800 static inline __u64
 801 ia64_get_irr2 (void)
 802 {
 803         __u64 val;
 804
 805         /* this is volatile because irr may change unbeknownst to gcc... */
 806         asm volatile("mov %0=cr.irr2" : "=r"(val));
 807         return val;
 808 }
 809
 810 static inline void
 811 ia64_set_irr3 (__u64 val)
 812 {
 813         asm volatile("mov cr.irr3=%0;;" :: "r"(val) : "memory");
 814         ia64_srlz_d();
 815 }
 816
 817 static inline __u64
 818 ia64_get_irr3 (void)
 819 {
 820         __u64 val;
 821
 822         /* this is volatile because irr may change unbeknownst to gcc... */
 823         asm volatile ("mov %0=cr.irr3" : "=r"(val));
 824         return val;
 825 }
 826
 827 static inline __u64
 828 ia64_get_gp(void)
 829 {
 830         __u64 val;
 831
 832         asm ("mov %0=gp" : "=r"(val));
 833         return val;
 834 }
 835
 836 static inline void
 837 ia64_set_ibr (__u64 regnum, __u64 value)
 838 {
 839         asm volatile ("mov ibr[%0]=%1" :: "r"(regnum), "r"(value));
 840 }
 841
 842 static inline void
 843 ia64_set_dbr (__u64 regnum, __u64 value)
 844 {
 845         asm volatile ("mov dbr[%0]=%1" :: "r"(regnum), "r"(value));
 846 #ifdef CONFIG_ITANIUM
 847         asm volatile (";; srlz.d");
 848 #endif
 849 }
 850
 851 static inline __u64
 852 ia64_get_ibr (__u64 regnum)
 853 {
 854         __u64 retval;
 855
 856         asm volatile ("mov %0=ibr[%1]" : "=r"(retval) : "r"(regnum));
 857         return retval;
 858 }
 859
 860 static inline __u64
 861 ia64_get_dbr (__u64 regnum)
 862 {
 863         __u64 retval;
 864
 865         asm volatile ("mov %0=dbr[%1]" : "=r"(retval) : "r"(regnum));
 866 #ifdef CONFIG_ITANIUM
 867         asm volatile (";; srlz.d");
 868 #endif
 869         return retval;
 870 }
 871
 872 /* XXX remove the handcoded version once we have a sufficiently clever compiler... */
 873 #ifdef SMART_COMPILER
 874 # define ia64_rotr(w,n)                         \
 875   ({                                            \
 876         __u64 _w = (w), _n = (n);               \
 877                                                 \
 878         (_w >> _n) | (_w << (64 - _n));         \
 879   })
 880 #else
 881 # define ia64_rotr(w,n)                                                 \
 882   ({                                                                    \
 883         __u64 result;                                                   \
 884         asm ("shrp %0=%1,%1,%2" : "=r"(result) : "r"(w), "i"(n));       \
 885         result;                                                         \
 886   })
 887 #endif
 888
 889 #define ia64_rotl(w,n)  ia64_rotr((w),(64)-(n))
 890
 891 static inline __u64
 892 ia64_thash (__u64 addr)
 893 {
 894         __u64 result;
 895         asm ("thash %0=%1" : "=r"(result) : "r" (addr));
 896         return result;
 897 }
 898
 899 static inline __u64
 900 ia64_tpa (__u64 addr)
 901 {
 902         __u64 result;
 903         asm ("tpa %0=%1" : "=r"(result) : "r"(addr));
 904         return result;
 905 }
 906
 907 /*
 908  * Take a mapped kernel address and return the equivalent address
 909  * in the region 7 identity mapped virtual area.
 910  */
 911 static inline void *
 912 ia64_imva (void *addr)
 913 {
 914         void *result;
 915         asm ("tpa %0=%1" : "=r"(result) : "r"(addr));
 916         return __va(result);
 917 }
 918
 919 #define ARCH_HAS_PREFETCH
 920 #define ARCH_HAS_PREFETCHW
 921 #define ARCH_HAS_SPINLOCK_PREFETCH
 922 #define PREFETCH_STRIDE 256
 923
 924 static inline void
 925 prefetch (const void *x)
 926 {
 927          __asm__ __volatile__ ("lfetch [%0]" : : "r"(x));
 928 }
 929
 930 static inline void
 931 prefetchw (const void *x)
 932 {
 933         __asm__ __volatile__ ("lfetch.excl [%0]" : : "r"(x));
 934 }
 935
 936 #define spin_lock_prefetch(x)   prefetchw(x)
 937
 938 #endif /* !__ASSEMBLY__ */
 939
 940 #endif /* _ASM_IA64_PROCESSOR_H */