arch/x86/kernel/vsyscall_64.c

   1 /*
   2  *  Copyright (C) 2001 Andrea Arcangeli <andrea@suse.de> SuSE
   3  *  Copyright 2003 Andi Kleen, SuSE Labs.
   4  *
   5  *  Thanks to hpa@transmeta.com for some useful hint.
   6  *  Special thanks to Ingo Molnar for his early experience with
   7  *  a different vsyscall implementation for Linux/IA32 and for the name.
   8  *
   9  *  vsyscall 1 is located at -10Mbyte, vsyscall 2 is located
  10  *  at virtual address -10Mbyte+1024bytes etc... There are at max 4
  11  *  vsyscalls. One vsyscall can reserve more than 1 slot to avoid
  12  *  jumping out of line if necessary. We cannot add more with this
  13  *  mechanism because older kernels won't return -ENOSYS.
  14  *  If we want more than four we need a vDSO.
  15  *
  16  *  Note: the concept clashes with user mode linux. If you use UML and
  17  *  want per guest time just set the kernel.vsyscall64 sysctl to 0.
  18  */
  19
  20 #include <linux/time.h>
  21 #include <linux/init.h>
  22 #include <linux/kernel.h>
  23 #include <linux/timer.h>
  24 #include <linux/seqlock.h>
  25 #include <linux/jiffies.h>
  26 #include <linux/sysctl.h>
  27 #include <linux/clocksource.h>
  28 #include <linux/getcpu.h>
  29 #include <linux/cpu.h>
  30 #include <linux/smp.h>
  31 #include <linux/notifier.h>
  32
  33 #include <asm/vsyscall.h>
  34 #include <asm/pgtable.h>
  35 #include <asm/page.h>
  36 #include <asm/unistd.h>
  37 #include <asm/fixmap.h>
  38 #include <asm/errno.h>
  39 #include <asm/io.h>
  40 #include <asm/segment.h>
  41 #include <asm/desc.h>
  42 #include <asm/topology.h>
  43 #include <asm/vgtod.h>
  44
  45 #define __vsyscall(nr) __attribute__ ((unused,__section__(".vsyscall_" #nr)))
  46 #define __syscall_clobber "r11","cx","memory"
  47 <<<<<<< HEAD:arch/x86/kernel/vsyscall_64.c
  48 #define __pa_vsymbol(x)                 \
  49         ({unsigned long v;              \
  50         extern char __vsyscall_0;       \
  51           asm("" : "=r" (v) : "0" (x)); \
  52           ((v - VSYSCALL_START) + __pa_symbol(&__vsyscall_0)); })
  53 =======
  54 >>>>>>> 264e3e889d86e552b4191d69bb60f4f3b383135a:arch/x86/kernel/vsyscall_64.c
  55
  56 /*
  57  * vsyscall_gtod_data contains data that is :
  58  * - readonly from vsyscalls
  59  * - written by timer interrupt or systcl (/proc/sys/kernel/vsyscall64)
  60  * Try to keep this structure as small as possible to avoid cache line ping pongs
  61  */
  62 int __vgetcpu_mode __section_vgetcpu_mode;
  63
  64 struct vsyscall_gtod_data __vsyscall_gtod_data __section_vsyscall_gtod_data =
  65 {
  66         .lock = SEQLOCK_UNLOCKED,
  67         .sysctl_enabled = 1,
  68 };
  69
  70 void update_vsyscall_tz(void)
  71 {
  72         unsigned long flags;
  73
  74         write_seqlock_irqsave(&vsyscall_gtod_data.lock, flags);
  75         /* sys_tz has changed */
  76         vsyscall_gtod_data.sys_tz = sys_tz;
  77         write_sequnlock_irqrestore(&vsyscall_gtod_data.lock, flags);
  78 }
  79
  80 void update_vsyscall(struct timespec *wall_time, struct clocksource *clock)
  81 {
  82         unsigned long flags;
  83
  84         write_seqlock_irqsave(&vsyscall_gtod_data.lock, flags);
  85         /* copy vsyscall data */
  86         vsyscall_gtod_data.clock.vread = clock->vread;
  87         vsyscall_gtod_data.clock.cycle_last = clock->cycle_last;
  88         vsyscall_gtod_data.clock.mask = clock->mask;
  89         vsyscall_gtod_data.clock.mult = clock->mult;
  90         vsyscall_gtod_data.clock.shift = clock->shift;
  91         vsyscall_gtod_data.wall_time_sec = wall_time->tv_sec;
  92         vsyscall_gtod_data.wall_time_nsec = wall_time->tv_nsec;
  93         vsyscall_gtod_data.wall_to_monotonic = wall_to_monotonic;
  94         write_sequnlock_irqrestore(&vsyscall_gtod_data.lock, flags);
  95 }
  96
  97 /* RED-PEN may want to readd seq locking, but then the variable should be
  98  * write-once.
  99  */
 100 static __always_inline void do_get_tz(struct timezone * tz)
 101 {
 102         *tz = __vsyscall_gtod_data.sys_tz;
 103 }
 104
 105 static __always_inline int gettimeofday(struct timeval *tv, struct timezone *tz)
 106 {
 107         int ret;
 108 <<<<<<< HEAD:arch/x86/kernel/vsyscall_64.c
 109         asm volatile("vsysc2: syscall"
 110 =======
 111         asm volatile("syscall"
 112 >>>>>>> 264e3e889d86e552b4191d69bb60f4f3b383135a:arch/x86/kernel/vsyscall_64.c
 113                 : "=a" (ret)
 114                 : "0" (__NR_gettimeofday),"D" (tv),"S" (tz)
 115                 : __syscall_clobber );
 116         return ret;
 117 }
 118
 119 static __always_inline long time_syscall(long *t)
 120 {
 121         long secs;
 122 <<<<<<< HEAD:arch/x86/kernel/vsyscall_64.c
 123         asm volatile("vsysc1: syscall"
 124 =======
 125         asm volatile("syscall"
 126 >>>>>>> 264e3e889d86e552b4191d69bb60f4f3b383135a:arch/x86/kernel/vsyscall_64.c
 127                 : "=a" (secs)
 128                 : "0" (__NR_time),"D" (t) : __syscall_clobber);
 129         return secs;
 130 }
 131
 132 static __always_inline void do_vgettimeofday(struct timeval * tv)
 133 {
 134         cycle_t now, base, mask, cycle_delta;
 135         unsigned seq;
 136         unsigned long mult, shift, nsec;
 137         cycle_t (*vread)(void);
 138         do {
 139                 seq = read_seqbegin(&__vsyscall_gtod_data.lock);
 140
 141                 vread = __vsyscall_gtod_data.clock.vread;
 142                 if (unlikely(!__vsyscall_gtod_data.sysctl_enabled || !vread)) {
 143                         gettimeofday(tv,NULL);
 144                         return;
 145                 }
 146                 now = vread();
 147                 base = __vsyscall_gtod_data.clock.cycle_last;
 148                 mask = __vsyscall_gtod_data.clock.mask;
 149                 mult = __vsyscall_gtod_data.clock.mult;
 150                 shift = __vsyscall_gtod_data.clock.shift;
 151
 152                 tv->tv_sec = __vsyscall_gtod_data.wall_time_sec;
 153                 nsec = __vsyscall_gtod_data.wall_time_nsec;
 154         } while (read_seqretry(&__vsyscall_gtod_data.lock, seq));
 155
 156         /* calculate interval: */
 157         cycle_delta = (now - base) & mask;
 158         /* convert to nsecs: */
 159         nsec += (cycle_delta * mult) >> shift;
 160
 161         while (nsec >= NSEC_PER_SEC) {
 162                 tv->tv_sec += 1;
 163                 nsec -= NSEC_PER_SEC;
 164         }
 165         tv->tv_usec = nsec / NSEC_PER_USEC;
 166 }
 167
 168 int __vsyscall(0) vgettimeofday(struct timeval * tv, struct timezone * tz)
 169 {
 170         if (tv)
 171                 do_vgettimeofday(tv);
 172         if (tz)
 173                 do_get_tz(tz);
 174         return 0;
 175 }
 176
 177 /* This will break when the xtime seconds get inaccurate, but that is
 178  * unlikely */
 179 time_t __vsyscall(1) vtime(time_t *t)
 180 {
 181         struct timeval tv;
 182         time_t result;
 183         if (unlikely(!__vsyscall_gtod_data.sysctl_enabled))
 184                 return time_syscall(t);
 185
 186         vgettimeofday(&tv, NULL);
 187         result = tv.tv_sec;
 188         if (t)
 189                 *t = result;
 190         return result;
 191 }
 192
 193 /* Fast way to get current CPU and node.
 194    This helps to do per node and per CPU caches in user space.
 195    The result is not guaranteed without CPU affinity, but usually
 196    works out because the scheduler tries to keep a thread on the same
 197    CPU.
 198
 199    tcache must point to a two element sized long array.
 200    All arguments can be NULL. */
 201 long __vsyscall(2)
 202 vgetcpu(unsigned *cpu, unsigned *node, struct getcpu_cache *tcache)
 203 {
 204         unsigned int p;
 205         unsigned long j = 0;
 206
 207         /* Fast cache - only recompute value once per jiffies and avoid
 208            relatively costly rdtscp/cpuid otherwise.
 209            This works because the scheduler usually keeps the process
 210            on the same CPU and this syscall doesn't guarantee its
 211            results anyways.
 212            We do this here because otherwise user space would do it on
 213            its own in a likely inferior way (no access to jiffies).
 214            If you don't like it pass NULL. */
 215         if (tcache && tcache->blob[0] == (j = __jiffies)) {
 216                 p = tcache->blob[1];
 217         } else if (__vgetcpu_mode == VGETCPU_RDTSCP) {
 218                 /* Load per CPU data from RDTSCP */
 219                 native_read_tscp(&p);
 220         } else {
 221                 /* Load per CPU data from GDT */
 222                 asm("lsl %1,%0" : "=r" (p) : "r" (__PER_CPU_SEG));
 223         }
 224         if (tcache) {
 225                 tcache->blob[0] = j;
 226                 tcache->blob[1] = p;
 227         }
 228         if (cpu)
 229                 *cpu = p & 0xfff;
 230         if (node)
 231                 *node = p >> 12;
 232         return 0;
 233 }
 234
 235 long __vsyscall(3) venosys_1(void)
 236 {
 237         return -ENOSYS;
 238 }
 239
 240 #ifdef CONFIG_SYSCTL
 241
 242 <<<<<<< HEAD:arch/x86/kernel/vsyscall_64.c
 243 #define SYSCALL 0x050f
 244 #define NOP2    0x9090
 245
 246 /*
 247  * NOP out syscall in vsyscall page when not needed.
 248  */
 249 static int vsyscall_sysctl_change(ctl_table *ctl, int write, struct file * filp,
 250                         void __user *buffer, size_t *lenp, loff_t *ppos)
 251 =======
 252 static int
 253 vsyscall_sysctl_change(ctl_table *ctl, int write, struct file * filp,
 254                        void __user *buffer, size_t *lenp, loff_t *ppos)
 255 >>>>>>> 264e3e889d86e552b4191d69bb60f4f3b383135a:arch/x86/kernel/vsyscall_64.c
 256 {
 257 <<<<<<< HEAD:arch/x86/kernel/vsyscall_64.c
 258         extern u16 vsysc1, vsysc2;
 259         u16 __iomem *map1;
 260         u16 __iomem *map2;
 261         int ret = proc_dointvec(ctl, write, filp, buffer, lenp, ppos);
 262         if (!write)
 263                 return ret;
 264         /* gcc has some trouble with __va(__pa()), so just do it this
 265            way. */
 266         map1 = ioremap(__pa_vsymbol(&vsysc1), 2);
 267         if (!map1)
 268                 return -ENOMEM;
 269         map2 = ioremap(__pa_vsymbol(&vsysc2), 2);
 270         if (!map2) {
 271                 ret = -ENOMEM;
 272                 goto out;
 273         }
 274         if (!vsyscall_gtod_data.sysctl_enabled) {
 275                 writew(SYSCALL, map1);
 276                 writew(SYSCALL, map2);
 277         } else {
 278                 writew(NOP2, map1);
 279                 writew(NOP2, map2);
 280         }
 281         iounmap(map2);
 282 out:
 283         iounmap(map1);
 284         return ret;
 285 =======
 286         return proc_dointvec(ctl, write, filp, buffer, lenp, ppos);
 287 >>>>>>> 264e3e889d86e552b4191d69bb60f4f3b383135a:arch/x86/kernel/vsyscall_64.c
 288 }
 289
 290 static ctl_table kernel_table2[] = {
 291         { .procname = "vsyscall64",
 292           .data = &vsyscall_gtod_data.sysctl_enabled, .maxlen = sizeof(int),
 293           .mode = 0644,
 294           .proc_handler = vsyscall_sysctl_change },
 295         {}
 296 };
 297
 298 static ctl_table kernel_root_table2[] = {
 299         { .ctl_name = CTL_KERN, .procname = "kernel", .mode = 0555,
 300           .child = kernel_table2 },
 301         {}
 302 };
 303 <<<<<<< HEAD:arch/x86/kernel/vsyscall_64.c
 304
 305 =======
 306 >>>>>>> 264e3e889d86e552b4191d69bb60f4f3b383135a:arch/x86/kernel/vsyscall_64.c
 307 #endif
 308
 309 /* Assume __initcall executes before all user space. Hopefully kmod
 310    doesn't violate that. We'll find out if it does. */
 311 static void __cpuinit vsyscall_set_cpu(int cpu)
 312 {
 313         unsigned long *d;
 314         unsigned long node = 0;
 315 #ifdef CONFIG_NUMA
 316         node = cpu_to_node(cpu);
 317 #endif
 318         if (cpu_has(&cpu_data(cpu), X86_FEATURE_RDTSCP))
 319                 write_rdtscp_aux((node << 12) | cpu);
 320
 321         /* Store cpu number in limit so that it can be loaded quickly
 322            in user space in vgetcpu.
 323            12 bits for the CPU and 8 bits for the node. */
 324         d = (unsigned long *)(get_cpu_gdt_table(cpu) + GDT_ENTRY_PER_CPU);
 325         *d = 0x0f40000000000ULL;
 326         *d |= cpu;
 327         *d |= (node & 0xf) << 12;
 328         *d |= (node >> 4) << 48;
 329 }
 330
 331 static void __cpuinit cpu_vsyscall_init(void *arg)
 332 {
 333         /* preemption should be already off */
 334         vsyscall_set_cpu(raw_smp_processor_id());
 335 }
 336
 337 static int __cpuinit
 338 cpu_vsyscall_notifier(struct notifier_block *n, unsigned long action, void *arg)
 339 {
 340         long cpu = (long)arg;
 341         if (action == CPU_ONLINE || action == CPU_ONLINE_FROZEN)
 342                 smp_call_function_single(cpu, cpu_vsyscall_init, NULL, 0, 1);
 343         return NOTIFY_DONE;
 344 }
 345
 346 void __init map_vsyscall(void)
 347 {
 348         extern char __vsyscall_0;
 349         unsigned long physaddr_page0 = __pa_symbol(&__vsyscall_0);
 350
 351         /* Note that VSYSCALL_MAPPED_PAGES must agree with the code below. */
 352         __set_fixmap(VSYSCALL_FIRST_PAGE, physaddr_page0, PAGE_KERNEL_VSYSCALL);
 353 }
 354
 355 static int __init vsyscall_init(void)
 356 {
 357         BUG_ON(((unsigned long) &vgettimeofday !=
 358                         VSYSCALL_ADDR(__NR_vgettimeofday)));
 359         BUG_ON((unsigned long) &vtime != VSYSCALL_ADDR(__NR_vtime));
 360         BUG_ON((VSYSCALL_ADDR(0) != __fix_to_virt(VSYSCALL_FIRST_PAGE)));
 361         BUG_ON((unsigned long) &vgetcpu != VSYSCALL_ADDR(__NR_vgetcpu));
 362 #ifdef CONFIG_SYSCTL
 363         register_sysctl_table(kernel_root_table2);
 364 #endif
 365         on_each_cpu(cpu_vsyscall_init, NULL, 0, 1);
 366         hotcpu_notifier(cpu_vsyscall_notifier, 0);
 367         return 0;
 368 }
 369
 370 __initcall(vsyscall_init);