sys/dev/virtual/hyperv/vmbus/x86_64/hyperv_machdep.c

   1 /*-
   2  * Copyright (c) 2016 Microsoft Corp.
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions
   7  * are met:
   8  * 1. Redistributions of source code must retain the above copyright
   9  *    notice unmodified, this list of conditions, and the following
  10  *    disclaimer.
  11  * 2. Redistributions in binary form must reproduce the above copyright
  12  *    notice, this list of conditions and the following disclaimer in the
  13  *    documentation and/or other materials provided with the distribution.
  14  *
  15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  16  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  17  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  18  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  19  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  20  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  21  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  22  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  23  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  24  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  25  */
  26
  27 #include <sys/param.h>
  28 #include <sys/systimer.h>
  29 #include <sys/systm.h>
  30
  31 #include <machine/cpufunc.h>
  32 #include <machine/specialreg.h>
  33 #include <machine/msi_machdep.h>
  34
  35 #include <dev/virtual/hyperv/hyperv_busdma.h>
  36 #include <dev/virtual/hyperv/vmbus/hyperv_reg.h>
  37 #include <dev/virtual/hyperv/vmbus/hyperv_var.h>
  38 #include <dev/virtual/hyperv/vmbus/hyperv_machdep.h>
  39
  40 struct hyperv_reftsc_ctx {
  41         struct hyperv_reftsc    *tsc_ref;
  42         struct hyperv_dma       tsc_ref_dma;
  43 };
  44
  45 static void             hyperv_tsc_cputimer_construct(struct cputimer *,
  46                             sysclock_t);
  47 static sysclock_t       hyperv_tsc_cputimer_count_mfence(void);
  48 static sysclock_t       hyperv_tsc_cputimer_count_lfence(void);
  49
  50 static struct hyperv_reftsc_ctx hyperv_ref_tsc;
  51 static hyperv_tc64_t    hyperv_tc64_saved;
  52
  53 static struct cputimer  hyperv_tsc_cputimer = {
  54         SLIST_ENTRY_INITIALIZER,
  55         "Hyper-V-TSC",
  56         CPUTIMER_PRI_VMM_HI,
  57         CPUTIMER_VMM1,
  58         NULL,                           /* based on CPU types. */
  59         cputimer_default_fromhz,
  60         cputimer_default_fromus,
  61         hyperv_tsc_cputimer_construct,
  62         cputimer_default_destruct,
  63         HYPERV_TIMER_FREQ,
  64         0, 0, 0
  65 };
  66
  67 uint64_t
  68 hypercall_md(volatile void *hc_addr, uint64_t in_val,
  69     uint64_t in_paddr, uint64_t out_paddr)
  70 {
  71         uint64_t status;
  72
  73         __asm__ __volatile__ ("mov %0, %%r8" : : "r" (out_paddr): "r8");
  74         __asm__ __volatile__ ("call *%3" : "=a" (status) :
  75             "c" (in_val), "d" (in_paddr), "m" (hc_addr));
  76         return (status);
  77 }
  78
  79 int
  80 hyperv_msi2vector(uint64_t msi_addr __unused, uint32_t msi_data)
  81 {
  82         return (msi_data & MSI_X86_DATA_INTVEC);
  83 }
  84
  85 #define HYPERV_TSC(fence)                                       \
  86 static uint64_t                                                 \
  87 hyperv_tsc_##fence(void)                                        \
  88 {                                                               \
  89         struct hyperv_reftsc *tsc_ref = hyperv_ref_tsc.tsc_ref; \
  90         uint32_t seq;                                           \
  91                                                                 \
  92         while ((seq = tsc_ref->tsc_seq) != 0) {                 \
  93                 uint64_t disc, ret, tsc;                        \
  94                 uint64_t scale;                                 \
  95                 int64_t ofs;                                    \
  96                                                                 \
  97                 cpu_ccfence();                                  \
  98                 scale = tsc_ref->tsc_scale;                     \
  99                 ofs = tsc_ref->tsc_ofs;                         \
 100                                                                 \
 101                 cpu_##fence();                                  \
 102                 tsc = rdtsc();                                  \
 103                                                                 \
 104                 /* ret = ((tsc * scale) >> 64) + ofs */         \
 105                 __asm__ __volatile__ ("mulq %3" :               \
 106                     "=d" (ret), "=a" (disc) :                   \
 107                     "a" (tsc), "r" (scale));                    \
 108                 ret += ofs;                                     \
 109                                                                 \
 110                 cpu_ccfence();                                  \
 111                 if (tsc_ref->tsc_seq == seq)                    \
 112                         return (ret);                           \
 113                                                                 \
 114                 /* Sequence changed; re-sync. */                \
 115         }                                                       \
 116         /* Fallback to the generic rdmsr. */                    \
 117         return (rdmsr(MSR_HV_TIME_REF_COUNT));                  \
 118 }                                                               \
 119 struct __hack
 120
 121 HYPERV_TSC(lfence);
 122 HYPERV_TSC(mfence);
 123
 124 static sysclock_t
 125 hyperv_tsc_cputimer_count_lfence(void)
 126 {
 127         uint64_t val;
 128
 129         val = hyperv_tsc_lfence();
 130         return (val + hyperv_tsc_cputimer.base);
 131 }
 132
 133 static sysclock_t
 134 hyperv_tsc_cputimer_count_mfence(void)
 135 {
 136         uint64_t val;
 137
 138         val = hyperv_tsc_mfence();
 139         return (val + hyperv_tsc_cputimer.base);
 140 }
 141
 142 static void
 143 hyperv_tsc_cputimer_construct(struct cputimer *timer, sysclock_t oldclock)
 144 {
 145         timer->base = 0;
 146         timer->base = oldclock - timer->count();
 147 }
 148
 149 void
 150 hyperv_md_init(void)
 151 {
 152         uint64_t val, orig;
 153
 154         if ((hyperv_features &
 155              (CPUID_HV_MSR_TIME_REFCNT | CPUID_HV_MSR_REFERENCE_TSC)) !=
 156             (CPUID_HV_MSR_TIME_REFCNT | CPUID_HV_MSR_REFERENCE_TSC) ||
 157             (cpu_feature & CPUID_SSE2) == 0)    /* SSE2 for mfence/lfence */
 158                 return;
 159
 160         hyperv_ref_tsc.tsc_ref = hyperv_dmamem_alloc(NULL, PAGE_SIZE, 0,
 161             sizeof(struct hyperv_reftsc), &hyperv_ref_tsc.tsc_ref_dma,
 162             BUS_DMA_WAITOK | BUS_DMA_ZERO);
 163         if (hyperv_ref_tsc.tsc_ref == NULL) {
 164                 kprintf("hyperv: reftsc page allocation failed\n");
 165                 return;
 166         }
 167
 168         hyperv_tc64_saved = hyperv_tc64;
 169         switch (cpu_vendor_id) {
 170         case CPU_VENDOR_AMD:
 171                 hyperv_tsc_cputimer.count = hyperv_tsc_cputimer_count_mfence;
 172                 hyperv_tc64 = hyperv_tsc_mfence;
 173                 break;
 174
 175         case CPU_VENDOR_INTEL:
 176                 hyperv_tsc_cputimer.count = hyperv_tsc_cputimer_count_lfence;
 177                 hyperv_tc64 = hyperv_tsc_lfence;
 178                 break;
 179
 180         default:
 181                 /* Unsupport CPU vendors. */
 182                 hyperv_dmamem_free(&hyperv_ref_tsc.tsc_ref_dma,
 183                     hyperv_ref_tsc.tsc_ref);
 184                 hyperv_ref_tsc.tsc_ref = NULL;
 185                 return;
 186         }
 187
 188         orig = rdmsr(MSR_HV_REFERENCE_TSC);
 189         val = MSR_HV_REFTSC_ENABLE | (orig & MSR_HV_REFTSC_RSVD_MASK) |
 190             ((hyperv_ref_tsc.tsc_ref_dma.hv_paddr >> PAGE_SHIFT) <<
 191              MSR_HV_REFTSC_PGSHIFT);
 192         wrmsr(MSR_HV_REFERENCE_TSC, val);
 193
 194         /* Register Hyper-V reference TSC systimer. */
 195         cputimer_register(&hyperv_tsc_cputimer);
 196         cputimer_select(&hyperv_tsc_cputimer, 0);
 197 }
 198
 199 void
 200 hyperv_md_uninit(void)
 201 {
 202         if (hyperv_ref_tsc.tsc_ref != NULL) {
 203                 uint64_t val;
 204
 205                 /* Deregister Hyper-V reference TSC systimer. */
 206                 cputimer_deregister(&hyperv_tsc_cputimer);
 207                 /* Revert tc64 change. */
 208                 hyperv_tc64 = hyperv_tc64_saved;
 209
 210                 val = rdmsr(MSR_HV_REFERENCE_TSC);
 211                 wrmsr(MSR_HV_REFERENCE_TSC, val & MSR_HV_REFTSC_RSVD_MASK);
 212
 213                 hyperv_dmamem_free(&hyperv_ref_tsc.tsc_ref_dma,
 214                     hyperv_ref_tsc.tsc_ref);
 215                 hyperv_ref_tsc.tsc_ref = NULL;
 216         }
 217 }