sys/dev/virtual/hyperv/vmbus/x86_64/hyperv_machdep.c

   1 /*-
   2  * Copyright (c) 2016 Microsoft Corp.
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions
   7  * are met:
   8  * 1. Redistributions of source code must retain the above copyright
   9  *    notice unmodified, this list of conditions, and the following
  10  *    disclaimer.
  11  * 2. Redistributions in binary form must reproduce the above copyright
  12  *    notice, this list of conditions and the following disclaimer in the
  13  *    documentation and/or other materials provided with the distribution.
  14  *
  15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  16  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  17  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  18  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  19  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  20  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  21  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  22  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  23  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  24  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  25  */
  26
  27 #include <sys/param.h>
  28 #include <sys/systimer.h>
  29 #include <sys/systm.h>
  30
  31 #include <machine/cpufunc.h>
  32 #include <machine/specialreg.h>
  33 #include <machine/msi_machdep.h>
  34
  35 #include <dev/virtual/hyperv/hyperv_busdma.h>
  36 #include <dev/virtual/hyperv/vmbus/hyperv_reg.h>
  37 #include <dev/virtual/hyperv/vmbus/hyperv_var.h>
  38 #include <dev/virtual/hyperv/vmbus/hyperv_machdep.h>
  39
  40 struct hyperv_reftsc_ctx {
  41         struct hyperv_reftsc    *tsc_ref;
  42         struct hyperv_dma       tsc_ref_dma;
  43 };
  44
  45 static void             hyperv_tsc_cputimer_construct(struct cputimer *,
  46                             sysclock_t);
  47 static sysclock_t       hyperv_tsc_cputimer_count_mfence(void);
  48 static sysclock_t       hyperv_tsc_cputimer_count_lfence(void);
  49
  50 static struct hyperv_reftsc_ctx hyperv_ref_tsc;
  51 static hyperv_tc64_t    hyperv_tc64_saved;
  52
  53 static struct cputimer  hyperv_tsc_cputimer = {
  54         .next           = SLIST_ENTRY_INITIALIZER,
  55         .name           = "Hyper-V-TSC",
  56         .pri            = CPUTIMER_PRI_VMM_HI,
  57         .type           = CPUTIMER_VMM1,
  58         .count          = NULL, /* determined later */
  59         .fromhz         = cputimer_default_fromhz,
  60         .fromus         = cputimer_default_fromus,
  61         .construct      = hyperv_tsc_cputimer_construct,
  62         .destruct       = cputimer_default_destruct,
  63         .freq           = HYPERV_TIMER_FREQ
  64 };
  65
  66 uint64_t
  67 hypercall_md(volatile void *hc_addr, uint64_t in_val,
  68     uint64_t in_paddr, uint64_t out_paddr)
  69 {
  70         uint64_t status;
  71
  72         __asm__ __volatile__ ("mov %0, %%r8" : : "r" (out_paddr): "r8");
  73         __asm__ __volatile__ ("call *%3" : "=a" (status) :
  74             "c" (in_val), "d" (in_paddr), "m" (hc_addr));
  75         return (status);
  76 }
  77
  78 int
  79 hyperv_msi2vector(uint64_t msi_addr __unused, uint32_t msi_data)
  80 {
  81         return (msi_data & MSI_X86_DATA_INTVEC);
  82 }
  83
  84 #define HYPERV_TSC(fence)                                       \
  85 static uint64_t                                                 \
  86 hyperv_tsc_##fence(void)                                        \
  87 {                                                               \
  88         struct hyperv_reftsc *tsc_ref = hyperv_ref_tsc.tsc_ref; \
  89         uint32_t seq;                                           \
  90                                                                 \
  91         while ((seq = tsc_ref->tsc_seq) != 0) {                 \
  92                 uint64_t disc, ret, tsc;                        \
  93                 uint64_t scale;                                 \
  94                 int64_t ofs;                                    \
  95                                                                 \
  96                 cpu_ccfence();                                  \
  97                 scale = tsc_ref->tsc_scale;                     \
  98                 ofs = tsc_ref->tsc_ofs;                         \
  99                                                                 \
 100                 cpu_##fence();                                  \
 101                 tsc = rdtsc();                                  \
 102                                                                 \
 103                 /* ret = ((tsc * scale) >> 64) + ofs */         \
 104                 __asm__ __volatile__ ("mulq %3" :               \
 105                     "=d" (ret), "=a" (disc) :                   \
 106                     "a" (tsc), "r" (scale));                    \
 107                 ret += ofs;                                     \
 108                                                                 \
 109                 cpu_ccfence();                                  \
 110                 if (tsc_ref->tsc_seq == seq)                    \
 111                         return (ret);                           \
 112                                                                 \
 113                 /* Sequence changed; re-sync. */                \
 114         }                                                       \
 115         /* Fallback to the generic rdmsr. */                    \
 116         return (rdmsr(MSR_HV_TIME_REF_COUNT));                  \
 117 }                                                               \
 118 struct __hack
 119
 120 HYPERV_TSC(lfence);
 121 HYPERV_TSC(mfence);
 122
 123 static sysclock_t
 124 hyperv_tsc_cputimer_count_lfence(void)
 125 {
 126         uint64_t val;
 127
 128         val = hyperv_tsc_lfence();
 129         return (val + hyperv_tsc_cputimer.base);
 130 }
 131
 132 static sysclock_t
 133 hyperv_tsc_cputimer_count_mfence(void)
 134 {
 135         uint64_t val;
 136
 137         val = hyperv_tsc_mfence();
 138         return (val + hyperv_tsc_cputimer.base);
 139 }
 140
 141 static void
 142 hyperv_tsc_cputimer_construct(struct cputimer *timer, sysclock_t oldclock)
 143 {
 144         timer->base = 0;
 145         timer->base = oldclock - timer->count();
 146 }
 147
 148 void
 149 hyperv_md_init(void)
 150 {
 151         hyperv_tc64_t tc64 = NULL;
 152         uint64_t val, orig;
 153
 154         if ((hyperv_features &
 155              (CPUID_HV_MSR_TIME_REFCNT | CPUID_HV_MSR_REFERENCE_TSC)) !=
 156             (CPUID_HV_MSR_TIME_REFCNT | CPUID_HV_MSR_REFERENCE_TSC) ||
 157             (cpu_feature & CPUID_SSE2) == 0)    /* SSE2 for mfence/lfence */
 158                 return;
 159
 160         switch (cpu_vendor_id) {
 161         case CPU_VENDOR_AMD:
 162                 hyperv_tsc_cputimer.count = hyperv_tsc_cputimer_count_mfence;
 163                 tc64 = hyperv_tsc_mfence;
 164                 break;
 165
 166         case CPU_VENDOR_INTEL:
 167                 hyperv_tsc_cputimer.count = hyperv_tsc_cputimer_count_lfence;
 168                 tc64 = hyperv_tsc_lfence;
 169                 break;
 170
 171         default:
 172                 /* Unsupport CPU vendors. */
 173                 return;
 174         }
 175
 176         hyperv_ref_tsc.tsc_ref = hyperv_dmamem_alloc(NULL, PAGE_SIZE, 0,
 177             sizeof(struct hyperv_reftsc), &hyperv_ref_tsc.tsc_ref_dma,
 178             BUS_DMA_WAITOK | BUS_DMA_ZERO);
 179         if (hyperv_ref_tsc.tsc_ref == NULL) {
 180                 kprintf("hyperv: reftsc page allocation failed\n");
 181                 return;
 182         }
 183
 184         orig = rdmsr(MSR_HV_REFERENCE_TSC);
 185         val = MSR_HV_REFTSC_ENABLE | (orig & MSR_HV_REFTSC_RSVD_MASK) |
 186             ((hyperv_ref_tsc.tsc_ref_dma.hv_paddr >> PAGE_SHIFT) <<
 187              MSR_HV_REFTSC_PGSHIFT);
 188         wrmsr(MSR_HV_REFERENCE_TSC, val);
 189
 190         /* Register Hyper-V reference TSC systimer. */
 191         cputimer_register(&hyperv_tsc_cputimer);
 192         cputimer_select(&hyperv_tsc_cputimer, 0);
 193         KASSERT(tc64 != NULL, ("tc64 is not set"));
 194         hyperv_tc64_saved = hyperv_tc64;
 195         hyperv_tc64 = tc64;
 196 }
 197
 198 void
 199 hyperv_md_uninit(void)
 200 {
 201         if (hyperv_ref_tsc.tsc_ref != NULL) {
 202                 uint64_t val;
 203
 204                 /* Deregister Hyper-V reference TSC systimer. */
 205                 cputimer_deregister(&hyperv_tsc_cputimer);
 206                 /* Revert tc64 change. */
 207                 hyperv_tc64 = hyperv_tc64_saved;
 208
 209                 val = rdmsr(MSR_HV_REFERENCE_TSC);
 210                 wrmsr(MSR_HV_REFERENCE_TSC, val & MSR_HV_REFTSC_RSVD_MASK);
 211
 212                 hyperv_dmamem_free(&hyperv_ref_tsc.tsc_ref_dma,
 213                     hyperv_ref_tsc.tsc_ref);
 214                 hyperv_ref_tsc.tsc_ref = NULL;
 215         }
 216 }