util/cacheflush.c

   1 /*
   2  * Info about, and flushing the host cpu caches.
   3  *
   4  * This work is licensed under the terms of the GNU GPL, version 2 or later.
   5  * See the COPYING file in the top-level directory.
   6  */
   7
   8 #include "qemu/osdep.h"
   9 #include "qemu/cacheflush.h"
  10 #include "qemu/cacheinfo.h"
  11 #include "qemu/bitops.h"
  12 #include "qemu/host-utils.h"
  13 #include "qemu/atomic.h"
  14
  15
  16 int qemu_icache_linesize = 0;
  17 int qemu_icache_linesize_log;
  18 int qemu_dcache_linesize = 0;
  19 int qemu_dcache_linesize_log;
  20
  21 /*
  22  * Operating system specific cache detection mechanisms.
  23  */
  24
  25 #if defined(_WIN32)
  26
  27 static void sys_cache_info(int *isize, int *dsize)
  28 {
  29     SYSTEM_LOGICAL_PROCESSOR_INFORMATION *buf;
  30     DWORD size = 0;
  31     BOOL success;
  32     size_t i, n;
  33
  34     /*
  35      * Check for the required buffer size first.  Note that if the zero
  36      * size we use for the probe results in success, then there is no
  37      * data available; fail in that case.
  38      */
  39     success = GetLogicalProcessorInformation(0, &size);
  40     if (success || GetLastError() != ERROR_INSUFFICIENT_BUFFER) {
  41         return;
  42     }
  43
  44     n = size / sizeof(SYSTEM_LOGICAL_PROCESSOR_INFORMATION);
  45     size = n * sizeof(SYSTEM_LOGICAL_PROCESSOR_INFORMATION);
  46     buf = g_new0(SYSTEM_LOGICAL_PROCESSOR_INFORMATION, n);
  47     if (!GetLogicalProcessorInformation(buf, &size)) {
  48         goto fail;
  49     }
  50
  51     for (i = 0; i < n; i++) {
  52         if (buf[i].Relationship == RelationCache
  53             && buf[i].Cache.Level == 1) {
  54             switch (buf[i].Cache.Type) {
  55             case CacheUnified:
  56                 *isize = *dsize = buf[i].Cache.LineSize;
  57                 break;
  58             case CacheInstruction:
  59                 *isize = buf[i].Cache.LineSize;
  60                 break;
  61             case CacheData:
  62                 *dsize = buf[i].Cache.LineSize;
  63                 break;
  64             default:
  65                 break;
  66             }
  67         }
  68     }
  69  fail:
  70     g_free(buf);
  71 }
  72
  73 #elif defined(CONFIG_DARWIN)
  74 # include <sys/sysctl.h>
  75 static void sys_cache_info(int *isize, int *dsize)
  76 {
  77     /* There's only a single sysctl for both I/D cache line sizes.  */
  78     long size;
  79     size_t len = sizeof(size);
  80     if (!sysctlbyname("hw.cachelinesize", &size, &len, NULL, 0)) {
  81         *isize = *dsize = size;
  82     }
  83 }
  84 #elif defined(__FreeBSD__) || defined(__FreeBSD_kernel__)
  85 # include <sys/sysctl.h>
  86 static void sys_cache_info(int *isize, int *dsize)
  87 {
  88     /* There's only a single sysctl for both I/D cache line sizes.  */
  89     int size;
  90     size_t len = sizeof(size);
  91     if (!sysctlbyname("machdep.cacheline_size", &size, &len, NULL, 0)) {
  92         *isize = *dsize = size;
  93     }
  94 }
  95 #else
  96 /* POSIX */
  97
  98 static void sys_cache_info(int *isize, int *dsize)
  99 {
 100 # ifdef _SC_LEVEL1_ICACHE_LINESIZE
 101     int tmp_isize = (int) sysconf(_SC_LEVEL1_ICACHE_LINESIZE);
 102     if (tmp_isize > 0) {
 103         *isize = tmp_isize;
 104     }
 105 # endif
 106 # ifdef _SC_LEVEL1_DCACHE_LINESIZE
 107     int tmp_dsize = (int) sysconf(_SC_LEVEL1_DCACHE_LINESIZE);
 108     if (tmp_dsize > 0) {
 109         *dsize = tmp_dsize;
 110     }
 111 # endif
 112 }
 113 #endif /* sys_cache_info */
 114
 115
 116 /*
 117  * Architecture (+ OS) specific cache detection mechanisms.
 118  */
 119
 120 #if defined(__powerpc__)
 121 static bool have_coherent_icache;
 122 #endif
 123
 124 #if defined(__aarch64__) && !defined(CONFIG_DARWIN)
 125 /* Apple does not expose CTR_EL0, so we must use system interfaces. */
 126 static uint64_t save_ctr_el0;
 127 static void arch_cache_info(int *isize, int *dsize)
 128 {
 129     uint64_t ctr;
 130
 131     /*
 132      * The real cache geometry is in CCSIDR_EL1/CLIDR_EL1/CSSELR_EL1,
 133      * but (at least under Linux) these are marked protected by the
 134      * kernel.  However, CTR_EL0 contains the minimum linesize in the
 135      * entire hierarchy, and is used by userspace cache flushing.
 136      *
 137      * We will also use this value in flush_idcache_range.
 138      */
 139     asm volatile("mrs\t%0, ctr_el0" : "=r"(ctr));
 140     save_ctr_el0 = ctr;
 141
 142     if (*isize == 0 || *dsize == 0) {
 143         if (*isize == 0) {
 144             *isize = 4 << (ctr & 0xf);
 145         }
 146         if (*dsize == 0) {
 147             *dsize = 4 << ((ctr >> 16) & 0xf);
 148         }
 149     }
 150 }
 151
 152 #elif defined(_ARCH_PPC) && defined(__linux__)
 153 # include "elf.h"
 154
 155 static void arch_cache_info(int *isize, int *dsize)
 156 {
 157     if (*isize == 0) {
 158         *isize = qemu_getauxval(AT_ICACHEBSIZE);
 159     }
 160     if (*dsize == 0) {
 161         *dsize = qemu_getauxval(AT_DCACHEBSIZE);
 162     }
 163     have_coherent_icache = qemu_getauxval(AT_HWCAP) & PPC_FEATURE_ICACHE_SNOOP;
 164 }
 165
 166 #else
 167 static void arch_cache_info(int *isize, int *dsize) { }
 168 #endif /* arch_cache_info */
 169
 170 /*
 171  * ... and if all else fails ...
 172  */
 173
 174 static void fallback_cache_info(int *isize, int *dsize)
 175 {
 176     /* If we can only find one of the two, assume they're the same.  */
 177     if (*isize) {
 178         if (*dsize) {
 179             /* Success! */
 180         } else {
 181             *dsize = *isize;
 182         }
 183     } else if (*dsize) {
 184         *isize = *dsize;
 185     } else {
 186 #if defined(_ARCH_PPC)
 187         /*
 188          * For PPC, we're going to use the cache sizes computed for
 189          * flush_idcache_range.  Which means that we must use the
 190          * architecture minimum.
 191          */
 192         *isize = *dsize = 16;
 193 #else
 194         /* Otherwise, 64 bytes is not uncommon.  */
 195         *isize = *dsize = 64;
 196 #endif
 197     }
 198 }
 199
 200 static void __attribute__((constructor)) init_cache_info(void)
 201 {
 202     int isize = 0, dsize = 0;
 203
 204     sys_cache_info(&isize, &dsize);
 205     arch_cache_info(&isize, &dsize);
 206     fallback_cache_info(&isize, &dsize);
 207
 208     assert((isize & (isize - 1)) == 0);
 209     assert((dsize & (dsize - 1)) == 0);
 210
 211     qemu_icache_linesize = isize;
 212     qemu_icache_linesize_log = ctz32(isize);
 213     qemu_dcache_linesize = dsize;
 214     qemu_dcache_linesize_log = ctz32(dsize);
 215
 216     qatomic64_init();
 217 }
 218
 219
 220 /*
 221  * Architecture (+ OS) specific cache flushing mechanisms.
 222  */
 223
 224 #if defined(__i386__) || defined(__x86_64__) || defined(__s390__)
 225
 226 /* Caches are coherent and do not require flushing; symbol inline. */
 227
 228 #elif defined(__aarch64__)
 229
 230 #ifdef CONFIG_DARWIN
 231 /* Apple does not expose CTR_EL0, so we must use system interfaces. */
 232 extern void sys_icache_invalidate(void *start, size_t len);
 233 extern void sys_dcache_flush(void *start, size_t len);
 234 void flush_idcache_range(uintptr_t rx, uintptr_t rw, size_t len)
 235 {
 236     sys_dcache_flush((void *)rw, len);
 237     sys_icache_invalidate((void *)rx, len);
 238 }
 239 #else
 240
 241 /*
 242  * This is a copy of gcc's __aarch64_sync_cache_range, modified
 243  * to fit this three-operand interface.
 244  */
 245 void flush_idcache_range(uintptr_t rx, uintptr_t rw, size_t len)
 246 {
 247     const unsigned CTR_IDC = 1u << 28;
 248     const unsigned CTR_DIC = 1u << 29;
 249     const uint64_t ctr_el0 = save_ctr_el0;
 250     const uintptr_t icache_lsize = qemu_icache_linesize;
 251     const uintptr_t dcache_lsize = qemu_dcache_linesize;
 252     uintptr_t p;
 253
 254     /*
 255      * If CTR_EL0.IDC is enabled, Data cache clean to the Point of Unification
 256      * is not required for instruction to data coherence.
 257      */
 258     if (!(ctr_el0 & CTR_IDC)) {
 259         /*
 260          * Loop over the address range, clearing one cache line at once.
 261          * Data cache must be flushed to unification first to make sure
 262          * the instruction cache fetches the updated data.
 263          */
 264         for (p = rw & -dcache_lsize; p < rw + len; p += dcache_lsize) {
 265             asm volatile("dc\tcvau, %0" : : "r" (p) : "memory");
 266         }
 267         asm volatile("dsb\tish" : : : "memory");
 268     }
 269
 270     /*
 271      * If CTR_EL0.DIC is enabled, Instruction cache cleaning to the Point
 272      * of Unification is not required for instruction to data coherence.
 273      */
 274     if (!(ctr_el0 & CTR_DIC)) {
 275         for (p = rx & -icache_lsize; p < rx + len; p += icache_lsize) {
 276             asm volatile("ic\tivau, %0" : : "r"(p) : "memory");
 277         }
 278         asm volatile ("dsb\tish" : : : "memory");
 279     }
 280
 281     asm volatile("isb" : : : "memory");
 282 }
 283 #endif /* CONFIG_DARWIN */
 284
 285 #elif defined(__mips__)
 286
 287 #ifdef __OpenBSD__
 288 #include <machine/sysarch.h>
 289 #else
 290 #include <sys/cachectl.h>
 291 #endif
 292
 293 void flush_idcache_range(uintptr_t rx, uintptr_t rw, size_t len)
 294 {
 295     if (rx != rw) {
 296         cacheflush((void *)rw, len, DCACHE);
 297     }
 298     cacheflush((void *)rx, len, ICACHE);
 299 }
 300
 301 #elif defined(__powerpc__)
 302
 303 void flush_idcache_range(uintptr_t rx, uintptr_t rw, size_t len)
 304 {
 305     uintptr_t p, b, e;
 306     size_t dsize, isize;
 307
 308     /*
 309      * Some processors have coherent caches and support a simplified
 310      * flushing procedure.  See
 311      *   POWER9 UM, 4.6.2.2 Instruction Cache Block Invalidate (icbi)
 312      *   https://ibm.ent.box.com/s/tmklq90ze7aj8f4n32er1mu3sy9u8k3k
 313      */
 314     if (have_coherent_icache) {
 315         asm volatile ("sync\n\t"
 316                       "icbi 0,%0\n\t"
 317                       "isync"
 318                       : : "r"(rx) : "memory");
 319         return;
 320     }
 321
 322     dsize = qemu_dcache_linesize;
 323     isize = qemu_icache_linesize;
 324
 325     b = rw & ~(dsize - 1);
 326     e = (rw + len + dsize - 1) & ~(dsize - 1);
 327     for (p = b; p < e; p += dsize) {
 328         asm volatile ("dcbst 0,%0" : : "r"(p) : "memory");
 329     }
 330     asm volatile ("sync" : : : "memory");
 331
 332     b = rx & ~(isize - 1);
 333     e = (rx + len + isize - 1) & ~(isize - 1);
 334     for (p = b; p < e; p += isize) {
 335         asm volatile ("icbi 0,%0" : : "r"(p) : "memory");
 336     }
 337     asm volatile ("sync" : : : "memory");
 338     asm volatile ("isync" : : : "memory");
 339 }
 340
 341 #elif defined(__sparc__)
 342
 343 void flush_idcache_range(uintptr_t rx, uintptr_t rw, size_t len)
 344 {
 345     /* No additional data flush to the RW virtual address required. */
 346     uintptr_t p, end = (rx + len + 7) & -8;
 347     for (p = rx & -8; p < end; p += 8) {
 348         __asm__ __volatile__("flush\t%0" : : "r" (p));
 349     }
 350 }
 351
 352 #else
 353
 354 void flush_idcache_range(uintptr_t rx, uintptr_t rw, size_t len)
 355 {
 356     if (rw != rx) {
 357         __builtin___clear_cache((char *)rw, (char *)rw + len);
 358     }
 359     __builtin___clear_cache((char *)rx, (char *)rx + len);
 360 }
 361
 362 #endif