util/cacheflush.c

   1 /*
   2  * Flush the host cpu caches.
   3  *
   4  * This work is licensed under the terms of the GNU GPL, version 2 or later.
   5  * See the COPYING file in the top-level directory.
   6  */
   7
   8 #include "qemu/osdep.h"
   9 #include "qemu/cacheflush.h"
  10 #include "qemu/cacheinfo.h"
  11 #include "qemu/bitops.h"
  12
  13
  14 #if defined(__i386__) || defined(__x86_64__) || defined(__s390__)
  15
  16 /* Caches are coherent and do not require flushing; symbol inline. */
  17
  18 #elif defined(__aarch64__)
  19
  20 #ifdef CONFIG_DARWIN
  21 /* Apple does not expose CTR_EL0, so we must use system interfaces. */
  22 extern void sys_icache_invalidate(void *start, size_t len);
  23 extern void sys_dcache_flush(void *start, size_t len);
  24 void flush_idcache_range(uintptr_t rx, uintptr_t rw, size_t len)
  25 {
  26     sys_dcache_flush((void *)rw, len);
  27     sys_icache_invalidate((void *)rx, len);
  28 }
  29 #else
  30
  31 /*
  32  * TODO: unify this with cacheinfo.c.
  33  * We want to save the whole contents of CTR_EL0, so that we
  34  * have more than the linesize, but also IDC and DIC.
  35  */
  36 static uint64_t save_ctr_el0;
  37 static void __attribute__((constructor)) init_ctr_el0(void)
  38 {
  39     asm volatile("mrs\t%0, ctr_el0" : "=r"(save_ctr_el0));
  40 }
  41
  42 /*
  43  * This is a copy of gcc's __aarch64_sync_cache_range, modified
  44  * to fit this three-operand interface.
  45  */
  46 void flush_idcache_range(uintptr_t rx, uintptr_t rw, size_t len)
  47 {
  48     const unsigned CTR_IDC = 1u << 28;
  49     const unsigned CTR_DIC = 1u << 29;
  50     const uint64_t ctr_el0 = save_ctr_el0;
  51     const uintptr_t icache_lsize = 4 << extract64(ctr_el0, 0, 4);
  52     const uintptr_t dcache_lsize = 4 << extract64(ctr_el0, 16, 4);
  53     uintptr_t p;
  54
  55     /*
  56      * If CTR_EL0.IDC is enabled, Data cache clean to the Point of Unification
  57      * is not required for instruction to data coherence.
  58      */
  59     if (!(ctr_el0 & CTR_IDC)) {
  60         /*
  61          * Loop over the address range, clearing one cache line at once.
  62          * Data cache must be flushed to unification first to make sure
  63          * the instruction cache fetches the updated data.
  64          */
  65         for (p = rw & -dcache_lsize; p < rw + len; p += dcache_lsize) {
  66             asm volatile("dc\tcvau, %0" : : "r" (p) : "memory");
  67         }
  68         asm volatile("dsb\tish" : : : "memory");
  69     }
  70
  71     /*
  72      * If CTR_EL0.DIC is enabled, Instruction cache cleaning to the Point
  73      * of Unification is not required for instruction to data coherence.
  74      */
  75     if (!(ctr_el0 & CTR_DIC)) {
  76         for (p = rx & -icache_lsize; p < rx + len; p += icache_lsize) {
  77             asm volatile("ic\tivau, %0" : : "r"(p) : "memory");
  78         }
  79         asm volatile ("dsb\tish" : : : "memory");
  80     }
  81
  82     asm volatile("isb" : : : "memory");
  83 }
  84 #endif /* CONFIG_DARWIN */
  85
  86 #elif defined(__mips__)
  87
  88 #ifdef __OpenBSD__
  89 #include <machine/sysarch.h>
  90 #else
  91 #include <sys/cachectl.h>
  92 #endif
  93
  94 void flush_idcache_range(uintptr_t rx, uintptr_t rw, size_t len)
  95 {
  96     if (rx != rw) {
  97         cacheflush((void *)rw, len, DCACHE);
  98     }
  99     cacheflush((void *)rx, len, ICACHE);
 100 }
 101
 102 #elif defined(__powerpc__)
 103
 104 void flush_idcache_range(uintptr_t rx, uintptr_t rw, size_t len)
 105 {
 106     uintptr_t p, b, e;
 107     size_t dsize = qemu_dcache_linesize;
 108     size_t isize = qemu_icache_linesize;
 109
 110     b = rw & ~(dsize - 1);
 111     e = (rw + len + dsize - 1) & ~(dsize - 1);
 112     for (p = b; p < e; p += dsize) {
 113         asm volatile ("dcbst 0,%0" : : "r"(p) : "memory");
 114     }
 115     asm volatile ("sync" : : : "memory");
 116
 117     b = rx & ~(isize - 1);
 118     e = (rx + len + isize - 1) & ~(isize - 1);
 119     for (p = b; p < e; p += isize) {
 120         asm volatile ("icbi 0,%0" : : "r"(p) : "memory");
 121     }
 122     asm volatile ("sync" : : : "memory");
 123     asm volatile ("isync" : : : "memory");
 124 }
 125
 126 #elif defined(__sparc__)
 127
 128 void flush_idcache_range(uintptr_t rx, uintptr_t rw, size_t len)
 129 {
 130     /* No additional data flush to the RW virtual address required. */
 131     uintptr_t p, end = (rx + len + 7) & -8;
 132     for (p = rx & -8; p < end; p += 8) {
 133         __asm__ __volatile__("flush\t%0" : : "r" (p));
 134     }
 135 }
 136
 137 #else
 138
 139 void flush_idcache_range(uintptr_t rx, uintptr_t rw, size_t len)
 140 {
 141     if (rw != rx) {
 142         __builtin___clear_cache((char *)rw, (char *)rw + len);
 143     }
 144     __builtin___clear_cache((char *)rx, (char *)rx + len);
 145 }
 146
 147 #endif