util/bufferiszero.c

   1 /*
   2  * Simple C functions to supplement the C library
   3  *
   4  * Copyright (c) 2006 Fabrice Bellard
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a copy
   7  * of this software and associated documentation files (the "Software"), to deal
   8  * in the Software without restriction, including without limitation the rights
   9  * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
  10  * copies of the Software, and to permit persons to whom the Software is
  11  * furnished to do so, subject to the following conditions:
  12  *
  13  * The above copyright notice and this permission notice shall be included in
  14  * all copies or substantial portions of the Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
  19  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  20  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  21  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  22  * THE SOFTWARE.
  23  */
  24 #include "qemu/osdep.h"
  25 #include "qemu-common.h"
  26 #include "qemu/cutils.h"
  27 #include "qemu/bswap.h"
  28
  29
  30 /* vector definitions */
  31
  32 extern void link_error(void);
  33
  34 #define ACCEL_BUFFER_ZERO(NAME, SIZE, VECTYPE, NONZERO)         \
  35 static bool NAME(const void *buf, size_t len)                   \
  36 {                                                               \
  37     const void *end = buf + len;                                \
  38     do {                                                        \
  39         const VECTYPE *p = buf;                                 \
  40         VECTYPE t;                                              \
  41         __builtin_prefetch(buf + SIZE);                         \
  42         barrier();                                              \
  43         if (SIZE == sizeof(VECTYPE) * 4) {                      \
  44             t = (p[0] | p[1]) | (p[2] | p[3]);                  \
  45         } else if (SIZE == sizeof(VECTYPE) * 8) {               \
  46             t  = p[0] | p[1];                                   \
  47             t |= p[2] | p[3];                                   \
  48             t |= p[4] | p[5];                                   \
  49             t |= p[6] | p[7];                                   \
  50         } else {                                                \
  51             link_error();                                       \
  52         }                                                       \
  53         if (unlikely(NONZERO(t))) {                             \
  54             return false;                                       \
  55         }                                                       \
  56         buf += SIZE;                                            \
  57     } while (buf < end);                                        \
  58     return true;                                                \
  59 }
  60
  61 static bool
  62 buffer_zero_int(const void *buf, size_t len)
  63 {
  64     if (unlikely(len < 8)) {
  65         /* For a very small buffer, simply accumulate all the bytes.  */
  66         const unsigned char *p = buf;
  67         const unsigned char *e = buf + len;
  68         unsigned char t = 0;
  69
  70         do {
  71             t |= *p++;
  72         } while (p < e);
  73
  74         return t == 0;
  75     } else {
  76         /* Otherwise, use the unaligned memory access functions to
  77            handle the beginning and end of the buffer, with a couple
  78            of loops handling the middle aligned section.  */
  79         uint64_t t = ldq_he_p(buf);
  80         const uint64_t *p = (uint64_t *)(((uintptr_t)buf + 8) & -8);
  81         const uint64_t *e = (uint64_t *)(((uintptr_t)buf + len) & -8);
  82
  83         for (; p + 8 <= e; p += 8) {
  84             __builtin_prefetch(p + 8);
  85             if (t) {
  86                 return false;
  87             }
  88             t = p[0] | p[1] | p[2] | p[3] | p[4] | p[5] | p[6] | p[7];
  89         }
  90         while (p < e) {
  91             t |= *p++;
  92         }
  93         t |= ldq_he_p(buf + len - 8);
  94
  95         return t == 0;
  96     }
  97 }
  98
  99 #if defined(CONFIG_AVX2_OPT) || (defined(CONFIG_CPUID_H) && defined(__SSE2__))
 100 #include <cpuid.h>
 101
 102 /* Do not use push_options pragmas unnecessarily, because clang
 103  * does not support them.
 104  */
 105 #ifndef __SSE2__
 106 #pragma GCC push_options
 107 #pragma GCC target("sse2")
 108 #endif
 109 #include <emmintrin.h>
 110 #define SSE2_NONZERO(X) \
 111     (_mm_movemask_epi8(_mm_cmpeq_epi8((X), _mm_setzero_si128())) != 0xFFFF)
 112 ACCEL_BUFFER_ZERO(buffer_zero_sse2, 64, __m128i, SSE2_NONZERO)
 113 #ifndef __SSE2__
 114 #pragma GCC pop_options
 115 #endif
 116
 117 #ifdef CONFIG_AVX2_OPT
 118 #pragma GCC push_options
 119 #pragma GCC target("sse4")
 120 #include <smmintrin.h>
 121 #define SSE4_NONZERO(X)  !_mm_testz_si128((X), (X))
 122 ACCEL_BUFFER_ZERO(buffer_zero_sse4, 64, __m128i, SSE4_NONZERO)
 123 #pragma GCC pop_options
 124
 125 #pragma GCC push_options
 126 #pragma GCC target("avx2")
 127 #include <immintrin.h>
 128 #define AVX2_NONZERO(X)  !_mm256_testz_si256((X), (X))
 129 ACCEL_BUFFER_ZERO(buffer_zero_avx2, 128, __m256i, AVX2_NONZERO)
 130 #pragma GCC pop_options
 131 #endif
 132
 133 #define CACHE_AVX2    2
 134 #define CACHE_AVX1    4
 135 #define CACHE_SSE4    8
 136 #define CACHE_SSE2    16
 137
 138 static unsigned cpuid_cache;
 139
 140 static void __attribute__((constructor)) init_cpuid_cache(void)
 141 {
 142     int max = __get_cpuid_max(0, NULL);
 143     int a, b, c, d;
 144     unsigned cache = 0;
 145
 146     if (max >= 1) {
 147         __cpuid(1, a, b, c, d);
 148         if (d & bit_SSE2) {
 149             cache |= CACHE_SSE2;
 150         }
 151 #ifdef CONFIG_AVX2_OPT
 152         if (c & bit_SSE4_1) {
 153             cache |= CACHE_SSE4;
 154         }
 155
 156         /* We must check that AVX is not just available, but usable.  */
 157         if ((c & bit_OSXSAVE) && (c & bit_AVX)) {
 158             __asm("xgetbv" : "=a"(a), "=d"(d) : "c"(0));
 159             if ((a & 6) == 6) {
 160                 cache |= CACHE_AVX1;
 161                 if (max >= 7) {
 162                     __cpuid_count(7, 0, a, b, c, d);
 163                     if (b & bit_AVX2) {
 164                         cache |= CACHE_AVX2;
 165                     }
 166                 }
 167             }
 168         }
 169 #endif
 170     }
 171     cpuid_cache = cache;
 172 }
 173
 174 #define HAVE_NEXT_ACCEL
 175 bool test_buffer_is_zero_next_accel(void)
 176 {
 177     /* If no bits set, we just tested buffer_zero_int, and there
 178        are no more acceleration options to test.  */
 179     if (cpuid_cache == 0) {
 180         return false;
 181     }
 182     /* Disable the accelerator we used before and select a new one.  */
 183     cpuid_cache &= cpuid_cache - 1;
 184     return true;
 185 }
 186
 187 static bool select_accel_fn(const void *buf, size_t len)
 188 {
 189     uintptr_t ibuf = (uintptr_t)buf;
 190 #ifdef CONFIG_AVX2_OPT
 191     if (len % 128 == 0 && ibuf % 32 == 0 && (cpuid_cache & CACHE_AVX2)) {
 192         return buffer_zero_avx2(buf, len);
 193     }
 194     if (len % 64 == 0 && ibuf % 16 == 0 && (cpuid_cache & CACHE_SSE4)) {
 195         return buffer_zero_sse4(buf, len);
 196     }
 197 #endif
 198     if (len % 64 == 0 && ibuf % 16 == 0 && (cpuid_cache & CACHE_SSE2)) {
 199         return buffer_zero_sse2(buf, len);
 200     }
 201     return buffer_zero_int(buf, len);
 202 }
 203
 204 #else
 205 #define select_accel_fn  buffer_zero_int
 206 #endif
 207
 208 #ifndef HAVE_NEXT_ACCEL
 209 bool test_buffer_is_zero_next_accel(void)
 210 {
 211     return false;
 212 }
 213 #endif
 214
 215 /*
 216  * Checks if a buffer is all zeroes
 217  */
 218 bool buffer_is_zero(const void *buf, size_t len)
 219 {
 220     if (unlikely(len == 0)) {
 221         return true;
 222     }
 223
 224     /* Fetch the beginning of the buffer while we select the accelerator.  */
 225     __builtin_prefetch(buf);
 226
 227     /* Use an optimized zero check if possible.  Note that this also
 228        includes a check for an unrolled loop over 64-bit integers.  */
 229     return select_accel_fn(buf, len);
 230 }