util/bufferiszero.c

   1 /*
   2  * Simple C functions to supplement the C library
   3  *
   4  * Copyright (c) 2006 Fabrice Bellard
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a copy
   7  * of this software and associated documentation files (the "Software"), to deal
   8  * in the Software without restriction, including without limitation the rights
   9  * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
  10  * copies of the Software, and to permit persons to whom the Software is
  11  * furnished to do so, subject to the following conditions:
  12  *
  13  * The above copyright notice and this permission notice shall be included in
  14  * all copies or substantial portions of the Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
  19  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  20  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  21  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  22  * THE SOFTWARE.
  23  */
  24 #include "qemu/osdep.h"
  25 #include "qemu/cutils.h"
  26 #include "qemu/bswap.h"
  27 #include "host/cpuinfo.h"
  28
  29 typedef bool (*biz_accel_fn)(const void *, size_t);
  30
  31 static bool buffer_is_zero_int_lt256(const void *buf, size_t len)
  32 {
  33     uint64_t t;
  34     const uint64_t *p, *e;
  35
  36     /*
  37      * Use unaligned memory access functions to handle
  38      * the beginning and end of the buffer.
  39      */
  40     if (unlikely(len <= 8)) {
  41         return (ldl_he_p(buf) | ldl_he_p(buf + len - 4)) == 0;
  42     }
  43
  44     t = ldq_he_p(buf) | ldq_he_p(buf + len - 8);
  45     p = QEMU_ALIGN_PTR_DOWN(buf + 8, 8);
  46     e = QEMU_ALIGN_PTR_DOWN(buf + len - 1, 8);
  47
  48     /* Read 0 to 31 aligned words from the middle. */
  49     while (p < e) {
  50         t |= *p++;
  51     }
  52     return t == 0;
  53 }
  54
  55 static bool buffer_is_zero_int_ge256(const void *buf, size_t len)
  56 {
  57     /*
  58      * Use unaligned memory access functions to handle
  59      * the beginning and end of the buffer.
  60      */
  61     uint64_t t = ldq_he_p(buf) | ldq_he_p(buf + len - 8);
  62     const uint64_t *p = QEMU_ALIGN_PTR_DOWN(buf + 8, 8);
  63     const uint64_t *e = QEMU_ALIGN_PTR_DOWN(buf + len - 1, 8);
  64
  65     /* Collect a partial block at the tail end. */
  66     t |= e[-7] | e[-6] | e[-5] | e[-4] | e[-3] | e[-2] | e[-1];
  67
  68     /*
  69      * Loop over 64 byte blocks.
  70      * With the head and tail removed, e - p >= 30,
  71      * so the loop must iterate at least 3 times.
  72      */
  73     do {
  74         if (t) {
  75             return false;
  76         }
  77         t = p[0] | p[1] | p[2] | p[3] | p[4] | p[5] | p[6] | p[7];
  78         p += 8;
  79     } while (p < e - 7);
  80
  81     return t == 0;
  82 }
  83
  84 #include "host/bufferiszero.c.inc"
  85
  86 static biz_accel_fn buffer_is_zero_accel;
  87 static unsigned accel_index;
  88
  89 bool buffer_is_zero_ool(const void *buf, size_t len)
  90 {
  91     if (unlikely(len == 0)) {
  92         return true;
  93     }
  94     if (!buffer_is_zero_sample3(buf, len)) {
  95         return false;
  96     }
  97     /* All bytes are covered for any len <= 3.  */
  98     if (unlikely(len <= 3)) {
  99         return true;
 100     }
 101
 102     if (likely(len >= 256)) {
 103         return buffer_is_zero_accel(buf, len);
 104     }
 105     return buffer_is_zero_int_lt256(buf, len);
 106 }
 107
 108 bool buffer_is_zero_ge256(const void *buf, size_t len)
 109 {
 110     return buffer_is_zero_accel(buf, len);
 111 }
 112
 113 bool test_buffer_is_zero_next_accel(void)
 114 {
 115     if (accel_index != 0) {
 116         buffer_is_zero_accel = accel_table[--accel_index];
 117         return true;
 118     }
 119     return false;
 120 }
 121
 122 static void __attribute__((constructor)) init_accel(void)
 123 {
 124     accel_index = best_accel();
 125     buffer_is_zero_accel = accel_table[accel_index];
 126 }