2 * Simple C functions to supplement the C library
4 * Copyright (c) 2006 Fabrice Bellard
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to deal
8 * in the Software without restriction, including without limitation the rights
9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10 * copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
24 #include "qemu/osdep.h"
25 #include "qemu-common.h"
26 #include "qemu/cutils.h"
27 #include "qemu/bswap.h"
30 /* vector definitions */
32 extern void link_error(void);
34 #define ACCEL_BUFFER_ZERO(NAME, SIZE, VECTYPE, NONZERO) \
35 static bool NAME(const void *buf, size_t len) \
37 const void *end = buf + len; \
39 const VECTYPE *p = buf; \
41 __builtin_prefetch(buf + SIZE); \
43 if (SIZE == sizeof(VECTYPE) * 4) { \
44 t = (p[0] | p[1]) | (p[2] | p[3]); \
45 } else if (SIZE == sizeof(VECTYPE) * 8) { \
53 if (unlikely(NONZERO(t))) { \
57 } while (buf < end); \
62 buffer_zero_int(const void *buf
, size_t len
)
64 if (unlikely(len
< 8)) {
65 /* For a very small buffer, simply accumulate all the bytes. */
66 const unsigned char *p
= buf
;
67 const unsigned char *e
= buf
+ len
;
76 /* Otherwise, use the unaligned memory access functions to
77 handle the beginning and end of the buffer, with a couple
78 of loops handling the middle aligned section. */
79 uint64_t t
= ldq_he_p(buf
);
80 const uint64_t *p
= (uint64_t *)(((uintptr_t)buf
+ 8) & -8);
81 const uint64_t *e
= (uint64_t *)(((uintptr_t)buf
+ len
) & -8);
83 for (; p
+ 8 <= e
; p
+= 8) {
84 __builtin_prefetch(p
+ 8);
88 t
= p
[0] | p
[1] | p
[2] | p
[3] | p
[4] | p
[5] | p
[6] | p
[7];
93 t
|= ldq_he_p(buf
+ len
- 8);
99 #if defined(CONFIG_AVX2_OPT) || (defined(CONFIG_CPUID_H) && defined(__SSE2__))
102 /* Do not use push_options pragmas unnecessarily, because clang
103 * does not support them.
106 #pragma GCC push_options
107 #pragma GCC target("sse2")
109 #include <emmintrin.h>
110 #define SSE2_NONZERO(X) \
111 (_mm_movemask_epi8(_mm_cmpeq_epi8((X), _mm_setzero_si128())) != 0xFFFF)
112 ACCEL_BUFFER_ZERO(buffer_zero_sse2
, 64, __m128i
, SSE2_NONZERO
)
114 #pragma GCC pop_options
117 #ifdef CONFIG_AVX2_OPT
118 #pragma GCC push_options
119 #pragma GCC target("sse4")
120 #include <smmintrin.h>
121 #define SSE4_NONZERO(X) !_mm_testz_si128((X), (X))
122 ACCEL_BUFFER_ZERO(buffer_zero_sse4
, 64, __m128i
, SSE4_NONZERO
)
123 #pragma GCC pop_options
125 #pragma GCC push_options
126 #pragma GCC target("avx2")
127 #include <immintrin.h>
128 #define AVX2_NONZERO(X) !_mm256_testz_si256((X), (X))
129 ACCEL_BUFFER_ZERO(buffer_zero_avx2
, 128, __m256i
, AVX2_NONZERO
)
130 #pragma GCC pop_options
136 #define CACHE_SSE2 16
138 static unsigned cpuid_cache
;
140 static void __attribute__((constructor
)) init_cpuid_cache(void)
142 int max
= __get_cpuid_max(0, NULL
);
147 __cpuid(1, a
, b
, c
, d
);
151 #ifdef CONFIG_AVX2_OPT
152 if (c
& bit_SSE4_1
) {
156 /* We must check that AVX is not just available, but usable. */
157 if ((c
& bit_OSXSAVE
) && (c
& bit_AVX
)) {
158 __asm("xgetbv" : "=a"(a
), "=d"(d
) : "c"(0));
162 __cpuid_count(7, 0, a
, b
, c
, d
);
174 #define HAVE_NEXT_ACCEL
175 bool test_buffer_is_zero_next_accel(void)
177 /* If no bits set, we just tested buffer_zero_int, and there
178 are no more acceleration options to test. */
179 if (cpuid_cache
== 0) {
182 /* Disable the accelerator we used before and select a new one. */
183 cpuid_cache
&= cpuid_cache
- 1;
187 static bool select_accel_fn(const void *buf
, size_t len
)
189 uintptr_t ibuf
= (uintptr_t)buf
;
190 #ifdef CONFIG_AVX2_OPT
191 if (len
% 128 == 0 && ibuf
% 32 == 0 && (cpuid_cache
& CACHE_AVX2
)) {
192 return buffer_zero_avx2(buf
, len
);
194 if (len
% 64 == 0 && ibuf
% 16 == 0 && (cpuid_cache
& CACHE_SSE4
)) {
195 return buffer_zero_sse4(buf
, len
);
198 if (len
% 64 == 0 && ibuf
% 16 == 0 && (cpuid_cache
& CACHE_SSE2
)) {
199 return buffer_zero_sse2(buf
, len
);
201 return buffer_zero_int(buf
, len
);
205 #define select_accel_fn buffer_zero_int
208 #ifndef HAVE_NEXT_ACCEL
209 bool test_buffer_is_zero_next_accel(void)
216 * Checks if a buffer is all zeroes
218 bool buffer_is_zero(const void *buf
, size_t len
)
220 if (unlikely(len
== 0)) {
224 /* Fetch the beginning of the buffer while we select the accelerator. */
225 __builtin_prefetch(buf
);
227 /* Use an optimized zero check if possible. Note that this also
228 includes a check for an unrolled loop over 64-bit integers. */
229 return select_accel_fn(buf
, len
);