?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Handle member initializer in C++ ctor.
[clang.git] / lib / Headers / avxintrin.h
blob884d31cb8917b966fdf74596f2920218223cf8fc
1 /*===---- avxintrin.h - AVX intrinsics -------------------------------------===
2 *
3 * Permission is hereby granted, free of charge, to any person obtaining a copy
4 * of this software and associated documentation files (the "Software"), to deal
5 * in the Software without restriction, including without limitation the rights
6 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7 * copies of the Software, and to permit persons to whom the Software is
8 * furnished to do so, subject to the following conditions:
9 *
10 * The above copyright notice and this permission notice shall be included in
11 * all copies or substantial portions of the Software.
12 *
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
19 * THE SOFTWARE.
20 *
21 *===-----------------------------------------------------------------------===
22 */
23
24 #ifndef __IMMINTRIN_H
25 #error "Never use <avxintrin.h> directly; include <immintrin.h> instead."
26 #endif
27
28 typedef double __v4df __attribute__ ((__vector_size__ (32)));
29 typedef float __v8sf __attribute__ ((__vector_size__ (32)));
30 typedef long long __v4di __attribute__ ((__vector_size__ (32)));
31 typedef int __v8si __attribute__ ((__vector_size__ (32)));
32 typedef short __v16hi __attribute__ ((__vector_size__ (32)));
33 typedef char __v32qi __attribute__ ((__vector_size__ (32)));
34
35 typedef float __m256 __attribute__ ((__vector_size__ (32)));
36 typedef double __m256d __attribute__((__vector_size__(32)));
37 typedef long long __m256i __attribute__((__vector_size__(32)));
38
39 /* Arithmetic */
40 static __inline __m256d __attribute__((__always_inline__, __nodebug__))
41 _mm256_add_pd(__m256d a, __m256d b)
42 {
43 return a+b;
44 }
45
46 static __inline __m256 __attribute__((__always_inline__, __nodebug__))
47 _mm256_add_ps(__m256 a, __m256 b)
48 {
49 return a+b;
50 }
51
52 static __inline __m256d __attribute__((__always_inline__, __nodebug__))
53 _mm256_sub_pd(__m256d a, __m256d b)
54 {
55 return a-b;
56 }
57
58 static __inline __m256 __attribute__((__always_inline__, __nodebug__))
59 _mm256_sub_ps(__m256 a, __m256 b)
60 {
61 return a-b;
62 }
63
64 static __inline __m256d __attribute__((__always_inline__, __nodebug__))
65 _mm256_addsub_pd(__m256d a, __m256d b)
66 {
67 return (__m256d)__builtin_ia32_addsubpd256((__v4df)a, (__v4df)b);
68 }
69
70 static __inline __m256 __attribute__((__always_inline__, __nodebug__))
71 _mm256_addsub_ps(__m256 a, __m256 b)
72 {
73 return (__m256)__builtin_ia32_addsubps256((__v8sf)a, (__v8sf)b);
74 }
75
76 static __inline __m256d __attribute__((__always_inline__, __nodebug__))
77 _mm256_div_pd(__m256d a, __m256d b)
78 {
79 return a / b;
80 }
81
82 static __inline __m256 __attribute__((__always_inline__, __nodebug__))
83 _mm256_div_ps(__m256 a, __m256 b)
84 {
85 return a / b;
86 }
87
88 static __inline __m256d __attribute__((__always_inline__, __nodebug__))
89 _mm256_max_pd(__m256d a, __m256d b)
90 {
91 return (__m256d)__builtin_ia32_maxpd256((__v4df)a, (__v4df)b);
92 }
93
94 static __inline __m256 __attribute__((__always_inline__, __nodebug__))
95 _mm256_max_ps(__m256 a, __m256 b)
96 {
97 return (__m256)__builtin_ia32_maxps256((__v8sf)a, (__v8sf)b);
98 }
99
100 static __inline __m256d __attribute__((__always_inline__, __nodebug__))
101 _mm256_min_pd(__m256d a, __m256d b)
102 {
103 return (__m256d)__builtin_ia32_minpd256((__v4df)a, (__v4df)b);
104 }
105
106 static __inline __m256 __attribute__((__always_inline__, __nodebug__))
107 _mm256_min_ps(__m256 a, __m256 b)
108 {
109 return (__m256)__builtin_ia32_minps256((__v8sf)a, (__v8sf)b);
110 }
111
112 static __inline __m256d __attribute__((__always_inline__, __nodebug__))
113 _mm256_mul_pd(__m256d a, __m256d b)
114 {
115 return a * b;
116 }
117
118 static __inline __m256 __attribute__((__always_inline__, __nodebug__))
119 _mm256_mul_ps(__m256 a, __m256 b)
120 {
121 return a * b;
122 }
123
124 static __inline __m256d __attribute__((__always_inline__, __nodebug__))
125 _mm256_sqrt_pd(__m256d a)
126 {
127 return (__m256d)__builtin_ia32_sqrtpd256((__v4df)a);
128 }
129
130 static __inline __m256 __attribute__((__always_inline__, __nodebug__))
131 _mm256_sqrt_ps(__m256 a)
132 {
133 return (__m256)__builtin_ia32_sqrtps256((__v8sf)a);
134 }
135
136 static __inline __m256 __attribute__((__always_inline__, __nodebug__))
137 _mm256_rsqrt_ps(__m256 a)
138 {
139 return (__m256)__builtin_ia32_rsqrtps256((__v8sf)a);
140 }
141
142 static __inline __m256 __attribute__((__always_inline__, __nodebug__))
143 _mm256_rcp_ps(__m256 a)
144 {
145 return (__m256)__builtin_ia32_rcpps256((__v8sf)a);
146 }
147
148 static __inline __m256d __attribute__((__always_inline__, __nodebug__))
149 _mm256_round_pd(__m256d v, const int m)
150 {
151 return (__m256d)__builtin_ia32_roundpd256((__v4df)v, m);
152 }
153
154 static __inline __m256 __attribute__((__always_inline__, __nodebug__))
155 _mm256_round_ps(__m256 v, const int m)
156 {
157 return (__m256)__builtin_ia32_roundps256((__v8sf)v, m);
158 }
159
160 #define _mm256_ceil_pd(V) _mm256_round_pd((V), _MM_FROUND_CEIL)
161 #define _mm256_floor_pd(V) _mm256_round_pd((V), _MM_FROUND_FLOOR)
162 #define _mm256_ceil_ps(V) _mm256_round_ps((V), _MM_FROUND_CEIL)
163 #define _mm256_floor_ps(V) _mm256_round_ps((V), _MM_FROUND_FLOOR)
164
165 /* Logical */
166 static __inline __m256d __attribute__((__always_inline__, __nodebug__))
167 _mm256_and_pd(__m256d a, __m256d b)
168 {
169 return (__m256d)((__v4di)a & (__v4di)b);
170 }
171
172 static __inline __m256 __attribute__((__always_inline__, __nodebug__))
173 _mm256_and_ps(__m256 a, __m256 b)
174 {
175 return (__m256)((__v8si)a & (__v8si)b);
176 }
177
178 static __inline __m256d __attribute__((__always_inline__, __nodebug__))
179 _mm256_andnot_pd(__m256d a, __m256d b)
180 {
181 return (__m256d)(~(__v4di)a & (__v4di)b);
182 }
183
184 static __inline __m256 __attribute__((__always_inline__, __nodebug__))
185 _mm256_andnot_ps(__m256 a, __m256 b)
186 {
187 return (__m256)(~(__v8si)a & (__v8si)b);
188 }
189
190 static __inline __m256d __attribute__((__always_inline__, __nodebug__))
191 _mm256_or_pd(__m256d a, __m256d b)
192 {
193 return (__m256d)((__v4di)a | (__v4di)b);
194 }
195
196 static __inline __m256 __attribute__((__always_inline__, __nodebug__))
197 _mm256_or_ps(__m256 a, __m256 b)
198 {
199 return (__m256)((__v8si)a | (__v8si)b);
200 }
201
202 static __inline __m256d __attribute__((__always_inline__, __nodebug__))
203 _mm256_xor_pd(__m256d a, __m256d b)
204 {
205 return (__m256d)((__v4di)a ^ (__v4di)b);
206 }
207
208 static __inline __m256 __attribute__((__always_inline__, __nodebug__))
209 _mm256_xor_ps(__m256 a, __m256 b)
210 {
211 return (__m256)((__v8si)a ^ (__v8si)b);
212 }
213
214 /* Horizontal arithmetic */
215 static __inline __m256d __attribute__((__always_inline__, __nodebug__))
216 _mm256_hadd_pd(__m256d a, __m256d b)
217 {
218 return (__m256d)__builtin_ia32_haddpd256((__v4df)a, (__v4df)b);
219 }
220
221 static __inline __m256 __attribute__((__always_inline__, __nodebug__))
222 _mm256_hadd_ps(__m256 a, __m256 b)
223 {
224 return (__m256)__builtin_ia32_haddps256((__v8sf)a, (__v8sf)b);
225 }
226
227 static __inline __m256d __attribute__((__always_inline__, __nodebug__))
228 _mm256_hsub_pd(__m256d a, __m256d b)
229 {
230 return (__m256d)__builtin_ia32_hsubpd256((__v4df)a, (__v4df)b);
231 }
232
233 static __inline __m256 __attribute__((__always_inline__, __nodebug__))
234 _mm256_hsub_ps(__m256 a, __m256 b)
235 {
236 return (__m256)__builtin_ia32_hsubps256((__v8sf)a, (__v8sf)b);
237 }
238
239 /* Vector permutations */
240 static __inline __m128d __attribute__((__always_inline__, __nodebug__))
241 _mm_permutevar_pd(__m128d a, __m128i c)
242 {
243 return (__m128d)__builtin_ia32_vpermilvarpd((__v2df)a, (__v2di)c);
244 }
245
246 static __inline __m256d __attribute__((__always_inline__, __nodebug__))
247 _mm256_permutevar_pd(__m256d a, __m256i c)
248 {
249 return (__m256d)__builtin_ia32_vpermilvarpd256((__v4df)a, (__v4di)c);
250 }
251
252 static __inline __m128 __attribute__((__always_inline__, __nodebug__))
253 _mm_permutevar_ps(__m128 a, __m128i c)
254 {
255 return (__m128)__builtin_ia32_vpermilvarps((__v4sf)a, (__v4si)c);
256 }
257
258 static __inline __m256 __attribute__((__always_inline__, __nodebug__))
259 _mm256_permutevar_ps(__m256 a, __m256i c)
260 {
261 return (__m256)__builtin_ia32_vpermilvarps256((__v8sf)a,
262 (__v8si)c);
263 }
264
265 static __inline __m128d __attribute__((__always_inline__, __nodebug__))
266 _mm_permute_pd(__m128d a, const int c)
267 {
268 return (__m128d)__builtin_ia32_vpermilpd((__v2df)a, c);
269 }
270
271 static __inline __m256d __attribute__((__always_inline__, __nodebug__))
272 _mm256_permute_pd(__m256d a, const int c)
273 {
274 return (__m256d)__builtin_ia32_vpermilpd256((__v4df)a, c);
275 }
276
277 static __inline __m128 __attribute__((__always_inline__, __nodebug__))
278 _mm_permute_ps(__m128 a, const int c)
279 {
280 return (__m128)__builtin_ia32_vpermilps((__v4sf)a, c);
281 }
282
283 static __inline __m256 __attribute__((__always_inline__, __nodebug__))
284 _mm256_permute_ps(__m256 a, const int c)
285 {
286 return (__m256)__builtin_ia32_vpermilps256((__v8sf)a, c);
287 }
288
289 static __inline __m256d __attribute__((__always_inline__, __nodebug__))
290 _mm256_permute2f128_pd(__m256d a, __m256d b, const int c)
291 {
292 return (__m256d)__builtin_ia32_vperm2f128_pd256((__v4df)a, (__v4df)b, c);
293 }
294
295 static __inline __m256 __attribute__((__always_inline__, __nodebug__))
296 _mm256_permute2f128_ps(__m256 a, __m256 b, const int c)
297 {
298 return (__m256)__builtin_ia32_vperm2f128_ps256((__v8sf)a, (__v8sf)b, c);
299 }
300
301 static __inline __m256i __attribute__((__always_inline__, __nodebug__))
302 _mm256_permute2f128_si256(__m256i a, __m256i b, const int c)
303 {
304 return (__m256i)__builtin_ia32_vperm2f128_si256((__v8si)a, (__v8si)b, c);
305 }
306
307 /* Vector Blend */
308 static __inline __m256d __attribute__((__always_inline__, __nodebug__))
309 _mm256_blend_pd(__m256d a, __m256d b, const int c)
310 {
311 return (__m256d)__builtin_ia32_blendpd256((__v4df)a, (__v4df)b, c);
312 }
313
314 static __inline __m256 __attribute__((__always_inline__, __nodebug__))
315 _mm256_blend_ps(__m256 a, __m256 b, const int c)
316 {
317 return (__m256)__builtin_ia32_blendps256((__v8sf)a, (__v8sf)b, c);
318 }
319
320 static __inline __m256d __attribute__((__always_inline__, __nodebug__))
321 _mm256_blendv_pd(__m256d a, __m256d b, __m256d c)
322 {
323 return (__m256d)__builtin_ia32_blendvpd256((__v4df)a, (__v4df)b, (__v4df)c);
324 }
325
326 static __inline __m256 __attribute__((__always_inline__, __nodebug__))
327 _mm256_blendv_ps(__m256 a, __m256 b, __m256 c)
328 {
329 return (__m256)__builtin_ia32_blendvps256((__v8sf)a, (__v8sf)b, (__v8sf)c);
330 }
331
332 /* Vector Dot Product */
333 static __inline __m256 __attribute__((__always_inline__, __nodebug__))
334 _mm256_dp_ps(__m256 a, __m256 b, const int c)
335 {
336 return (__m256)__builtin_ia32_dpps256((__v8sf)a, (__v8sf)b, c);
337 }
338
339 /* Vector shuffle */
340 #define _mm256_shuffle_ps(a, b, mask) \
341 (__builtin_shufflevector((__v8sf)(a), (__v8sf)(b), \
342 (mask) & 0x3, ((mask) & 0xc) >> 2, \
343 (((mask) & 0x30) >> 4) + 8, (((mask) & 0xc0) >> 6) + 8, \
344 (mask) & 0x3 + 4, (((mask) & 0xc) >> 2) + 4, \
345 (((mask) & 0x30) >> 4) + 12, (((mask) & 0xc0) >> 6) + 12))
346
347 #define _mm256_shuffle_pd(a, b, mask) \
348 (__builtin_shufflevector((__v4df)(a), (__v4df)(b), \
349 (mask) & 0x1, \
350 (((mask) & 0x2) >> 1) + 4, \
351 (((mask) & 0x4) >> 2) + 2, \
352 (((mask) & 0x8) >> 3) + 6))
353
354 /* Compare */
355 #define _CMP_EQ_OQ 0x00 /* Equal (ordered, non-signaling) */
356 #define _CMP_LT_OS 0x01 /* Less-than (ordered, signaling) */
357 #define _CMP_LE_OS 0x02 /* Less-than-or-equal (ordered, signaling) */
358 #define _CMP_UNORD_Q 0x03 /* Unordered (non-signaling) */
359 #define _CMP_NEQ_UQ 0x04 /* Not-equal (unordered, non-signaling) */
360 #define _CMP_NLT_US 0x05 /* Not-less-than (unordered, signaling) */
361 #define _CMP_NLE_US 0x06 /* Not-less-than-or-equal (unordered, signaling) */
362 #define _CMP_ORD_Q 0x07 /* Ordered (nonsignaling) */
363 #define _CMP_EQ_UQ 0x08 /* Equal (unordered, non-signaling) */
364 #define _CMP_NGE_US 0x09 /* Not-greater-than-or-equal (unord, signaling) */
365 #define _CMP_NGT_US 0x0a /* Not-greater-than (unordered, signaling) */
366 #define _CMP_FALSE_OQ 0x0b /* False (ordered, non-signaling) */
367 #define _CMP_NEQ_OQ 0x0c /* Not-equal (ordered, non-signaling) */
368 #define _CMP_GE_OS 0x0d /* Greater-than-or-equal (ordered, signaling) */
369 #define _CMP_GT_OS 0x0e /* Greater-than (ordered, signaling) */
370 #define _CMP_TRUE_UQ 0x0f /* True (unordered, non-signaling) */
371 #define _CMP_EQ_OS 0x10 /* Equal (ordered, signaling) */
372 #define _CMP_LT_OQ 0x11 /* Less-than (ordered, non-signaling) */
373 #define _CMP_LE_OQ 0x12 /* Less-than-or-equal (ordered, non-signaling) */
374 #define _CMP_UNORD_S 0x13 /* Unordered (signaling) */
375 #define _CMP_NEQ_US 0x14 /* Not-equal (unordered, signaling) */
376 #define _CMP_NLT_UQ 0x15 /* Not-less-than (unordered, non-signaling) */
377 #define _CMP_NLE_UQ 0x16 /* Not-less-than-or-equal (unord, non-signaling) */
378 #define _CMP_ORD_S 0x17 /* Ordered (signaling) */
379 #define _CMP_EQ_US 0x18 /* Equal (unordered, signaling) */
380 #define _CMP_NGE_UQ 0x19 /* Not-greater-than-or-equal (unord, non-sign) */
381 #define _CMP_NGT_UQ 0x1a /* Not-greater-than (unordered, non-signaling) */
382 #define _CMP_FALSE_OS 0x1b /* False (ordered, signaling) */
383 #define _CMP_NEQ_OS 0x1c /* Not-equal (ordered, signaling) */
384 #define _CMP_GE_OQ 0x1d /* Greater-than-or-equal (ordered, non-signaling) */
385 #define _CMP_GT_OQ 0x1e /* Greater-than (ordered, non-signaling) */
386 #define _CMP_TRUE_US 0x1f /* True (unordered, signaling) */
387
388 static __inline __m128d __attribute__((__always_inline__, __nodebug__))
389 _mm_cmp_pd(__m128d a, __m128d b, const int c)
390 {
391 return (__m128d)__builtin_ia32_cmppd((__v2df)a, (__v2df)b, c);
392 }
393
394 static __inline __m128 __attribute__((__always_inline__, __nodebug__))
395 _mm_cmp_ps(__m128 a, __m128 b, const int c)
396 {
397 return (__m128)__builtin_ia32_cmpps((__v4sf)a, (__v4sf)b, c);
398 }
399
400 static __inline __m256d __attribute__((__always_inline__, __nodebug__))
401 _mm256_cmp_pd(__m256d a, __m256d b, const int c)
402 {
403 return (__m256d)__builtin_ia32_cmppd256((__v4df)a, (__v4df)b, c);
404 }
405
406 static __inline __m256 __attribute__((__always_inline__, __nodebug__))
407 _mm256_cmp_ps(__m256 a, __m256 b, const int c)
408 {
409 return (__m256)__builtin_ia32_cmpps256((__v8sf)a, (__v8sf)b, c);
410 }
411
412 static __inline __m128d __attribute__((__always_inline__, __nodebug__))
413 _mm_cmp_sd(__m128d a, __m128d b, const int c)
414 {
415 return (__m128d)__builtin_ia32_cmpsd((__v2df)a, (__v2df)b, c);
416 }
417
418 static __inline __m128 __attribute__((__always_inline__, __nodebug__))
419 _mm_cmp_ss(__m128 a, __m128 b, const int c)
420 {
421 return (__m128)__builtin_ia32_cmpss((__v4sf)a, (__v4sf)b, c);
422 }
423
424 /* Vector extract */
425 static __inline __m128d __attribute__((__always_inline__, __nodebug__))
426 _mm256_extractf128_pd(__m256d a, const int o)
427 {
428 return (__m128d)__builtin_ia32_vextractf128_pd256((__v4df)a, o);
429 }
430
431 static __inline __m128 __attribute__((__always_inline__, __nodebug__))
432 _mm256_extractf128_ps(__m256 a, const int o)
433 {
434 return (__m128)__builtin_ia32_vextractf128_ps256((__v8sf)a, o);
435 }
436
437 static __inline __m128i __attribute__((__always_inline__, __nodebug__))
438 _mm256_extractf128_si256(__m256i a, const int o)
439 {
440 return (__m128i)__builtin_ia32_vextractf128_si256((__v8si)a, o);
441 }
442
443 static __inline int __attribute__((__always_inline__, __nodebug__))
444 _mm256_extract_epi32(__m256i a, int const imm)
445 {
446 __v8si b = (__v8si)a;
447 return b[imm];
448 }
449
450 static __inline int __attribute__((__always_inline__, __nodebug__))
451 _mm256_extract_epi16(__m256i a, int const imm)
452 {
453 __v16hi b = (__v16hi)a;
454 return b[imm];
455 }
456
457 static __inline int __attribute__((__always_inline__, __nodebug__))
458 _mm256_extract_epi8(__m256i a, int const imm)
459 {
460 __v32qi b = (__v32qi)a;
461 return b[imm];
462 }
463
464 #ifdef __x86_64__
465 static __inline long long __attribute__((__always_inline__, __nodebug__))
466 _mm256_extract_epi64(__m256i a, const int imm)
467 {
468 __v4di b = (__v4di)a;
469 return b[imm];
470 }
471 #endif
472
473 /* Vector insert */
474 static __inline __m256d __attribute__((__always_inline__, __nodebug__))
475 _mm256_insertf128_pd(__m256d a, __m128d b, const int o)
476 {
477 return (__m256d)__builtin_ia32_vinsertf128_pd256((__v4df)a, (__v2df)b, o);
478 }
479
480 static __inline __m256 __attribute__((__always_inline__, __nodebug__))
481 _mm256_insertf128_ps(__m256 a, __m128 b, const int o)
482 {
483 return (__m256)__builtin_ia32_vinsertf128_ps256((__v8sf)a, (__v4sf)b, o);
484 }
485
486 static __inline __m256i __attribute__((__always_inline__, __nodebug__))
487 _mm256_insertf128_si256(__m256i a, __m128i b, const int o)
488 {
489 return (__m256i)__builtin_ia32_vinsertf128_si256((__v8si)a, (__v4si)b, o);
490 }
491
492 static __inline __m256i __attribute__((__always_inline__, __nodebug__))
493 _mm256_insert_epi32(__m256i a, int b, int const imm)
494 {
495 __v8si c = (__v8si)a;
496 c[imm & 7] = b;
497 return (__m256i)c;
498 }
499
500 static __inline __m256i __attribute__((__always_inline__, __nodebug__))
501 _mm256_insert_epi16(__m256i a, int b, int const imm)
502 {
503 __v16hi c = (__v16hi)a;
504 c[imm & 15] = b;
505 return (__m256i)c;
506 }
507
508 static __inline __m256i __attribute__((__always_inline__, __nodebug__))
509 _mm256_insert_epi8(__m256i a, int b, int const imm)
510 {
511 __v32qi c = (__v32qi)a;
512 c[imm & 31] = b;
513 return (__m256i)c;
514 }
515
516 #ifdef __x86_64__
517 static __inline __m256i __attribute__((__always_inline__, __nodebug__))
518 _mm256_insert_epi64(__m256i a, int b, int const imm)
519 {
520 __v4di c = (__v4di)a;
521 c[imm & 3] = b;
522 return (__m256i)c;
523 }
524 #endif
525
526 /* Conversion */
527 static __inline __m256d __attribute__((__always_inline__, __nodebug__))
528 _mm256_cvtepi32_pd(__m128i a)
529 {
530 return (__m256d)__builtin_ia32_cvtdq2pd256((__v4si) a);
531 }
532
533 static __inline __m256 __attribute__((__always_inline__, __nodebug__))
534 _mm256_cvtepi32_ps(__m256i a)
535 {
536 return (__m256)__builtin_ia32_cvtdq2ps256((__v8si) a);
537 }
538
539 static __inline __m128 __attribute__((__always_inline__, __nodebug__))
540 _mm256_cvtpd_ps(__m256d a)
541 {
542 return (__m128)__builtin_ia32_cvtpd2ps256((__v4df) a);
543 }
544
545 static __inline __m256i __attribute__((__always_inline__, __nodebug__))
546 _mm256_cvtps_epi32(__m256 a)
547 {
548 return (__m256i)__builtin_ia32_cvtps2dq256((__v8sf) a);
549 }
550
551 static __inline __m256d __attribute__((__always_inline__, __nodebug__))
552 _mm256_cvtps_pd(__m128 a)
553 {
554 return (__m256d)__builtin_ia32_cvtps2pd256((__v4sf) a);
555 }
556
557 static __inline __m128i __attribute__((__always_inline__, __nodebug__))
558 _mm256_cvttpd_epi32(__m256d a)
559 {
560 return (__m128i)__builtin_ia32_cvttpd2dq256((__v4df) a);
561 }
562
563 static __inline __m128i __attribute__((__always_inline__, __nodebug__))
564 _mm256_cvtpd_epi32(__m256d a)
565 {
566 return (__m128i)__builtin_ia32_cvtpd2dq256((__v4df) a);
567 }
568
569 static __inline __m256i __attribute__((__always_inline__, __nodebug__))
570 _mm256_cvttps_epi32(__m256 a)
571 {
572 return (__m256i)__builtin_ia32_cvttps2dq256((__v8sf) a);
573 }
574
575 /* Vector replicate */
576 static __inline __m256 __attribute__((__always_inline__, __nodebug__))
577 _mm256_movehdup_ps(__m256 a)
578 {
579 return __builtin_shufflevector(a, a, 1, 1, 3, 3, 5, 5, 7, 7);
580 }
581
582 static __inline __m256 __attribute__((__always_inline__, __nodebug__))
583 _mm256_moveldup_ps(__m256 a)
584 {
585 return __builtin_shufflevector(a, a, 0, 0, 2, 2, 4, 4, 6, 6);
586 }
587
588 static __inline __m256d __attribute__((__always_inline__, __nodebug__))
589 _mm256_movedup_pd(__m256d a)
590 {
591 return __builtin_shufflevector(a, a, 0, 0, 2, 2);
592 }
593
594 /* Unpack and Interleave */
595 static __inline __m256d __attribute__((__always_inline__, __nodebug__))
596 _mm256_unpackhi_pd(__m256d a, __m256d b)
597 {
598 return __builtin_shufflevector(a, b, 1, 5, 1+2, 5+2);
599 }
600
601 static __inline __m256d __attribute__((__always_inline__, __nodebug__))
602 _mm256_unpacklo_pd(__m256d a, __m256d b)
603 {
604 return __builtin_shufflevector(a, b, 0, 4, 0+2, 4+2);
605 }
606
607 static __inline __m256 __attribute__((__always_inline__, __nodebug__))
608 _mm256_unpackhi_ps(__m256 a, __m256 b)
609 {
610 return __builtin_shufflevector(a, b, 2, 10, 2+1, 10+1, 6, 14, 6+1, 14+1);
611 }
612
613 static __inline __m256 __attribute__((__always_inline__, __nodebug__))
614 _mm256_unpacklo_ps(__m256 a, __m256 b)
615 {
616 return __builtin_shufflevector(a, b, 0, 8, 0+1, 8+1, 4, 12, 4+1, 12+1);
617 }
618
619 /* Bit Test */
620 static __inline int __attribute__((__always_inline__, __nodebug__))
621 _mm_testz_pd(__m128d a, __m128d b)
622 {
623 return __builtin_ia32_vtestzpd((__v2df)a, (__v2df)b);
624 }
625
626 static __inline int __attribute__((__always_inline__, __nodebug__))
627 _mm_testc_pd(__m128d a, __m128d b)
628 {
629 return __builtin_ia32_vtestcpd((__v2df)a, (__v2df)b);
630 }
631
632 static __inline int __attribute__((__always_inline__, __nodebug__))
633 _mm_testnzc_pd(__m128d a, __m128d b)
634 {
635 return __builtin_ia32_vtestnzcpd((__v2df)a, (__v2df)b);
636 }
637
638 static __inline int __attribute__((__always_inline__, __nodebug__))
639 _mm_testz_ps(__m128 a, __m128 b)
640 {
641 return __builtin_ia32_vtestzps((__v4sf)a, (__v4sf)b);
642 }
643
644 static __inline int __attribute__((__always_inline__, __nodebug__))
645 _mm_testc_ps(__m128 a, __m128 b)
646 {
647 return __builtin_ia32_vtestcps((__v4sf)a, (__v4sf)b);
648 }
649
650 static __inline int __attribute__((__always_inline__, __nodebug__))
651 _mm_testnzc_ps(__m128 a, __m128 b)
652 {
653 return __builtin_ia32_vtestnzcps((__v4sf)a, (__v4sf)b);
654 }
655
656 static __inline int __attribute__((__always_inline__, __nodebug__))
657 _mm256_testz_pd(__m256d a, __m256d b)
658 {
659 return __builtin_ia32_vtestzpd256((__v4df)a, (__v4df)b);
660 }
661
662 static __inline int __attribute__((__always_inline__, __nodebug__))
663 _mm256_testc_pd(__m256d a, __m256d b)
664 {
665 return __builtin_ia32_vtestcpd256((__v4df)a, (__v4df)b);
666 }
667
668 static __inline int __attribute__((__always_inline__, __nodebug__))
669 _mm256_testnzc_pd(__m256d a, __m256d b)
670 {
671 return __builtin_ia32_vtestnzcpd256((__v4df)a, (__v4df)b);
672 }
673
674 static __inline int __attribute__((__always_inline__, __nodebug__))
675 _mm256_testz_ps(__m256 a, __m256 b)
676 {
677 return __builtin_ia32_vtestzps256((__v8sf)a, (__v8sf)b);
678 }
679
680 static __inline int __attribute__((__always_inline__, __nodebug__))
681 _mm256_testc_ps(__m256 a, __m256 b)
682 {
683 return __builtin_ia32_vtestcps256((__v8sf)a, (__v8sf)b);
684 }
685
686 static __inline int __attribute__((__always_inline__, __nodebug__))
687 _mm256_testnzc_ps(__m256 a, __m256 b)
688 {
689 return __builtin_ia32_vtestnzcps256((__v8sf)a, (__v8sf)b);
690 }
691
692 static __inline int __attribute__((__always_inline__, __nodebug__))
693 _mm256_testz_si256(__m256i a, __m256i b)
694 {
695 return __builtin_ia32_ptestz256((__v4di)a, (__v4di)b);
696 }
697
698 static __inline int __attribute__((__always_inline__, __nodebug__))
699 _mm256_testc_si256(__m256i a, __m256i b)
700 {
701 return __builtin_ia32_ptestc256((__v4di)a, (__v4di)b);
702 }
703
704 static __inline int __attribute__((__always_inline__, __nodebug__))
705 _mm256_testnzc_si256(__m256i a, __m256i b)
706 {
707 return __builtin_ia32_ptestnzc256((__v4di)a, (__v4di)b);
708 }
709
710 /* Vector extract sign mask */
711 static __inline int __attribute__((__always_inline__, __nodebug__))
712 _mm256_movemask_pd(__m256d a)
713 {
714 return __builtin_ia32_movmskpd256((__v4df)a);
715 }
716
717 static __inline int __attribute__((__always_inline__, __nodebug__))
718 _mm256_movemask_ps(__m256 a)
719 {
720 return __builtin_ia32_movmskps256((__v8sf)a);
721 }
722
723 /* Vector zero */
724 static __inline void __attribute__((__always_inline__, __nodebug__))
725 _mm256_zeroall(void)
726 {
727 __builtin_ia32_vzeroall();
728 }
729
730 static __inline void __attribute__((__always_inline__, __nodebug__))
731 _mm256_zeroupper(void)
732 {
733 __builtin_ia32_vzeroupper();
734 }
735
736 /* Vector load with broadcast */
737 static __inline __m128 __attribute__((__always_inline__, __nodebug__))
738 _mm_broadcast_ss(float const *a)
739 {
740 return (__m128)__builtin_ia32_vbroadcastss(a);
741 }
742
743 static __inline __m256d __attribute__((__always_inline__, __nodebug__))
744 _mm256_broadcast_sd(double const *a)
745 {
746 return (__m256d)__builtin_ia32_vbroadcastsd256(a);
747 }
748
749 static __inline __m256 __attribute__((__always_inline__, __nodebug__))
750 _mm256_broadcast_ss(float const *a)
751 {
752 return (__m256)__builtin_ia32_vbroadcastss256(a);
753 }
754
755 static __inline __m256d __attribute__((__always_inline__, __nodebug__))
756 _mm256_broadcast_pd(__m128d const *a)
757 {
758 return (__m256d)__builtin_ia32_vbroadcastf128_pd256(a);
759 }
760
761 static __inline __m256 __attribute__((__always_inline__, __nodebug__))
762 _mm256_broadcast_ps(__m128 const *a)
763 {
764 return (__m256)__builtin_ia32_vbroadcastf128_ps256(a);
765 }
766
767 /* SIMD load ops */
768 static __inline __m256d __attribute__((__always_inline__, __nodebug__))
769 _mm256_load_pd(double const *p)
770 {
771 return *(__m256d *)p;
772 }
773
774 static __inline __m256 __attribute__((__always_inline__, __nodebug__))
775 _mm256_load_ps(float const *p)
776 {
777 return *(__m256 *)p;
778 }
779
780 static __inline __m256d __attribute__((__always_inline__, __nodebug__))
781 _mm256_loadu_pd(double const *p)
782 {
783 return (__m256d)__builtin_ia32_loadupd256(p);
784 }
785
786 static __inline __m256 __attribute__((__always_inline__, __nodebug__))
787 _mm256_loadu_ps(float const *p)
788 {
789 return (__m256)__builtin_ia32_loadups256(p);
790 }
791
792 static __inline __m256i __attribute__((__always_inline__, __nodebug__))
793 _mm256_load_si256(__m256i const *p)
794 {
795 return *p;
796 }
797
798 static __inline __m256i __attribute__((__always_inline__, __nodebug__))
799 _mm256_loadu_si256(__m256i const *p)
800 {
801 return (__m256i)__builtin_ia32_loaddqu256((char const *)p);
802 }
803
804 static __inline __m256i __attribute__((__always_inline__, __nodebug__))
805 _mm256_lddqu_si256(__m256i const *p)
806 {
807 return (__m256i)__builtin_ia32_lddqu256((char const *)p);
808 }
809
810 /* SIMD store ops */
811 static __inline void __attribute__((__always_inline__, __nodebug__))
812 _mm256_store_pd(double *p, __m256d a)
813 {
814 *(__m256d *)p = a;
815 }
816
817 static __inline void __attribute__((__always_inline__, __nodebug__))
818 _mm256_store_ps(float *p, __m256 a)
819 {
820 *(__m256 *)p = a;
821 }
822
823 static __inline void __attribute__((__always_inline__, __nodebug__))
824 _mm256_storeu_pd(double *p, __m256d a)
825 {
826 __builtin_ia32_storeupd256(p, (__v4df)a);
827 }
828
829 static __inline void __attribute__((__always_inline__, __nodebug__))
830 _mm256_storeu_ps(float *p, __m256 a)
831 {
832 __builtin_ia32_storeups256(p, (__v8sf)a);
833 }
834
835 static __inline void __attribute__((__always_inline__, __nodebug__))
836 _mm256_store_si256(__m256i *p, __m256i a)
837 {
838 *p = a;
839 }
840
841 static __inline void __attribute__((__always_inline__, __nodebug__))
842 _mm256_storeu_si256(__m256i *p, __m256i a)
843 {
844 __builtin_ia32_storedqu256((char *)p, (__v32qi)a);
845 }
846
847 /* Conditional load ops */
848 static __inline __m128d __attribute__((__always_inline__, __nodebug__))
849 _mm_maskload_pd(double const *p, __m128d m)
850 {
851 return (__m128d)__builtin_ia32_maskloadpd((const __v2df *)p, (__v2df)m);
852 }
853
854 static __inline __m256d __attribute__((__always_inline__, __nodebug__))
855 _mm256_maskload_pd(double const *p, __m256d m)
856 {
857 return (__m256d)__builtin_ia32_maskloadpd256((const __v4df *)p, (__v4df)m);
858 }
859
860 static __inline __m128 __attribute__((__always_inline__, __nodebug__))
861 _mm_maskload_ps(float const *p, __m128 m)
862 {
863 return (__m128)__builtin_ia32_maskloadps((const __v4sf *)p, (__v4sf)m);
864 }
865
866 static __inline __m256 __attribute__((__always_inline__, __nodebug__))
867 _mm256_maskload_ps(float const *p, __m256 m)
868 {
869 return (__m256)__builtin_ia32_maskloadps256((const __v8sf *)p, (__v8sf)m);
870 }
871
872 /* Conditional store ops */
873 static __inline void __attribute__((__always_inline__, __nodebug__))
874 _mm256_maskstore_ps(float *p, __m256 m, __m256 a)
875 {
876 __builtin_ia32_maskstoreps256((__v8sf *)p, (__v8sf)m, (__v8sf)a);
877 }
878
879 static __inline void __attribute__((__always_inline__, __nodebug__))
880 _mm_maskstore_pd(double *p, __m128d m, __m128d a)
881 {
882 __builtin_ia32_maskstorepd((__v2df *)p, (__v2df)m, (__v2df)a);
883 }
884
885 static __inline void __attribute__((__always_inline__, __nodebug__))
886 _mm256_maskstore_pd(double *p, __m256d m, __m256d a)
887 {
888 __builtin_ia32_maskstorepd256((__v4df *)p, (__v4df)m, (__v4df)a);
889 }
890
891 static __inline void __attribute__((__always_inline__, __nodebug__))
892 _mm_maskstore_ps(float *p, __m128 m, __m128 a)
893 {
894 __builtin_ia32_maskstoreps((__v4sf *)p, (__v4sf)m, (__v4sf)a);
895 }
896
897 /* Cacheability support ops */
898 static __inline void __attribute__((__always_inline__, __nodebug__))
899 _mm256_stream_si256(__m256i *a, __m256i b)
900 {
901 __builtin_ia32_movntdq256((__v4di *)a, (__v4di)b);
902 }
903
904 static __inline void __attribute__((__always_inline__, __nodebug__))
905 _mm256_stream_pd(double *a, __m256d b)
906 {
907 __builtin_ia32_movntpd256(a, (__v4df)b);
908 }
909
910 static __inline void __attribute__((__always_inline__, __nodebug__))
911 _mm256_stream_ps(float *p, __m256 a)
912 {
913 __builtin_ia32_movntps256(p, (__v8sf)a);
914 }
915
916 /* Create vectors */
917 static __inline __m256d __attribute__((__always_inline__, __nodebug__))
918 _mm256_set_pd(double a, double b, double c, double d)
919 {
920 return (__m256d){ d, c, b, a };
921 }
922
923 static __inline __m256 __attribute__((__always_inline__, __nodebug__))
924 _mm256_set_ps(float a, float b, float c, float d,
925 float e, float f, float g, float h)
926 {
927 return (__m256){ h, g, f, e, d, c, b, a };
928 }
929
930 static __inline __m256i __attribute__((__always_inline__, __nodebug__))
931 _mm256_set_epi32(int i0, int i1, int i2, int i3,
932 int i4, int i5, int i6, int i7)
933 {
934 return (__m256i)(__v8si){ i7, i6, i5, i4, i3, i2, i1, i0 };
935 }
936
937 static __inline __m256i __attribute__((__always_inline__, __nodebug__))
938 _mm256_set_epi16(short w15, short w14, short w13, short w12,
939 short w11, short w10, short w09, short w08,
940 short w07, short w06, short w05, short w04,
941 short w03, short w02, short w01, short w00)
942 {
943 return (__m256i)(__v16hi){ w00, w01, w02, w03, w04, w05, w06, w07,
944 w08, w09, w10, w11, w12, w13, w14, w15 };
945 }
946
947 static __inline __m256i __attribute__((__always_inline__, __nodebug__))
948 _mm256_set_epi8(char b31, char b30, char b29, char b28,
949 char b27, char b26, char b25, char b24,
950 char b23, char b22, char b21, char b20,
951 char b19, char b18, char b17, char b16,
952 char b15, char b14, char b13, char b12,
953 char b11, char b10, char b09, char b08,
954 char b07, char b06, char b05, char b04,
955 char b03, char b02, char b01, char b00)
956 {
957 return (__m256i)(__v32qi){
958 b00, b01, b02, b03, b04, b05, b06, b07,
959 b08, b09, b10, b11, b12, b13, b14, b15,
960 b16, b17, b18, b19, b20, b21, b22, b23,
961 b24, b25, b26, b27, b28, b29, b30, b31
962 };
963 }
964
965 static __inline __m256i __attribute__((__always_inline__, __nodebug__))
966 _mm256_set_epi64x(long long a, long long b, long long c, long long d)
967 {
968 return (__m256i)(__v4di){ d, c, b, a };
969 }
970
971 /* Create vectors with elements in reverse order */
972 static __inline __m256d __attribute__((__always_inline__, __nodebug__))
973 _mm256_setr_pd(double a, double b, double c, double d)
974 {
975 return (__m256d){ a, b, c, d };
976 }
977
978 static __inline __m256 __attribute__((__always_inline__, __nodebug__))
979 _mm256_setr_ps(float a, float b, float c, float d,
980 float e, float f, float g, float h)
981 {
982 return (__m256){ a, b, c, d, e, f, g, h };
983 }
984
985 static __inline __m256i __attribute__((__always_inline__, __nodebug__))
986 _mm256_setr_epi32(int i0, int i1, int i2, int i3,
987 int i4, int i5, int i6, int i7)
988 {
989 return (__m256i)(__v8si){ i0, i1, i2, i3, i4, i5, i6, i7 };
990 }
991
992 static __inline __m256i __attribute__((__always_inline__, __nodebug__))
993 _mm256_setr_epi16(short w15, short w14, short w13, short w12,
994 short w11, short w10, short w09, short w08,
995 short w07, short w06, short w05, short w04,
996 short w03, short w02, short w01, short w00)
997 {
998 return (__m256i)(__v16hi){ w15, w14, w13, w12, w11, w10, w09, w08,
999 w07, w06, w05, w04, w03, w02, w01, w00 };
1000 }
1001
1002 static __inline __m256i __attribute__((__always_inline__, __nodebug__))
1003 _mm256_setr_epi8(char b31, char b30, char b29, char b28,
1004 char b27, char b26, char b25, char b24,
1005 char b23, char b22, char b21, char b20,
1006 char b19, char b18, char b17, char b16,
1007 char b15, char b14, char b13, char b12,
1008 char b11, char b10, char b09, char b08,
1009 char b07, char b06, char b05, char b04,
1010 char b03, char b02, char b01, char b00)
1011 {
1012 return (__m256i)(__v32qi){
1013 b31, b30, b29, b28, b27, b26, b25, b24,
1014 b23, b22, b21, b20, b19, b18, b17, b16,
1015 b15, b14, b13, b12, b11, b10, b09, b08,
1016 b07, b06, b05, b04, b03, b02, b01, b00 };
1017 }
1018
1019 static __inline __m256i __attribute__((__always_inline__, __nodebug__))
1020 _mm256_setr_epi64x(long long a, long long b, long long c, long long d)
1021 {
1022 return (__m256i)(__v4di){ a, b, c, d };
1023 }
1024
1025 /* Create vectors with repeated elements */
1026 static __inline __m256d __attribute__((__always_inline__, __nodebug__))
1027 _mm256_set1_pd(double w)
1028 {
1029 return (__m256d){ w, w, w, w };
1030 }
1031
1032 static __inline __m256 __attribute__((__always_inline__, __nodebug__))
1033 _mm256_set1_ps(float w)
1034 {
1035 return (__m256){ w, w, w, w, w, w, w, w };
1036 }
1037
1038 static __inline __m256i __attribute__((__always_inline__, __nodebug__))
1039 _mm256_set1_epi32(int i)
1040 {
1041 return (__m256i)(__v8si){ i, i, i, i, i, i, i, i };
1042 }
1043
1044 static __inline __m256i __attribute__((__always_inline__, __nodebug__))
1045 _mm256_set1_epi16(short w)
1046 {
1047 return (__m256i)(__v16hi){ w, w, w, w, w, w, w, w, w, w, w, w, w, w, w, w };
1048 }
1049
1050 static __inline __m256i __attribute__((__always_inline__, __nodebug__))
1051 _mm256_set1_epi8(char b)
1052 {
1053 return (__m256i)(__v32qi){ b, b, b, b, b, b, b, b, b, b, b, b, b, b, b, b,
1054 b, b, b, b, b, b, b, b, b, b, b, b, b, b, b, b };
1055 }
1056
1057 static __inline __m256i __attribute__((__always_inline__, __nodebug__))
1058 _mm256_set1_epi64x(long long q)
1059 {
1060 return (__m256i)(__v4di){ q, q, q, q };
1061 }
1062
1063 /* Create zeroed vectors */
1064 static __inline __m256d __attribute__((__always_inline__, __nodebug__))
1065 _mm256_setzero_pd(void)
1066 {
1067 return (__m256d){ 0, 0, 0, 0 };
1068 }
1069
1070 static __inline __m256 __attribute__((__always_inline__, __nodebug__))
1071 _mm256_setzero_ps(void)
1072 {
1073 return (__m256){ 0, 0, 0, 0, 0, 0, 0, 0 };
1074 }
1075
1076 static __inline __m256i __attribute__((__always_inline__, __nodebug__))
1077 _mm256_setzero_si256(void)
1078 {
1079 return (__m256i){ 0LL, 0LL, 0LL, 0LL };
1080 }
1081
1082 /* Cast between vector types */
1083 static __inline __m256 __attribute__((__always_inline__, __nodebug__))
1084 _mm256_castpd_ps(__m256d in)
1085 {
1086 return (__m256)in;
1087 }
1088
1089 static __inline __m256i __attribute__((__always_inline__, __nodebug__))
1090 _mm256_castpd_si256(__m256d in)
1091 {
1092 return (__m256i)in;
1093 }
1094
1095 static __inline __m256d __attribute__((__always_inline__, __nodebug__))
1096 _mm256_castps_pd(__m256 in)
1097 {
1098 return (__m256d)in;
1099 }
1100
1101 static __inline __m256i __attribute__((__always_inline__, __nodebug__))
1102 _mm256_castps_si256(__m256 in)
1103 {
1104 return (__m256i)in;
1105 }
1106
1107 static __inline __m256 __attribute__((__always_inline__, __nodebug__))
1108 _mm256_castsi256_ps(__m256i in)
1109 {
1110 return (__m256)in;
1111 }
1112
1113 static __inline __m256d __attribute__((__always_inline__, __nodebug__))
1114 _mm256_castsi256_pd(__m256i in)
1115 {
1116 return (__m256d)in;
1117 }
1118
1119 static __inline __m128d __attribute__((__always_inline__, __nodebug__))
1120 _mm256_castpd256_pd128(__m256d in)
1121 {
1122 return __builtin_shufflevector(in, in, 0, 1);
1123 }
1124
1125 static __inline __m128 __attribute__((__always_inline__, __nodebug__))
1126 _mm256_castps256_ps128(__m256 in)
1127 {
1128 return __builtin_shufflevector(in, in, 0, 1, 2, 3);
1129 }
1130
1131 static __inline __m128i __attribute__((__always_inline__, __nodebug__))
1132 _mm256_castsi256_si128(__m256i in)
1133 {
1134 return __builtin_shufflevector(in, in, 0, 1);
1135 }
1136
1137 static __inline __m256d __attribute__((__always_inline__, __nodebug__))
1138 _mm256_castpd128_pd256(__m128d in)
1139 {
1140 __m128d zero = _mm_setzero_pd();
1141 return __builtin_shufflevector(in, zero, 0, 1, 2, 2);
1142 }
1143
1144 static __inline __m256 __attribute__((__always_inline__, __nodebug__))
1145 _mm256_castps128_ps256(__m128 in)
1146 {
1147 __m128 zero = _mm_setzero_ps();
1148 return __builtin_shufflevector(in, zero, 0, 1, 2, 3, 4, 4, 4, 4);
1149 }
1150
1151 static __inline __m256i __attribute__((__always_inline__, __nodebug__))
1152 _mm256_castsi128_si256(__m128i in)
1153 {
1154 __m128i zero = _mm_setzero_si128();
1155 return __builtin_shufflevector(in, zero, 0, 1, 2, 2);
1156 }
GIT mirror of clang project