1 /* Copyright (C) 2002 Free Software Foundation, Inc.
3 This file is part of GNU CC.
5 GNU CC is free software; you can redistribute it and/or modify
6 it under the terms of the GNU General Public License as published by
7 the Free Software Foundation; either version 2, or (at your option)
10 GNU CC is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 GNU General Public License for more details.
15 You should have received a copy of the GNU General Public License
16 along with GNU CC; see the file COPYING. If not, write to
17 the Free Software Foundation, 59 Temple Place - Suite 330,
18 Boston, MA 02111-1307, USA. */
20 /* As a special exception, if you include this header file into source
21 files compiled by GCC, this header file does not by itself cause
22 the resulting executable to be covered by the GNU General Public
23 License. This exception does not however invalidate any other
24 reasons why the executable file might be covered by the GNU General
27 /* Implemented from the specification included in the Intel C++ Compiler
28 User Guide and Reference, version 5.0. */
30 #ifndef _XMMINTRIN_H_INCLUDED
31 #define _XMMINTRIN_H_INCLUDED
33 /* We need type definitions from the MMX header file. */
36 /* The data type indended for user use. */
37 typedef int __m128
__attribute__ ((__mode__(__V4SF__
)));
39 /* Internal data types for implementing the instrinsics. */
40 typedef int __v4sf
__attribute__ ((__mode__(__V4SF__
)));
41 typedef int __v4si
__attribute__ ((__mode__(__V4SI__
)));
43 /* Create a selector for use with the SHUFPS instruction. */
44 #define _MM_SHUFFLE(fp3,fp2,fp1,fp0) \
45 (((fp3) << 6) | ((fp2) << 4) | ((fp1) << 2) | (fp0))
47 /* Constants for use with _mm_prefetch. */
56 /* Bits in the MXCSR. */
57 #define _MM_EXCEPT_MASK 0x003f
58 #define _MM_EXCEPT_INVALID 0x0001
59 #define _MM_EXCEPT_DENORM 0x0002
60 #define _MM_EXCEPT_DIV_ZERO 0x0004
61 #define _MM_EXCEPT_OVERFLOW 0x0008
62 #define _MM_EXCEPT_UNDERFLOW 0x0010
63 #define _MM_EXCEPT_INEXACT 0x0020
65 #define _MM_MASK_MASK 0x1f80
66 #define _MM_MASK_INVALID 0x0080
67 #define _MM_MASK_DENORM 0x0100
68 #define _MM_MASK_DIV_ZERO 0x0200
69 #define _MM_MASK_OVERFLOW 0x0400
70 #define _MM_MASK_UNDERFLOW 0x0800
71 #define _MM_MASK_INEXACT 0x1000
73 #define _MM_ROUND_MASK 0x6000
74 #define _MM_ROUND_NEAREST 0x0000
75 #define _MM_ROUND_DOWN 0x2000
76 #define _MM_ROUND_UP 0x4000
77 #define _MM_ROUND_TOWARD_ZERO 0x6000
79 #define _MM_FLUSH_ZERO_MASK 0x8000
80 #define _MM_FLUSH_ZERO_ON 0x8000
81 #define _MM_FLUSH_ZERO_OFF 0x0000
83 /* Perform the respective operation on the lower SPFP (single-precision
84 floating-point) values of A and B; the upper three SPFP values are
85 passed through from A. */
87 static __inline __m128
88 _mm_add_ss (__m128 __A
, __m128 __B
)
90 return (__m128
) __builtin_ia32_addss ((__v4sf
)__A
, (__v4sf
)__B
);
93 static __inline __m128
94 _mm_sub_ss (__m128 __A
, __m128 __B
)
96 return (__m128
) __builtin_ia32_subss ((__v4sf
)__A
, (__v4sf
)__B
);
99 static __inline __m128
100 _mm_mul_ss (__m128 __A
, __m128 __B
)
102 return (__m128
) __builtin_ia32_mulss ((__v4sf
)__A
, (__v4sf
)__B
);
105 static __inline __m128
106 _mm_div_ss (__m128 __A
, __m128 __B
)
108 return (__m128
) __builtin_ia32_divss ((__v4sf
)__A
, (__v4sf
)__B
);
111 static __inline __m128
112 _mm_sqrt_ss (__m128 __A
)
114 return (__m128
) __builtin_ia32_sqrtss ((__v4sf
)__A
);
117 static __inline __m128
118 _mm_rcp_ss (__m128 __A
)
120 return (__m128
) __builtin_ia32_rcpss ((__v4sf
)__A
);
123 static __inline __m128
124 _mm_rsqrt_ss (__m128 __A
)
126 return (__m128
) __builtin_ia32_rsqrtss ((__v4sf
)__A
);
129 static __inline __m128
130 _mm_min_ss (__m128 __A
, __m128 __B
)
132 return (__m128
) __builtin_ia32_minss ((__v4sf
)__A
, (__v4sf
)__B
);
135 static __inline __m128
136 _mm_max_ss (__m128 __A
, __m128 __B
)
138 return (__m128
) __builtin_ia32_maxss ((__v4sf
)__A
, (__v4sf
)__B
);
141 /* Perform the respective operation on the four SPFP values in A and B. */
143 static __inline __m128
144 _mm_add_ps (__m128 __A
, __m128 __B
)
146 return (__m128
) __builtin_ia32_addps ((__v4sf
)__A
, (__v4sf
)__B
);
149 static __inline __m128
150 _mm_sub_ps (__m128 __A
, __m128 __B
)
152 return (__m128
) __builtin_ia32_subps ((__v4sf
)__A
, (__v4sf
)__B
);
155 static __inline __m128
156 _mm_mul_ps (__m128 __A
, __m128 __B
)
158 return (__m128
) __builtin_ia32_mulps ((__v4sf
)__A
, (__v4sf
)__B
);
161 static __inline __m128
162 _mm_div_ps (__m128 __A
, __m128 __B
)
164 return (__m128
) __builtin_ia32_divps ((__v4sf
)__A
, (__v4sf
)__B
);
167 static __inline __m128
168 _mm_sqrt_ps (__m128 __A
)
170 return (__m128
) __builtin_ia32_sqrtps ((__v4sf
)__A
);
173 static __inline __m128
174 _mm_rcp_ps (__m128 __A
)
176 return (__m128
) __builtin_ia32_rcpps ((__v4sf
)__A
);
179 static __inline __m128
180 _mm_rsqrt_ps (__m128 __A
)
182 return (__m128
) __builtin_ia32_rsqrtps ((__v4sf
)__A
);
185 static __inline __m128
186 _mm_min_ps (__m128 __A
, __m128 __B
)
188 return (__m128
) __builtin_ia32_minps ((__v4sf
)__A
, (__v4sf
)__B
);
191 static __inline __m128
192 _mm_max_ps (__m128 __A
, __m128 __B
)
194 return (__m128
) __builtin_ia32_maxps ((__v4sf
)__A
, (__v4sf
)__B
);
197 /* Perform logical bit-wise operations on 128-bit values. */
199 static __inline __m128
200 _mm_and_ps (__m128 __A
, __m128 __B
)
202 return __builtin_ia32_andps (__A
, __B
);
205 static __inline __m128
206 _mm_andnot_ps (__m128 __A
, __m128 __B
)
208 return __builtin_ia32_andnps (__A
, __B
);
211 static __inline __m128
212 _mm_or_ps (__m128 __A
, __m128 __B
)
214 return __builtin_ia32_orps (__A
, __B
);
217 static __inline __m128
218 _mm_xor_ps (__m128 __A
, __m128 __B
)
220 return __builtin_ia32_xorps (__A
, __B
);
223 /* Perform a comparison on the lower SPFP values of A and B. If the
224 comparison is true, place a mask of all ones in the result, otherwise a
225 mask of zeros. The upper three SPFP values are passed through from A. */
227 static __inline __m128
228 _mm_cmpeq_ss (__m128 __A
, __m128 __B
)
230 return (__m128
) __builtin_ia32_cmpeqss ((__v4sf
)__A
, (__v4sf
)__B
);
233 static __inline __m128
234 _mm_cmplt_ss (__m128 __A
, __m128 __B
)
236 return (__m128
) __builtin_ia32_cmpltss ((__v4sf
)__A
, (__v4sf
)__B
);
239 static __inline __m128
240 _mm_cmple_ss (__m128 __A
, __m128 __B
)
242 return (__m128
) __builtin_ia32_cmpless ((__v4sf
)__A
, (__v4sf
)__B
);
245 static __inline __m128
246 _mm_cmpgt_ss (__m128 __A
, __m128 __B
)
248 return (__m128
) __builtin_ia32_movss ((__v4sf
) __A
,
250 __builtin_ia32_cmpltss ((__v4sf
) __B
,
255 static __inline __m128
256 _mm_cmpge_ss (__m128 __A
, __m128 __B
)
258 return (__m128
) __builtin_ia32_movss ((__v4sf
) __A
,
260 __builtin_ia32_cmpless ((__v4sf
) __B
,
265 static __inline __m128
266 _mm_cmpneq_ss (__m128 __A
, __m128 __B
)
268 return (__m128
) __builtin_ia32_cmpneqss ((__v4sf
)__A
, (__v4sf
)__B
);
271 static __inline __m128
272 _mm_cmpnlt_ss (__m128 __A
, __m128 __B
)
274 return (__m128
) __builtin_ia32_cmpnltss ((__v4sf
)__A
, (__v4sf
)__B
);
277 static __inline __m128
278 _mm_cmpnle_ss (__m128 __A
, __m128 __B
)
280 return (__m128
) __builtin_ia32_cmpnless ((__v4sf
)__A
, (__v4sf
)__B
);
283 static __inline __m128
284 _mm_cmpngt_ss (__m128 __A
, __m128 __B
)
286 return (__m128
) __builtin_ia32_movss ((__v4sf
) __A
,
288 __builtin_ia32_cmpnltss ((__v4sf
) __B
,
293 static __inline __m128
294 _mm_cmpnge_ss (__m128 __A
, __m128 __B
)
296 return (__m128
) __builtin_ia32_movss ((__v4sf
) __A
,
298 __builtin_ia32_cmpnless ((__v4sf
) __B
,
303 static __inline __m128
304 _mm_cmpord_ss (__m128 __A
, __m128 __B
)
306 return (__m128
) __builtin_ia32_cmpordss ((__v4sf
)__A
, (__v4sf
)__B
);
309 static __inline __m128
310 _mm_cmpunord_ss (__m128 __A
, __m128 __B
)
312 return (__m128
) __builtin_ia32_cmpunordss ((__v4sf
)__A
, (__v4sf
)__B
);
315 /* Perform a comparison on the four SPFP values of A and B. For each
316 element, if the comparison is true, place a mask of all ones in the
317 result, otherwise a mask of zeros. */
319 static __inline __m128
320 _mm_cmpeq_ps (__m128 __A
, __m128 __B
)
322 return (__m128
) __builtin_ia32_cmpeqps ((__v4sf
)__A
, (__v4sf
)__B
);
325 static __inline __m128
326 _mm_cmplt_ps (__m128 __A
, __m128 __B
)
328 return (__m128
) __builtin_ia32_cmpltps ((__v4sf
)__A
, (__v4sf
)__B
);
331 static __inline __m128
332 _mm_cmple_ps (__m128 __A
, __m128 __B
)
334 return (__m128
) __builtin_ia32_cmpleps ((__v4sf
)__A
, (__v4sf
)__B
);
337 static __inline __m128
338 _mm_cmpgt_ps (__m128 __A
, __m128 __B
)
340 return (__m128
) __builtin_ia32_cmpgtps ((__v4sf
)__A
, (__v4sf
)__B
);
343 static __inline __m128
344 _mm_cmpge_ps (__m128 __A
, __m128 __B
)
346 return (__m128
) __builtin_ia32_cmpgeps ((__v4sf
)__A
, (__v4sf
)__B
);
349 static __inline __m128
350 _mm_cmpneq_ps (__m128 __A
, __m128 __B
)
352 return (__m128
) __builtin_ia32_cmpneqps ((__v4sf
)__A
, (__v4sf
)__B
);
355 static __inline __m128
356 _mm_cmpnlt_ps (__m128 __A
, __m128 __B
)
358 return (__m128
) __builtin_ia32_cmpnltps ((__v4sf
)__A
, (__v4sf
)__B
);
361 static __inline __m128
362 _mm_cmpnle_ps (__m128 __A
, __m128 __B
)
364 return (__m128
) __builtin_ia32_cmpnleps ((__v4sf
)__A
, (__v4sf
)__B
);
367 static __inline __m128
368 _mm_cmpngt_ps (__m128 __A
, __m128 __B
)
370 return (__m128
) __builtin_ia32_cmpngtps ((__v4sf
)__A
, (__v4sf
)__B
);
373 static __inline __m128
374 _mm_cmpnge_ps (__m128 __A
, __m128 __B
)
376 return (__m128
) __builtin_ia32_cmpngeps ((__v4sf
)__A
, (__v4sf
)__B
);
379 static __inline __m128
380 _mm_cmpord_ps (__m128 __A
, __m128 __B
)
382 return (__m128
) __builtin_ia32_cmpordps ((__v4sf
)__A
, (__v4sf
)__B
);
385 static __inline __m128
386 _mm_cmpunord_ps (__m128 __A
, __m128 __B
)
388 return (__m128
) __builtin_ia32_cmpunordps ((__v4sf
)__A
, (__v4sf
)__B
);
391 /* Compare the lower SPFP values of A and B and return 1 if true
395 _mm_comieq_ss (__m128 __A
, __m128 __B
)
397 return __builtin_ia32_comieq ((__v4sf
)__A
, (__v4sf
)__B
);
401 _mm_comilt_ss (__m128 __A
, __m128 __B
)
403 return __builtin_ia32_comilt ((__v4sf
)__A
, (__v4sf
)__B
);
407 _mm_comile_ss (__m128 __A
, __m128 __B
)
409 return __builtin_ia32_comile ((__v4sf
)__A
, (__v4sf
)__B
);
413 _mm_comigt_ss (__m128 __A
, __m128 __B
)
415 return __builtin_ia32_comigt ((__v4sf
)__A
, (__v4sf
)__B
);
419 _mm_comige_ss (__m128 __A
, __m128 __B
)
421 return __builtin_ia32_comige ((__v4sf
)__A
, (__v4sf
)__B
);
425 _mm_comineq_ss (__m128 __A
, __m128 __B
)
427 return __builtin_ia32_comineq ((__v4sf
)__A
, (__v4sf
)__B
);
431 _mm_ucomieq_ss (__m128 __A
, __m128 __B
)
433 return __builtin_ia32_ucomieq ((__v4sf
)__A
, (__v4sf
)__B
);
437 _mm_ucomilt_ss (__m128 __A
, __m128 __B
)
439 return __builtin_ia32_ucomilt ((__v4sf
)__A
, (__v4sf
)__B
);
443 _mm_ucomile_ss (__m128 __A
, __m128 __B
)
445 return __builtin_ia32_ucomile ((__v4sf
)__A
, (__v4sf
)__B
);
449 _mm_ucomigt_ss (__m128 __A
, __m128 __B
)
451 return __builtin_ia32_ucomigt ((__v4sf
)__A
, (__v4sf
)__B
);
455 _mm_ucomige_ss (__m128 __A
, __m128 __B
)
457 return __builtin_ia32_ucomige ((__v4sf
)__A
, (__v4sf
)__B
);
461 _mm_ucomineq_ss (__m128 __A
, __m128 __B
)
463 return __builtin_ia32_ucomineq ((__v4sf
)__A
, (__v4sf
)__B
);
466 /* Convert the lower SPFP value to a 32-bit integer according to the current
469 _mm_cvtss_si32 (__m128 __A
)
471 return __builtin_ia32_cvtss2si ((__v4sf
) __A
);
474 /* Convert the two lower SPFP values to 32-bit integers according to the
475 current rounding mode. Return the integers in packed form. */
476 static __inline __m64
477 _mm_cvtps_pi32 (__m128 __A
)
479 return (__m64
) __builtin_ia32_cvtps2pi ((__v4sf
) __A
);
482 /* Truncate the lower SPFP value to a 32-bit integer. */
484 _mm_cvttss_si32 (__m128 __A
)
486 return __builtin_ia32_cvttss2si ((__v4sf
) __A
);
489 /* Truncate the two lower SPFP values to 32-bit integers. Return the
490 integers in packed form. */
491 static __inline __m64
492 _mm_cvttps_pi32 (__m128 __A
)
494 return (__m64
) __builtin_ia32_cvttps2pi ((__v4sf
) __A
);
497 /* Convert B to a SPFP value and insert it as element zero in A. */
498 static __inline __m128
499 _mm_cvtsi32_ss (__m128 __A
, int __B
)
501 return (__m128
) __builtin_ia32_cvtsi2ss ((__v4sf
) __A
, __B
);
504 /* Convert the two 32-bit values in B to SPFP form and insert them
505 as the two lower elements in A. */
506 static __inline __m128
507 _mm_cvtpi32_ps (__m128 __A
, __m64 __B
)
509 return (__m128
) __builtin_ia32_cvtpi2ps ((__v4sf
) __A
, (__v2si
)__B
);
512 /* Convert the four signed 16-bit values in A to SPFP form. */
513 static __inline __m128
514 _mm_cvtpi16_ps (__m64 __A
)
517 __v2si __hisi
, __losi
;
520 /* This comparison against zero gives us a mask that can be used to
521 fill in the missing sign bits in the unpack operations below, so
522 that we get signed values after unpacking. */
523 __sign
= (__v4hi
) __builtin_ia32_mmx_zero ();
524 __sign
= __builtin_ia32_pcmpgtw (__sign
, (__v4hi
)__A
);
526 /* Convert the four words to doublewords. */
527 __hisi
= (__v2si
) __builtin_ia32_punpckhwd ((__v4hi
)__A
, __sign
);
528 __losi
= (__v2si
) __builtin_ia32_punpcklwd ((__v4hi
)__A
, __sign
);
530 /* Convert the doublewords to floating point two at a time. */
531 __r
= (__v4sf
) __builtin_ia32_setzerops ();
532 __r
= __builtin_ia32_cvtpi2ps (__r
, __hisi
);
533 __r
= __builtin_ia32_movlhps (__r
, __r
);
534 __r
= __builtin_ia32_cvtpi2ps (__r
, __losi
);
539 /* Convert the four unsigned 16-bit values in A to SPFP form. */
540 static __inline __m128
541 _mm_cvtpu16_ps (__m64 __A
)
543 __v4hi __zero
= (__v4hi
) __builtin_ia32_mmx_zero ();
544 __v2si __hisi
, __losi
;
547 /* Convert the four words to doublewords. */
548 __hisi
= (__v2si
) __builtin_ia32_punpckhwd ((__v4hi
)__A
, __zero
);
549 __losi
= (__v2si
) __builtin_ia32_punpcklwd ((__v4hi
)__A
, __zero
);
551 /* Convert the doublewords to floating point two at a time. */
552 __r
= (__v4sf
) __builtin_ia32_setzerops ();
553 __r
= __builtin_ia32_cvtpi2ps (__r
, __hisi
);
554 __r
= __builtin_ia32_movlhps (__r
, __r
);
555 __r
= __builtin_ia32_cvtpi2ps (__r
, __losi
);
560 /* Convert the low four signed 8-bit values in A to SPFP form. */
561 static __inline __m128
562 _mm_cvtpi8_ps (__m64 __A
)
566 /* This comparison against zero gives us a mask that can be used to
567 fill in the missing sign bits in the unpack operations below, so
568 that we get signed values after unpacking. */
569 __sign
= (__v8qi
) __builtin_ia32_mmx_zero ();
570 __sign
= __builtin_ia32_pcmpgtb (__sign
, (__v8qi
)__A
);
572 /* Convert the four low bytes to words. */
573 __A
= (__m64
) __builtin_ia32_punpcklbw ((__v8qi
)__A
, __sign
);
575 return _mm_cvtpi16_ps(__A
);
578 /* Convert the low four unsigned 8-bit values in A to SPFP form. */
579 static __inline __m128
580 _mm_cvtpu8_ps(__m64 __A
)
582 __v8qi __zero
= (__v8qi
) __builtin_ia32_mmx_zero ();
583 __A
= (__m64
) __builtin_ia32_punpcklbw ((__v8qi
)__A
, __zero
);
584 return _mm_cvtpu16_ps(__A
);
587 /* Convert the four signed 32-bit values in A and B to SPFP form. */
588 static __inline __m128
589 _mm_cvtpi32x2_ps(__m64 __A
, __m64 __B
)
591 __v4sf __zero
= (__v4sf
) __builtin_ia32_setzerops ();
592 __v4sf __sfa
= __builtin_ia32_cvtpi2ps (__zero
, (__v2si
)__A
);
593 __v4sf __sfb
= __builtin_ia32_cvtpi2ps (__zero
, (__v2si
)__B
);
594 return (__m128
) __builtin_ia32_movlhps (__sfa
, __sfb
);
597 /* Convert the four SPFP values in A to four signed 16-bit integers. */
598 static __inline __m64
599 _mm_cvtps_pi16(__m128 __A
)
601 __v4sf __hisf
= (__v4sf
)__A
;
602 __v4sf __losf
= __builtin_ia32_movhlps (__hisf
, __hisf
);
603 __v2si __hisi
= __builtin_ia32_cvtps2pi (__hisf
);
604 __v2si __losi
= __builtin_ia32_cvtps2pi (__losf
);
605 return (__m64
) __builtin_ia32_packssdw (__losi
, __hisi
);
608 /* Convert the four SPFP values in A to four signed 8-bit integers. */
609 static __inline __m64
610 _mm_cvtps_pi8(__m128 __A
)
612 __v4hi __tmp
= (__v4hi
) _mm_cvtps_pi16 (__A
);
613 __v4hi __zero
= (__v4hi
) __builtin_ia32_mmx_zero ();
614 return (__m64
) __builtin_ia32_packsswb (__tmp
, __zero
);
617 /* Selects four specific SPFP values from A and B based on MASK. */
619 static __inline __m128
620 _mm_shuffle_ps (__m128 __A
, __m128 __B
, int __mask
)
622 return (__m128
) __builtin_ia32_shufps ((__v4sf
)__A
, (__v4sf
)__B
, __mask
);
625 #define _mm_shuffle_ps(A, B, MASK) \
626 ((__m128) __builtin_ia32_shufps ((__v4sf)(A), (__v4sf)(B), (MASK)))
630 /* Selects and interleaves the upper two SPFP values from A and B. */
631 static __inline __m128
632 _mm_unpackhi_ps (__m128 __A
, __m128 __B
)
634 return (__m128
) __builtin_ia32_unpckhps ((__v4sf
)__A
, (__v4sf
)__B
);
637 /* Selects and interleaves the lower two SPFP values from A and B. */
638 static __inline __m128
639 _mm_unpacklo_ps (__m128 __A
, __m128 __B
)
641 return (__m128
) __builtin_ia32_unpcklps ((__v4sf
)__A
, (__v4sf
)__B
);
644 /* Sets the upper two SPFP values with 64-bits of data loaded from P;
645 the lower two values are passed through from A. */
646 static __inline __m128
647 _mm_loadh_pi (__m128 __A
, __m64
*__P
)
649 return (__m128
) __builtin_ia32_loadhps ((__v4sf
)__A
, (__v2si
*)__P
);
652 /* Stores the upper two SPFP values of A into P. */
654 _mm_storeh_pi (__m64
*__P
, __m128 __A
)
656 __builtin_ia32_storehps ((__v2si
*)__P
, (__v4sf
)__A
);
659 /* Moves the upper two values of B into the lower two values of A. */
660 static __inline __m128
661 _mm_movehl_ps (__m128 __A
, __m128 __B
)
663 return (__m128
) __builtin_ia32_movhlps ((__v4sf
)__A
, (__v4sf
)__B
);
666 /* Moves the lower two values of B into the upper two values of A. */
667 static __inline __m128
668 _mm_movelh_ps (__m128 __A
, __m128 __B
)
670 return (__m128
) __builtin_ia32_movlhps ((__v4sf
)__A
, (__v4sf
)__B
);
673 /* Sets the lower two SPFP values with 64-bits of data loaded from P;
674 the upper two values are passed through from A. */
675 static __inline __m128
676 _mm_loadl_pi (__m128 __A
, __m64
*__P
)
678 return (__m128
) __builtin_ia32_loadlps ((__v4sf
)__A
, (__v2si
*)__P
);
681 /* Stores the lower two SPFP values of A into P. */
683 _mm_storel_pi (__m64
*__P
, __m128 __A
)
685 __builtin_ia32_storelps ((__v2si
*)__P
, (__v4sf
)__A
);
688 /* Creates a 4-bit mask from the most significant bits of the SPFP values. */
690 _mm_movemask_ps (__m128 __A
)
692 return __builtin_ia32_movmskps ((__v4sf
)__A
);
695 /* Return the contents of the control register. */
696 static __inline
unsigned int
699 return __builtin_ia32_stmxcsr ();
702 /* Read exception bits from the control register. */
703 static __inline
unsigned int
704 _MM_GET_EXCEPTION_STATE (void)
706 return _mm_getcsr() & _MM_EXCEPT_MASK
;
709 static __inline
unsigned int
710 _MM_GET_EXCEPTION_MASK (void)
712 return _mm_getcsr() & _MM_MASK_MASK
;
715 static __inline
unsigned int
716 _MM_GET_ROUNDING_MODE (void)
718 return _mm_getcsr() & _MM_ROUND_MASK
;
721 static __inline
unsigned int
722 _MM_GET_FLUSH_ZERO_MODE (void)
724 return _mm_getcsr() & _MM_FLUSH_ZERO_MASK
;
727 /* Set the control register to I. */
729 _mm_setcsr (unsigned int __I
)
731 __builtin_ia32_ldmxcsr (__I
);
734 /* Set exception bits in the control register. */
736 _MM_SET_EXCEPTION_STATE(unsigned int __mask
)
738 _mm_setcsr((_mm_getcsr() & ~_MM_EXCEPT_MASK
) | __mask
);
742 _MM_SET_EXCEPTION_MASK (unsigned int __mask
)
744 _mm_setcsr((_mm_getcsr() & ~_MM_MASK_MASK
) | __mask
);
748 _MM_SET_ROUNDING_MODE (unsigned int __mode
)
750 _mm_setcsr((_mm_getcsr() & ~_MM_ROUND_MASK
) | __mode
);
754 _MM_SET_FLUSH_ZERO_MODE (unsigned int __mode
)
756 _mm_setcsr((_mm_getcsr() & ~_MM_FLUSH_ZERO_MASK
) | __mode
);
759 /* Create a vector with element 0 as *P and the rest zero. */
760 static __inline __m128
761 _mm_load_ss (float *__P
)
763 return (__m128
) __builtin_ia32_loadss (__P
);
766 /* Create a vector with all four elements equal to *P. */
767 static __inline __m128
768 _mm_load1_ps (float *__P
)
770 __v4sf __tmp
= __builtin_ia32_loadss (__P
);
771 return (__m128
) __builtin_ia32_shufps (__tmp
, __tmp
, _MM_SHUFFLE (0,0,0,0));
774 static __inline __m128
775 _mm_load_ps1 (float *__P
)
777 return _mm_load1_ps (__P
);
780 /* Load four SPFP values from P. The address must be 16-byte aligned. */
781 static __inline __m128
782 _mm_load_ps (float *__P
)
784 return (__m128
) __builtin_ia32_loadaps (__P
);
787 /* Load four SPFP values from P. The address need not be 16-byte aligned. */
788 static __inline __m128
789 _mm_loadu_ps (float *__P
)
791 return (__m128
) __builtin_ia32_loadups (__P
);
794 /* Load four SPFP values in reverse order. The address must be aligned. */
795 static __inline __m128
796 _mm_loadr_ps (float *__P
)
798 __v4sf __tmp
= __builtin_ia32_loadaps (__P
);
799 return (__m128
) __builtin_ia32_shufps (__tmp
, __tmp
, _MM_SHUFFLE (0,1,2,3));
802 /* Create a vector with element 0 as F and the rest zero. */
803 static __inline __m128
804 _mm_set_ss (float __F
)
806 return (__m128
) __builtin_ia32_loadss (&__F
);
809 /* Create a vector with all four elements equal to F. */
810 static __inline __m128
811 _mm_set1_ps (float __F
)
813 __v4sf __tmp
= __builtin_ia32_loadss (&__F
);
814 return (__m128
) __builtin_ia32_shufps (__tmp
, __tmp
, _MM_SHUFFLE (0,0,0,0));
817 static __inline __m128
818 _mm_set_ps1 (float __F
)
820 return _mm_set1_ps (__F
);
823 /* Create the vector [Z Y X W]. */
824 static __inline __m128
825 _mm_set_ps (float __Z
, float __Y
, float __X
, float __W
)
840 /* Create the vector [W X Y Z]. */
841 static __inline __m128
842 _mm_setr_ps (float __Z
, float __Y
, float __X
, float __W
)
844 return _mm_set_ps (__W
, __X
, __Y
, __Z
);
847 /* Create a vector of zeros. */
848 static __inline __m128
849 _mm_setzero_ps (void)
851 return (__m128
) __builtin_ia32_setzerops ();
854 /* Stores the lower SPFP value. */
856 _mm_store_ss (float *__P
, __m128 __A
)
858 __builtin_ia32_storess (__P
, (__v4sf
)__A
);
861 /* Store the lower SPFP value across four words. */
863 _mm_store1_ps (float *__P
, __m128 __A
)
865 __v4sf __va
= (__v4sf
)__A
;
866 __v4sf __tmp
= __builtin_ia32_shufps (__va
, __va
, _MM_SHUFFLE (0,0,0,0));
867 __builtin_ia32_storeaps (__P
, __tmp
);
871 _mm_store_ps1 (float *__P
, __m128 __A
)
873 _mm_store1_ps (__P
, __A
);
876 /* Store four SPFP values. The address must be 16-byte aligned. */
878 _mm_store_ps (float *__P
, __m128 __A
)
880 __builtin_ia32_storeaps (__P
, (__v4sf
)__A
);
883 /* Store four SPFP values. The address need not be 16-byte aligned. */
885 _mm_storeu_ps (float *__P
, __m128 __A
)
887 __builtin_ia32_storeups (__P
, (__v4sf
)__A
);
890 /* Store four SPFP values in reverse order. The address must be aligned. */
892 _mm_storer_ps (float *__P
, __m128 __A
)
894 __v4sf __va
= (__v4sf
)__A
;
895 __v4sf __tmp
= __builtin_ia32_shufps (__va
, __va
, _MM_SHUFFLE (0,1,2,3));
896 __builtin_ia32_storeaps (__P
, __tmp
);
899 /* Sets the low SPFP value of A from the low value of B. */
900 static __inline __m128
901 _mm_move_ss (__m128 __A
, __m128 __B
)
903 return (__m128
) __builtin_ia32_movss ((__v4sf
)__A
, (__v4sf
)__B
);
906 /* Extracts one of the four words of A. The selector N must be immediate. */
909 _mm_extract_pi16 (__m64 __A
, int __N
)
911 return __builtin_ia32_pextrw ((__v4hi
)__A
, __N
);
914 #define _mm_extract_pi16(A, N) \
915 __builtin_ia32_pextrw ((__v4hi)(A), (N))
918 /* Inserts word D into one of four words of A. The selector N must be
921 static __inline __m64
922 _mm_insert_pi16 (__m64 __A
, int __D
, int __N
)
924 return (__m64
)__builtin_ia32_pinsrw ((__v4hi
)__A
, __D
, __N
);
927 #define _mm_insert_pi16(A, D, N) \
928 ((__m64) __builtin_ia32_pinsrw ((__v4hi)(A), (D), (N)))
931 /* Compute the element-wise maximum of signed 16-bit values. */
932 static __inline __m64
933 _mm_max_pi16 (__m64 __A
, __m64 __B
)
935 return (__m64
) __builtin_ia32_pmaxsw ((__v4hi
)__A
, (__v4hi
)__B
);
938 /* Compute the element-wise maximum of unsigned 8-bit values. */
939 static __inline __m64
940 _mm_max_pu8 (__m64 __A
, __m64 __B
)
942 return (__m64
) __builtin_ia32_pmaxub ((__v8qi
)__A
, (__v8qi
)__B
);
945 /* Compute the element-wise minimum of signed 16-bit values. */
946 static __inline __m64
947 _mm_min_pi16 (__m64 __A
, __m64 __B
)
949 return (__m64
) __builtin_ia32_pminsw ((__v4hi
)__A
, (__v4hi
)__B
);
952 /* Compute the element-wise minimum of unsigned 8-bit values. */
953 static __inline __m64
954 _mm_min_pu8 (__m64 __A
, __m64 __B
)
956 return (__m64
) __builtin_ia32_pminub ((__v8qi
)__A
, (__v8qi
)__B
);
959 /* Create an 8-bit mask of the signs of 8-bit values. */
961 _mm_movemask_pi8 (__m64 __A
)
963 return __builtin_ia32_pmovmskb ((__v8qi
)__A
);
966 /* Multiply four unsigned 16-bit values in A by four unsigned 16-bit values
967 in B and produce the high 16 bits of the 32-bit results. */
968 static __inline __m64
969 _mm_mulhi_pu16 (__m64 __A
, __m64 __B
)
971 return (__m64
) __builtin_ia32_pmulhuw ((__v4hi
)__A
, (__v4hi
)__B
);
974 /* Return a combination of the four 16-bit values in A. The selector
975 must be an immediate. */
977 static __inline __m64
978 _mm_shuffle_pi16 (__m64 __A
, int __N
)
980 return (__m64
) __builtin_ia32_pshufw ((__v4hi
)__A
, __N
);
983 #define _mm_shuffle_pi16(A, N) \
984 ((__m64) __builtin_ia32_pshufw ((__v4hi)(A), (N)))
987 /* Conditionally store byte elements of A into P. The high bit of each
988 byte in the selector N determines whether the corresponding byte from
991 _mm_maskmove_si64 (__m64 __A
, __m64 __N
, char *__P
)
993 __builtin_ia32_maskmovq ((__v8qi
)__A
, (__v8qi
)__N
, __P
);
996 /* Compute the rounded averages of the unsigned 8-bit values in A and B. */
997 static __inline __m64
998 _mm_avg_pu8 (__m64 __A
, __m64 __B
)
1000 return (__m64
) __builtin_ia32_pavgb ((__v8qi
)__A
, (__v8qi
)__B
);
1003 /* Compute the rounded averages of the unsigned 16-bit values in A and B. */
1004 static __inline __m64
1005 _mm_avg_pu16 (__m64 __A
, __m64 __B
)
1007 return (__m64
) __builtin_ia32_pavgw ((__v4hi
)__A
, (__v4hi
)__B
);
1010 /* Compute the sum of the absolute differences of the unsigned 8-bit
1011 values in A and B. Return the value in the lower 16-bit word; the
1012 upper words are cleared. */
1013 static __inline __m64
1014 _mm_sad_pu8 (__m64 __A
, __m64 __B
)
1016 return (__m64
) __builtin_ia32_psadbw ((__v8qi
)__A
, (__v8qi
)__B
);
1019 /* Loads one cache line from address P to a location "closer" to the
1020 processor. The selector I specifies the type of prefetch operation. */
1022 static __inline
void
1023 _mm_prefetch (void *__P
, enum _mm_hint __I
)
1025 __builtin_prefetch (__P
, 0, __I
);
1028 #define _mm_prefetch(P, I) \
1029 __builtin_prefetch ((P), 0, (I))
1032 /* Stores the data in A to the address P without polluting the caches. */
1033 static __inline
void
1034 _mm_stream_pi (__m64
*__P
, __m64 __A
)
1036 __builtin_ia32_movntq (__P
, __A
);
1039 /* Likewise. The address must be 16-byte aligned. */
1040 static __inline
void
1041 _mm_stream_ps (float *__P
, __m128 __A
)
1043 __builtin_ia32_movntps (__P
, (__v4sf
)__A
);
1046 /* Guarantees that every preceeding store is globally visible before
1047 any subsequent store. */
1048 static __inline
void
1051 __builtin_ia32_sfence ();
1054 /* The execution of the next instruction is delayed by an implementation
1055 specific amount of time. The instruction does not modify the
1056 architectural state. */
1057 static __inline
void
1060 __asm__
__volatile__ ("rep; nop" : : );
1063 /* Transpose the 4x4 matrix composed of row[0-3]. */
1064 #define _MM_TRANSPOSE4_PS(row0, row1, row2, row3) \
1066 __v4sf __r0 = (row0), __r1 = (row1), __r2 = (row2), __r3 = (row3); \
1067 __v4sf __t0 = __builtin_ia32_shufps (__r0, __r1, 0x44); \
1068 __v4sf __t1 = __builtin_ia32_shufps (__r0, __r1, 0xEE); \
1069 __v4sf __t2 = __builtin_ia32_shufps (__r2, __r3, 0x44); \
1070 __v4sf __t3 = __builtin_ia32_shufps (__r2, __r3, 0xEE); \
1071 (row0) = __builtin_ia32_shufps (__t0, __t1, 0x88); \
1072 (row1) = __builtin_ia32_shufps (__t0, __t1, 0xDD); \
1073 (row2) = __builtin_ia32_shufps (__t2, __t3, 0x88); \
1074 (row3) = __builtin_ia32_shufps (__t2, __t3, 0xDD); \
1079 typedef int __v2df
__attribute__ ((mode (V2DF
)));
1080 typedef int __v2di
__attribute__ ((mode (V2DI
)));
1081 typedef int __v4si
__attribute__ ((mode (V4SI
)));
1082 typedef int __v8hi
__attribute__ ((mode (V8HI
)));
1083 typedef int __v16qi
__attribute__ ((mode (V16QI
)));
1085 #define __m128i __v2di
1086 #define __m128d __v2df
1088 static __inline __m128d
1089 _mm_add_pd (__m128d __A
, __m128d __B
)
1091 return (__m128d
)__builtin_ia32_addpd ((__v2df
)__A
, (__v2df
)__B
);
1094 static __inline __m128d
1095 _mm_add_sd (__m128d __A
, __m128d __B
)
1097 return (__m128d
)__builtin_ia32_addsd ((__v2df
)__A
, (__v2df
)__B
);
1100 static __inline __m128d
1101 _mm_sub_pd (__m128d __A
, __m128d __B
)
1103 return (__m128d
)__builtin_ia32_subpd ((__v2df
)__A
, (__v2df
)__B
);
1106 static __inline __m128d
1107 _mm_sub_sd (__m128d __A
, __m128d __B
)
1109 return (__m128d
)__builtin_ia32_subsd ((__v2df
)__A
, (__v2df
)__B
);
1112 static __inline __m128d
1113 _mm_mul_pd (__m128d __A
, __m128d __B
)
1115 return (__m128d
)__builtin_ia32_mulpd ((__v2df
)__A
, (__v2df
)__B
);
1118 static __inline __m128d
1119 _mm_mul_sd (__m128d __A
, __m128d __B
)
1121 return (__m128d
)__builtin_ia32_mulsd ((__v2df
)__A
, (__v2df
)__B
);
1124 static __inline __m128d
1125 _mm_div_pd (__m128d __A
, __m128d __B
)
1127 return (__m128d
)__builtin_ia32_divpd ((__v2df
)__A
, (__v2df
)__B
);
1130 static __inline __m128d
1131 _mm_div_sd (__m128d __A
, __m128d __B
)
1133 return (__m128d
)__builtin_ia32_divsd ((__v2df
)__A
, (__v2df
)__B
);
1136 static __inline __m128d
1137 _mm_sqrt_pd (__m128d __A
)
1139 return (__m128d
)__builtin_ia32_sqrtpd ((__v2df
)__A
);
1142 static __inline __m128d
1143 _mm_sqrt_sd (__m128d __A
)
1145 return (__m128d
)__builtin_ia32_sqrtsd ((__v2df
)__A
);
1148 static __inline __m128d
1149 _mm_min_pd (__m128d __A
, __m128d __B
)
1151 return (__m128d
)__builtin_ia32_minpd ((__v2df
)__A
, (__v2df
)__B
);
1154 static __inline __m128d
1155 _mm_min_sd (__m128d __A
, __m128d __B
)
1157 return (__m128d
)__builtin_ia32_minsd ((__v2df
)__A
, (__v2df
)__B
);
1160 static __inline __m128d
1161 _mm_max_pd (__m128d __A
, __m128d __B
)
1163 return (__m128d
)__builtin_ia32_maxpd ((__v2df
)__A
, (__v2df
)__B
);
1166 static __inline __m128d
1167 _mm_max_sd (__m128d __A
, __m128d __B
)
1169 return (__m128d
)__builtin_ia32_minsd ((__v2df
)__A
, (__v2df
)__B
);
1172 static __inline __m128d
1173 _mm_and_pd (__m128d __A
, __m128d __B
)
1175 return (__m128d
)__builtin_ia32_andpd ((__v2df
)__A
, (__v2df
)__B
);
1178 static __inline __m128d
1179 _mm_andnot_pd (__m128d __A
, __m128d __B
)
1181 return (__m128d
)__builtin_ia32_andnpd ((__v2df
)__A
, (__v2df
)__B
);
1184 static __inline __m128d
1185 _mm_or_pd (__m128d __A
, __m128d __B
)
1187 return (__m128d
)__builtin_ia32_orpd ((__v2df
)__A
, (__v2df
)__B
);
1190 static __inline __m128d
1191 _mm_xor_pd (__m128d __A
, __m128d __B
)
1193 return (__m128d
)__builtin_ia32_xorpd ((__v2df
)__A
, (__v2df
)__B
);
1196 static __inline __m128d
1197 _mm_cmpeq_pd (__m128d __A
, __m128d __B
)
1199 return (__m128d
)__builtin_ia32_cmpeqpd ((__v2df
)__A
, (__v2df
)__B
);
1202 static __inline __m128d
1203 _mm_cmplt_pd (__m128d __A
, __m128d __B
)
1205 return (__m128d
)__builtin_ia32_cmpltpd ((__v2df
)__A
, (__v2df
)__B
);
1208 static __inline __m128d
1209 _mm_cmple_pd (__m128d __A
, __m128d __B
)
1211 return (__m128d
)__builtin_ia32_cmplepd ((__v2df
)__A
, (__v2df
)__B
);
1214 static __inline __m128d
1215 _mm_cmpgt_pd (__m128d __A
, __m128d __B
)
1217 return (__m128d
)__builtin_ia32_cmpgtpd ((__v2df
)__A
, (__v2df
)__B
);
1220 static __inline __m128d
1221 _mm_cmpge_pd (__m128d __A
, __m128d __B
)
1223 return (__m128d
)__builtin_ia32_cmpgepd ((__v2df
)__A
, (__v2df
)__B
);
1226 static __inline __m128d
1227 _mm_cmpneq_pd (__m128d __A
, __m128d __B
)
1229 return (__m128d
)__builtin_ia32_cmpneqpd ((__v2df
)__A
, (__v2df
)__B
);
1232 static __inline __m128d
1233 _mm_cmpnlt_pd (__m128d __A
, __m128d __B
)
1235 return (__m128d
)__builtin_ia32_cmpnltpd ((__v2df
)__A
, (__v2df
)__B
);
1238 static __inline __m128d
1239 _mm_cmpnle_pd (__m128d __A
, __m128d __B
)
1241 return (__m128d
)__builtin_ia32_cmpnlepd ((__v2df
)__A
, (__v2df
)__B
);
1244 static __inline __m128d
1245 _mm_cmpngt_pd (__m128d __A
, __m128d __B
)
1247 return (__m128d
)__builtin_ia32_cmpngtpd ((__v2df
)__A
, (__v2df
)__B
);
1250 static __inline __m128d
1251 _mm_cmpnge_pd (__m128d __A
, __m128d __B
)
1253 return (__m128d
)__builtin_ia32_cmpngepd ((__v2df
)__A
, (__v2df
)__B
);
1256 static __inline __m128d
1257 _mm_cmpord_pd (__m128d __A
, __m128d __B
)
1259 return (__m128d
)__builtin_ia32_cmpordpd ((__v2df
)__A
, (__v2df
)__B
);
1262 static __inline __m128d
1263 _mm_cmpunord_pd (__m128d __A
, __m128d __B
)
1265 return (__m128d
)__builtin_ia32_cmpunordpd ((__v2df
)__A
, (__v2df
)__B
);
1268 static __inline __m128d
1269 _mm_cmpeq_sd (__m128d __A
, __m128d __B
)
1271 return (__m128d
)__builtin_ia32_cmpeqsd ((__v2df
)__A
, (__v2df
)__B
);
1274 static __inline __m128d
1275 _mm_cmplt_sd (__m128d __A
, __m128d __B
)
1277 return (__m128d
)__builtin_ia32_cmpltsd ((__v2df
)__A
, (__v2df
)__B
);
1280 static __inline __m128d
1281 _mm_cmple_sd (__m128d __A
, __m128d __B
)
1283 return (__m128d
)__builtin_ia32_cmplesd ((__v2df
)__A
, (__v2df
)__B
);
1286 static __inline __m128d
1287 _mm_cmpgt_sd (__m128d __A
, __m128d __B
)
1289 return (__m128d
) __builtin_ia32_movsd ((__v2df
) __A
,
1291 __builtin_ia32_cmpltsd ((__v2df
) __B
,
1296 static __inline __m128d
1297 _mm_cmpge_sd (__m128d __A
, __m128d __B
)
1299 return (__m128d
) __builtin_ia32_movsd ((__v2df
) __A
,
1301 __builtin_ia32_cmplesd ((__v2df
) __B
,
1306 static __inline __m128d
1307 _mm_cmpneq_sd (__m128d __A
, __m128d __B
)
1309 return (__m128d
)__builtin_ia32_cmpneqsd ((__v2df
)__A
, (__v2df
)__B
);
1312 static __inline __m128d
1313 _mm_cmpnlt_sd (__m128d __A
, __m128d __B
)
1315 return (__m128d
)__builtin_ia32_cmpnltsd ((__v2df
)__A
, (__v2df
)__B
);
1318 static __inline __m128d
1319 _mm_cmpnle_sd (__m128d __A
, __m128d __B
)
1321 return (__m128d
)__builtin_ia32_cmpnlesd ((__v2df
)__A
, (__v2df
)__B
);
1324 static __inline __m128d
1325 _mm_cmpngt_sd (__m128d __A
, __m128d __B
)
1327 return (__m128d
) __builtin_ia32_movsd ((__v2df
) __A
,
1329 __builtin_ia32_cmpnltsd ((__v2df
) __B
,
1334 static __inline __m128d
1335 _mm_cmpnge_sd (__m128d __A
, __m128d __B
)
1337 return (__m128d
) __builtin_ia32_movsd ((__v2df
) __A
,
1339 __builtin_ia32_cmpnlesd ((__v2df
) __B
,
1344 static __inline __m128d
1345 _mm_cmpord_sd (__m128d __A
, __m128d __B
)
1347 return (__m128d
)__builtin_ia32_cmpordsd ((__v2df
)__A
, (__v2df
)__B
);
1350 static __inline __m128d
1351 _mm_cmpunord_sd (__m128d __A
, __m128d __B
)
1353 return (__m128d
)__builtin_ia32_cmpunordsd ((__v2df
)__A
, (__v2df
)__B
);
1357 _mm_comieq_sd (__m128d __A
, __m128d __B
)
1359 return __builtin_ia32_comisdeq ((__v2df
)__A
, (__v2df
)__B
);
1363 _mm_comilt_sd (__m128d __A
, __m128d __B
)
1365 return __builtin_ia32_comisdlt ((__v2df
)__A
, (__v2df
)__B
);
1369 _mm_comile_sd (__m128d __A
, __m128d __B
)
1371 return __builtin_ia32_comisdle ((__v2df
)__A
, (__v2df
)__B
);
1375 _mm_comigt_sd (__m128d __A
, __m128d __B
)
1377 return __builtin_ia32_comisdgt ((__v2df
)__A
, (__v2df
)__B
);
1381 _mm_comige_sd (__m128d __A
, __m128d __B
)
1383 return __builtin_ia32_comisdge ((__v2df
)__A
, (__v2df
)__B
);
1387 _mm_comineq_sd (__m128d __A
, __m128d __B
)
1389 return __builtin_ia32_comisdneq ((__v2df
)__A
, (__v2df
)__B
);
1393 _mm_ucomieq_sd (__m128d __A
, __m128d __B
)
1395 return __builtin_ia32_ucomisdeq ((__v2df
)__A
, (__v2df
)__B
);
1399 _mm_ucomilt_sd (__m128d __A
, __m128d __B
)
1401 return __builtin_ia32_ucomisdlt ((__v2df
)__A
, (__v2df
)__B
);
1405 _mm_ucomile_sd (__m128d __A
, __m128d __B
)
1407 return __builtin_ia32_ucomisdle ((__v2df
)__A
, (__v2df
)__B
);
1411 _mm_ucomigt_sd (__m128d __A
, __m128d __B
)
1413 return __builtin_ia32_ucomisdgt ((__v2df
)__A
, (__v2df
)__B
);
1417 _mm_ucomige_sd (__m128d __A
, __m128d __B
)
1419 return __builtin_ia32_ucomisdge ((__v2df
)__A
, (__v2df
)__B
);
1423 _mm_ucomineq_sd (__m128d __A
, __m128d __B
)
1425 return __builtin_ia32_ucomisdneq ((__v2df
)__A
, (__v2df
)__B
);
1428 static __inline __m128d
1429 _mm_cvtepi32_pd (__m128i __A
)
1431 return (__m128d
)__builtin_ia32_cvtdq2pd ((__v4si
) __A
);
1434 static __inline __m128d
1435 _mm_cvtepi32_ps (__m128i __A
)
1437 return (__m128d
)__builtin_ia32_cvtdq2ps ((__v4si
) __A
);
1440 static __inline __m128d
1441 _mm_cvtpd_epi32 (__m128d __A
)
1443 return (__m128d
)__builtin_ia32_cvtpd2dq ((__v2df
) __A
);
1446 static __inline __m64
1447 _mm_cvtpd_pi32 (__m128d __A
)
1449 return (__m64
)__builtin_ia32_cvtpd2pi ((__v2df
) __A
);
1452 static __inline __m128d
1453 _mm_cvtpd_ps (__m128d __A
)
1455 return (__m128d
)__builtin_ia32_cvtpd2ps ((__v2df
) __A
);
1458 static __inline __m128d
1459 _mm_cvttpd_epi32 (__m128d __A
)
1461 return (__m128d
)__builtin_ia32_cvttpd2dq ((__v2df
) __A
);
1464 static __inline __m64
1465 _mm_cvttpd_pi32 (__m128d __A
)
1467 return (__m64
)__builtin_ia32_cvttpd2pi ((__v2df
) __A
);
1470 static __inline __m128d
1471 _mm_cvtpi32_pd (__m64 __A
)
1473 return (__m128d
)__builtin_ia32_cvtpi2pd ((__v2si
) __A
);
1476 static __inline __m128d
1477 _mm_cvtps_epi32 (__m128d __A
)
1479 return (__m128d
)__builtin_ia32_cvtps2dq ((__v4sf
) __A
);
1482 static __inline __m128d
1483 _mm_cvttps_epi32 (__m128d __A
)
1485 return (__m128d
)__builtin_ia32_cvttps2dq ((__v4sf
) __A
);
1488 static __inline __m128d
1489 _mm_cvtps_pd (__m128d __A
)
1491 return (__m128d
)__builtin_ia32_cvtps2pd ((__v4sf
) __A
);
1495 _mm_cvtsd_si32 (__m128d __A
)
1497 return __builtin_ia32_cvtsd2si ((__v2df
) __A
);
1501 _mm_cvttsd_si32 (__m128d __A
)
1503 return __builtin_ia32_cvttsd2si ((__v2df
) __A
);
1506 static __inline __m128d
1507 _mm_cvtsd_ss (__m128d __A
, __m128d __B
)
1509 return (__m128d
)__builtin_ia32_cvtsd2ss ((__v4sf
) __A
, (__v2df
) __B
);
1512 static __inline __m128d
1513 _mm_cvtsi32_sd (__m128d __A
, int __B
)
1515 return (__m128d
)__builtin_ia32_cvtsi2sd ((__v2df
) __A
, __B
);
1518 static __inline __m128d
1519 _mm_cvtss_sd (__m128d __A
, __m128d __B
)
1521 return (__m128d
)__builtin_ia32_cvtss2sd ((__v2df
) __A
, (__v4sf
)__B
);
1524 #define _mm_shuffle_pd(__A, __B, __C) ((__m128d)__builtin_ia32_shufpd ((__v2df)__A, (__v2df)__B, (__C)))
1526 static __inline __m128d
1527 _mm_unpackhi_pd (__m128d __A
, __m128d __B
)
1529 return (__m128d
)__builtin_ia32_unpckhpd ((__v2df
)__A
, (__v2df
)__B
);
1532 static __inline __m128d
1533 _mm_unpacklo_pd (__m128d __A
, __m128d __B
)
1535 return (__m128d
)__builtin_ia32_unpcklpd ((__v2df
)__A
, (__v2df
)__B
);
1538 static __inline __m128d
1539 _mm_loadh_pd (__m128d __A
, double *__B
)
1541 return (__m128d
)__builtin_ia32_loadhpd ((__v2df
)__A
, (__v2si
*)__B
);
1544 static __inline
void
1545 _mm_storeh_pd (__m128d
*__A
, double __B
)
1547 __v2df __tmp
= __builtin_ia32_loadsd (&__B
);
1548 __builtin_ia32_storehpd ((__v2si
*)__A
, __tmp
);
1551 static __inline __m128d
1552 _mm_loadl_pd (__m128d __A
, double *__B
)
1554 return (__m128d
)__builtin_ia32_loadlpd ((__v2df
)__A
, (__v2si
*)__B
);
1557 static __inline
void
1558 _mm_storel_pd (__m128d
*__A
, double __B
)
1560 __v2df __tmp
= __builtin_ia32_loadsd (&__B
);
1561 __builtin_ia32_storelpd ((__v2si
*)__A
, __tmp
);
1565 _mm_movemask_pd (__m128d __A
)
1567 return __builtin_ia32_movmskpd ((__v2df
)__A
);
1570 static __inline __m128i
1571 _mm_packs_epi16 (__m128i __A
, __m128i __B
)
1573 return (__m128i
)__builtin_ia32_packsswb128 ((__v8hi
)__A
, (__v8hi
)__B
);
1576 static __inline __m128i
1577 _mm_packs_epi32 (__m128i __A
, __m128i __B
)
1579 return (__m128i
)__builtin_ia32_packssdw128 ((__v4si
)__A
, (__v4si
)__B
);
1582 static __inline __m128i
1583 _mm_packus_epi16 (__m128i __A
, __m128i __B
)
1585 return (__m128i
)__builtin_ia32_packuswb128 ((__v8hi
)__A
, (__v8hi
)__B
);
1588 static __inline __m128i
1589 _mm_unpackhi_epi8 (__m128i __A
, __m128i __B
)
1591 return (__m128i
)__builtin_ia32_punpckhbw128 ((__v16qi
)__A
, (__v16qi
)__B
);
1594 static __inline __m128i
1595 _mm_unpackhi_epi16 (__m128i __A
, __m128i __B
)
1597 return (__m128i
)__builtin_ia32_punpckhwd128 ((__v8hi
)__A
, (__v8hi
)__B
);
1600 static __inline __m128i
1601 _mm_unpackhi_epi32 (__m128i __A
, __m128i __B
)
1603 return (__m128i
)__builtin_ia32_punpckhdq128 ((__v4si
)__A
, (__v4si
)__B
);
1606 static __inline __m128i
1607 _mm_unpacklo_epi8 (__m128i __A
, __m128i __B
)
1609 return (__m128i
)__builtin_ia32_punpcklbw128 ((__v16qi
)__A
, (__v16qi
)__B
);
1612 static __inline __m128i
1613 _mm_unpacklo_epi16 (__m128i __A
, __m128i __B
)
1615 return (__m128i
)__builtin_ia32_punpcklwd128 ((__v8hi
)__A
, (__v8hi
)__B
);
1618 static __inline __m128i
1619 _mm_unpacklo_epi32 (__m128i __A
, __m128i __B
)
1621 return (__m128i
)__builtin_ia32_punpckldq128 ((__v4si
)__A
, (__v4si
)__B
);
1624 static __inline __m128i
1625 _mm_add_epi8 (__m128i __A
, __m128i __B
)
1627 return (__m128i
)__builtin_ia32_paddb128 ((__v16qi
)__A
, (__v16qi
)__B
);
1630 static __inline __m128i
1631 _mm_add_epi16 (__m128i __A
, __m128i __B
)
1633 return (__m128i
)__builtin_ia32_paddw128 ((__v8hi
)__A
, (__v8hi
)__B
);
1636 static __inline __m128i
1637 _mm_add_epi32 (__m128i __A
, __m128i __B
)
1639 return (__m128i
)__builtin_ia32_paddd128 ((__v4si
)__A
, (__v4si
)__B
);
1642 static __inline __m128i
1643 _mm_add_epi64 (__m128i __A
, __m128i __B
)
1645 return (__m128i
)__builtin_ia32_paddq128 ((__v4si
)__A
, (__v4si
)__B
);
1648 static __inline __m128i
1649 _mm_adds_epi8 (__m128i __A
, __m128i __B
)
1651 return (__m128i
)__builtin_ia32_paddsb128 ((__v16qi
)__A
, (__v16qi
)__B
);
1654 static __inline __m128i
1655 _mm_adds_epi16 (__m128i __A
, __m128i __B
)
1657 return (__m128i
)__builtin_ia32_paddsw128 ((__v8hi
)__A
, (__v8hi
)__B
);
1660 static __inline __m128i
1661 _mm_adds_epu8 (__m128i __A
, __m128i __B
)
1663 return (__m128i
)__builtin_ia32_paddusb128 ((__v16qi
)__A
, (__v16qi
)__B
);
1666 static __inline __m128i
1667 _mm_adds_epu16 (__m128i __A
, __m128i __B
)
1669 return (__m128i
)__builtin_ia32_paddusw128 ((__v8hi
)__A
, (__v8hi
)__B
);
1672 static __inline __m128i
1673 _mm_sub_epi8 (__m128i __A
, __m128i __B
)
1675 return (__m128i
)__builtin_ia32_psubb128 ((__v16qi
)__A
, (__v16qi
)__B
);
1678 static __inline __m128i
1679 _mm_sub_epi16 (__m128i __A
, __m128i __B
)
1681 return (__m128i
)__builtin_ia32_psubw128 ((__v8hi
)__A
, (__v8hi
)__B
);
1684 static __inline __m128i
1685 _mm_sub_epi32 (__m128i __A
, __m128i __B
)
1687 return (__m128i
)__builtin_ia32_psubd128 ((__v4si
)__A
, (__v4si
)__B
);
1690 static __inline __m128i
1691 _mm_sub_epi64 (__m128i __A
, __m128i __B
)
1693 return (__m128i
)__builtin_ia32_psubq128 ((__v4si
)__A
, (__v4si
)__B
);
1696 static __inline __m128i
1697 _mm_subs_epi8 (__m128i __A
, __m128i __B
)
1699 return (__m128i
)__builtin_ia32_psubsb128 ((__v16qi
)__A
, (__v16qi
)__B
);
1702 static __inline __m128i
1703 _mm_subs_epi16 (__m128i __A
, __m128i __B
)
1705 return (__m128i
)__builtin_ia32_psubsw128 ((__v8hi
)__A
, (__v8hi
)__B
);
1708 static __inline __m128i
1709 _mm_subs_epu8 (__m128i __A
, __m128i __B
)
1711 return (__m128i
)__builtin_ia32_psubusb128 ((__v16qi
)__A
, (__v16qi
)__B
);
1714 static __inline __m128i
1715 _mm_subs_epu16 (__m128i __A
, __m128i __B
)
1717 return (__m128i
)__builtin_ia32_psubusw128 ((__v8hi
)__A
, (__v8hi
)__B
);
1720 static __inline __m128i
1721 _mm_madd_epi16 (__m128i __A
, __m128i __B
)
1723 return (__m128i
)__builtin_ia32_pmaddwd128 ((__v8hi
)__A
, (__v8hi
)__B
);
1726 static __inline __m128i
1727 _mm_mulhi_epi16 (__m128i __A
, __m128i __B
)
1729 return (__m128i
)__builtin_ia32_pmulhw128 ((__v8hi
)__A
, (__v8hi
)__B
);
1732 static __inline __m128i
1733 _mm_mullo_epi16 (__m128i __A
, __m128i __B
)
1735 return (__m128i
)__builtin_ia32_pmullw128 ((__v8hi
)__A
, (__v8hi
)__B
);
1738 static __inline __m64
1739 _mm_mul_pu16 (__m64 __A
, __m64 __B
)
1741 return (__m64
)__builtin_ia32_pmuludq ((__v2si
)__A
, (__v2si
)__B
);
1744 static __inline __m128i
1745 _mm_mul_epu16 (__m128i __A
, __m128i __B
)
1747 return (__m128i
)__builtin_ia32_pmuludq128 ((__v4si
)__A
, (__v4si
)__B
);
1750 static __inline __m128i
1751 _mm_sll_epi16 (__m128i __A
, __m128i __B
)
1753 return (__m128i
)__builtin_ia32_psllw128 ((__v8hi
)__A
, (__v2di
)__B
);
1756 static __inline __m128i
1757 _mm_sll_epi32 (__m128i __A
, __m128i __B
)
1759 return (__m128i
)__builtin_ia32_pslld128 ((__v4si
)__A
, (__v2di
)__B
);
1762 static __inline __m128i
1763 _mm_sll_epi64 (__m128i __A
, __m128i __B
)
1765 return (__m128i
)__builtin_ia32_psllq128 ((__v2di
)__A
, (__v2di
)__B
);
1768 static __inline __m128i
1769 _mm_sra_epi16 (__m128i __A
, __m128i __B
)
1771 return (__m128i
)__builtin_ia32_psraw128 ((__v8hi
)__A
, (__v2di
)__B
);
1774 static __inline __m128i
1775 _mm_sra_epi32 (__m128i __A
, __m128i __B
)
1777 return (__m128i
)__builtin_ia32_psrad128 ((__v4si
)__A
, (__v2di
)__B
);
1780 static __inline __m128i
1781 _mm_srl_epi16 (__m128i __A
, __m128i __B
)
1783 return (__m128i
)__builtin_ia32_psrlw128 ((__v8hi
)__A
, (__v2di
)__B
);
1786 static __inline __m128i
1787 _mm_srl_epi32 (__m128i __A
, __m128i __B
)
1789 return (__m128i
)__builtin_ia32_psrld128 ((__v4si
)__A
, (__v2di
)__B
);
1792 static __inline __m128i
1793 _mm_srl_epi64 (__m128i __A
, __m128i __B
)
1795 return (__m128i
)__builtin_ia32_psrlq128 ((__v2di
)__A
, (__v2di
)__B
);
1798 static __inline __m128i
1799 _mm_slli_epi16 (__m128i __A
, int __B
)
1801 return (__m128i
)__builtin_ia32_psllwi128 ((__v8hi
)__A
, __B
);
1804 static __inline __m128i
1805 _mm_slli_epi32 (__m128i __A
, int __B
)
1807 return (__m128i
)__builtin_ia32_pslldi128 ((__v4si
)__A
, __B
);
1810 static __inline __m128i
1811 _mm_slli_epi64 (__m128i __A
, int __B
)
1813 return (__m128i
)__builtin_ia32_psllqi128 ((__v2di
)__A
, __B
);
1816 static __inline __m128i
1817 _mm_srai_epi16 (__m128i __A
, int __B
)
1819 return (__m128i
)__builtin_ia32_psrawi128 ((__v8hi
)__A
, __B
);
1822 static __inline __m128i
1823 _mm_srai_epi32 (__m128i __A
, int __B
)
1825 return (__m128i
)__builtin_ia32_psradi128 ((__v4si
)__A
, __B
);
1828 static __inline __m128i
1829 _mm_srli_epi16 (__m128i __A
, int __B
)
1831 return (__m128i
)__builtin_ia32_psrlwi128 ((__v8hi
)__A
, __B
);
1834 static __inline __m128i
1835 _mm_srli_epi32 (__m128i __A
, int __B
)
1837 return (__m128i
)__builtin_ia32_psrldi128 ((__v4si
)__A
, __B
);
1840 static __inline __m128i
1841 _mm_srli_epi64 (__m128i __A
, int __B
)
1843 return (__m128i
)__builtin_ia32_psrlqi128 ((__v2di
)__A
, __B
);
1846 static __inline __m128i
1847 _mm_and_si128 (__m128i __A
, __m128i __B
)
1849 return (__m128i
)__builtin_ia32_pand128 ((__v2di
)__A
, (__v2di
)__B
);
1852 static __inline __m128i
1853 _mm_andnot_si128 (__m128i __A
, __m128i __B
)
1855 return (__m128i
)__builtin_ia32_pandn128 ((__v2di
)__A
, (__v2di
)__B
);
1858 static __inline __m128i
1859 _mm_or_si128 (__m128i __A
, __m128i __B
)
1861 return (__m128i
)__builtin_ia32_por128 ((__v2di
)__A
, (__v2di
)__B
);
1864 static __inline __m128i
1865 _mm_xor_si128 (__m128i __A
, __m128i __B
)
1867 return (__m128i
)__builtin_ia32_pxor128 ((__v2di
)__A
, (__v2di
)__B
);
1870 static __inline __m128i
1871 _mm_cmpeq_epi8 (__m128i __A
, __m128i __B
)
1873 return (__m128i
)__builtin_ia32_pcmpeqb128 ((__v16qi
)__A
, (__v16qi
)__B
);
1876 static __inline __m128i
1877 _mm_cmpeq_epi16 (__m128i __A
, __m128i __B
)
1879 return (__m128i
)__builtin_ia32_pcmpeqw128 ((__v8hi
)__A
, (__v8hi
)__B
);
1882 static __inline __m128i
1883 _mm_cmpeq_epi32 (__m128i __A
, __m128i __B
)
1885 return (__m128i
)__builtin_ia32_pcmpeqd128 ((__v4si
)__A
, (__v4si
)__B
);
1888 static __inline __m128i
1889 _mm_cmpgt_epi8 (__m128i __A
, __m128i __B
)
1891 return (__m128i
)__builtin_ia32_pcmpgtb128 ((__v16qi
)__A
, (__v16qi
)__B
);
1894 static __inline __m128i
1895 _mm_cmpgt_epi16 (__m128i __A
, __m128i __B
)
1897 return (__m128i
)__builtin_ia32_pcmpgtw128 ((__v8hi
)__A
, (__v8hi
)__B
);
1900 static __inline __m128i
1901 _mm_cmpgt_epi32 (__m128i __A
, __m128i __B
)
1903 return (__m128i
)__builtin_ia32_pcmpgtd128 ((__v4si
)__A
, (__v4si
)__B
);
1906 #define _mm_extract_epi16(__A, __B) __builtin_ia32_pextrw128 ((__v8hi)__A, __B)
1908 #define _mm_insert_epi16 (__A, __B, __C) ((__m128i)__builtin_ia32_pinsrw128 ((__v8hi)__A, __B, __C))
1910 static __inline __m128i
1911 _mm_max_epi16 (__m128i __A
, __m128i __B
)
1913 return (__m128i
)__builtin_ia32_pmaxsw128 ((__v8hi
)__A
, (__v8hi
)__B
);
1916 static __inline __m128i
1917 _mm_max_epu8 (__m128i __A
, __m128i __B
)
1919 return (__m128i
)__builtin_ia32_pmaxub128 ((__v16qi
)__A
, (__v16qi
)__B
);
1922 static __inline __m128i
1923 _mm_min_epi16 (__m128i __A
, __m128i __B
)
1925 return (__m128i
)__builtin_ia32_pminsw128 ((__v8hi
)__A
, (__v8hi
)__B
);
1928 static __inline __m128i
1929 _mm_min_epu8 (__m128i __A
, __m128i __B
)
1931 return (__m128i
)__builtin_ia32_pminub128 ((__v16qi
)__A
, (__v16qi
)__B
);
1935 _mm_movemask_epi8 (__m128i __A
)
1937 return __builtin_ia32_pmovmskb128 ((__v16qi
)__A
);
1940 static __inline __m128i
1941 _mm_mulhi_epu16 (__m128i __A
, __m128i __B
)
1943 return (__m128i
)__builtin_ia32_pmulhuw128 ((__v8hi
)__A
, (__v8hi
)__B
);
1946 #define _mm_shufflehi_epi16(__A, __B) ((__m128i)__builtin_ia32_pshufhw128 ((__v8hi)__A, __B))
1947 #define _mm_shufflelo_epi16(__A, __B) ((__m128i)__builtin_ia32_pshuflw128 ((__v8hi)__A, __B))
1948 #define _mm_shuffle_epi32(__A, __B) ((__m128i)__builtin_ia32_pshufd ((__v4si)__A, __B))
1950 static __inline
void
1951 _mm_maskmoveu_si128 (__m128i __A
, __m128i __B
, char *__C
)
1953 __builtin_ia32_maskmovdqu ((__v16qi
)__A
, (__v16qi
)__B
, __C
);
1956 static __inline __m128i
1957 _mm_avg_epu8 (__m128i __A
, __m128i __B
)
1959 return (__m128i
)__builtin_ia32_pavgb128 ((__v16qi
)__A
, (__v16qi
)__B
);
1962 static __inline __m128i
1963 _mm_avg_epu16 (__m128i __A
, __m128i __B
)
1965 return (__m128i
)__builtin_ia32_pavgw128 ((__v8hi
)__A
, (__v8hi
)__B
);
1968 static __inline __m128i
1969 _mm_sad_epu8 (__m128i __A
, __m128i __B
)
1971 return (__m128i
)__builtin_ia32_psadbw128 ((__v16qi
)__A
, (__v16qi
)__B
);
1974 static __inline
void
1975 _mm_stream_si32 (int *__A
, int __B
)
1977 __builtin_ia32_movnti (__A
, __B
);
1980 static __inline
void
1981 _mm_stream_si128 (__m128i
*__A
, __m128i __B
)
1983 __builtin_ia32_movntdq ((__v2di
*)__A
, (__v2di
)__B
);
1986 static __inline
void
1987 _mm_stream_pd (__m128d
*__A
, __m128d __B
)
1989 __builtin_ia32_movntpd (__A
, (__v2df
)__B
);
1992 static __inline __m128i
1993 _mm_movpi64_epi64 (__m64 __A
)
1995 return (__m128i
)__builtin_ia32_movq2dq ((unsigned long long)__A
);
1998 static __inline
void
1999 _mm_clflush (void *__A
)
2001 return __builtin_ia32_clflush (__A
);
2004 static __inline
void
2007 __builtin_ia32_lfence ();
2010 static __inline
void
2013 __builtin_ia32_mfence ();
2016 #endif /* __SSE2_BUILTINS__ */
2018 #endif /* _XMMINTRIN_H_INCLUDED */