1 /* Copyright (C) 2002, 2003, 2004, 2005, 2006
2 Free Software Foundation, Inc.
4 This file is part of GCC.
6 GCC is free software; you can redistribute it and/or modify
7 it under the terms of the GNU General Public License as published by
8 the Free Software Foundation; either version 2, or (at your option)
11 GCC is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 GNU General Public License for more details.
16 You should have received a copy of the GNU General Public License
17 along with GCC; see the file COPYING. If not, write to
18 the Free Software Foundation, 51 Franklin Street, Fifth Floor,
19 Boston, MA 02110-1301, USA. */
21 /* As a special exception, if you include this header file into source
22 files compiled by GCC, this header file does not by itself cause
23 the resulting executable to be covered by the GNU General Public
24 License. This exception does not however invalidate any other
25 reasons why the executable file might be covered by the GNU General
28 /* Implemented from the specification included in the Intel C++ Compiler
29 User Guide and Reference, version 9.0. */
31 #ifndef _XMMINTRIN_H_INCLUDED
32 #define _XMMINTRIN_H_INCLUDED
35 # error "SSE instruction set not enabled"
38 /* We need type definitions from the MMX header file. */
41 /* Get _mm_malloc () and _mm_free (). */
42 #include <mm_malloc.h>
44 /* The data type intended for user use. */
45 typedef float __m128
__attribute__ ((__vector_size__ (16)));
47 /* Internal data types for implementing the intrinsics. */
48 typedef float __v4sf
__attribute__ ((__vector_size__ (16)));
50 /* Create a selector for use with the SHUFPS instruction. */
51 #define _MM_SHUFFLE(fp3,fp2,fp1,fp0) \
52 (((fp3) << 6) | ((fp2) << 4) | ((fp1) << 2) | (fp0))
54 /* Constants for use with _mm_prefetch. */
63 /* Bits in the MXCSR. */
64 #define _MM_EXCEPT_MASK 0x003f
65 #define _MM_EXCEPT_INVALID 0x0001
66 #define _MM_EXCEPT_DENORM 0x0002
67 #define _MM_EXCEPT_DIV_ZERO 0x0004
68 #define _MM_EXCEPT_OVERFLOW 0x0008
69 #define _MM_EXCEPT_UNDERFLOW 0x0010
70 #define _MM_EXCEPT_INEXACT 0x0020
72 #define _MM_MASK_MASK 0x1f80
73 #define _MM_MASK_INVALID 0x0080
74 #define _MM_MASK_DENORM 0x0100
75 #define _MM_MASK_DIV_ZERO 0x0200
76 #define _MM_MASK_OVERFLOW 0x0400
77 #define _MM_MASK_UNDERFLOW 0x0800
78 #define _MM_MASK_INEXACT 0x1000
80 #define _MM_ROUND_MASK 0x6000
81 #define _MM_ROUND_NEAREST 0x0000
82 #define _MM_ROUND_DOWN 0x2000
83 #define _MM_ROUND_UP 0x4000
84 #define _MM_ROUND_TOWARD_ZERO 0x6000
86 #define _MM_FLUSH_ZERO_MASK 0x8000
87 #define _MM_FLUSH_ZERO_ON 0x8000
88 #define _MM_FLUSH_ZERO_OFF 0x0000
90 /* Create a vector of zeros. */
91 static __inline __m128
__attribute__((__always_inline__
))
94 return __extension__ (__m128
){ 0.0f
, 0.0f
, 0.0f
, 0.0f
};
97 /* Perform the respective operation on the lower SPFP (single-precision
98 floating-point) values of A and B; the upper three SPFP values are
99 passed through from A. */
101 static __inline __m128
__attribute__((__always_inline__
))
102 _mm_add_ss (__m128 __A
, __m128 __B
)
104 return (__m128
) __builtin_ia32_addss ((__v4sf
)__A
, (__v4sf
)__B
);
107 static __inline __m128
__attribute__((__always_inline__
))
108 _mm_sub_ss (__m128 __A
, __m128 __B
)
110 return (__m128
) __builtin_ia32_subss ((__v4sf
)__A
, (__v4sf
)__B
);
113 static __inline __m128
__attribute__((__always_inline__
))
114 _mm_mul_ss (__m128 __A
, __m128 __B
)
116 return (__m128
) __builtin_ia32_mulss ((__v4sf
)__A
, (__v4sf
)__B
);
119 static __inline __m128
__attribute__((__always_inline__
))
120 _mm_div_ss (__m128 __A
, __m128 __B
)
122 return (__m128
) __builtin_ia32_divss ((__v4sf
)__A
, (__v4sf
)__B
);
125 static __inline __m128
__attribute__((__always_inline__
))
126 _mm_sqrt_ss (__m128 __A
)
128 return (__m128
) __builtin_ia32_sqrtss ((__v4sf
)__A
);
131 static __inline __m128
__attribute__((__always_inline__
))
132 _mm_rcp_ss (__m128 __A
)
134 return (__m128
) __builtin_ia32_rcpss ((__v4sf
)__A
);
137 static __inline __m128
__attribute__((__always_inline__
))
138 _mm_rsqrt_ss (__m128 __A
)
140 return (__m128
) __builtin_ia32_rsqrtss ((__v4sf
)__A
);
143 static __inline __m128
__attribute__((__always_inline__
))
144 _mm_min_ss (__m128 __A
, __m128 __B
)
146 return (__m128
) __builtin_ia32_minss ((__v4sf
)__A
, (__v4sf
)__B
);
149 static __inline __m128
__attribute__((__always_inline__
))
150 _mm_max_ss (__m128 __A
, __m128 __B
)
152 return (__m128
) __builtin_ia32_maxss ((__v4sf
)__A
, (__v4sf
)__B
);
155 /* Perform the respective operation on the four SPFP values in A and B. */
157 static __inline __m128
__attribute__((__always_inline__
))
158 _mm_add_ps (__m128 __A
, __m128 __B
)
160 return (__m128
) __builtin_ia32_addps ((__v4sf
)__A
, (__v4sf
)__B
);
163 static __inline __m128
__attribute__((__always_inline__
))
164 _mm_sub_ps (__m128 __A
, __m128 __B
)
166 return (__m128
) __builtin_ia32_subps ((__v4sf
)__A
, (__v4sf
)__B
);
169 static __inline __m128
__attribute__((__always_inline__
))
170 _mm_mul_ps (__m128 __A
, __m128 __B
)
172 return (__m128
) __builtin_ia32_mulps ((__v4sf
)__A
, (__v4sf
)__B
);
175 static __inline __m128
__attribute__((__always_inline__
))
176 _mm_div_ps (__m128 __A
, __m128 __B
)
178 return (__m128
) __builtin_ia32_divps ((__v4sf
)__A
, (__v4sf
)__B
);
181 static __inline __m128
__attribute__((__always_inline__
))
182 _mm_sqrt_ps (__m128 __A
)
184 return (__m128
) __builtin_ia32_sqrtps ((__v4sf
)__A
);
187 static __inline __m128
__attribute__((__always_inline__
))
188 _mm_rcp_ps (__m128 __A
)
190 return (__m128
) __builtin_ia32_rcpps ((__v4sf
)__A
);
193 static __inline __m128
__attribute__((__always_inline__
))
194 _mm_rsqrt_ps (__m128 __A
)
196 return (__m128
) __builtin_ia32_rsqrtps ((__v4sf
)__A
);
199 static __inline __m128
__attribute__((__always_inline__
))
200 _mm_min_ps (__m128 __A
, __m128 __B
)
202 return (__m128
) __builtin_ia32_minps ((__v4sf
)__A
, (__v4sf
)__B
);
205 static __inline __m128
__attribute__((__always_inline__
))
206 _mm_max_ps (__m128 __A
, __m128 __B
)
208 return (__m128
) __builtin_ia32_maxps ((__v4sf
)__A
, (__v4sf
)__B
);
211 /* Perform logical bit-wise operations on 128-bit values. */
213 static __inline __m128
__attribute__((__always_inline__
))
214 _mm_and_ps (__m128 __A
, __m128 __B
)
216 return __builtin_ia32_andps (__A
, __B
);
219 static __inline __m128
__attribute__((__always_inline__
))
220 _mm_andnot_ps (__m128 __A
, __m128 __B
)
222 return __builtin_ia32_andnps (__A
, __B
);
225 static __inline __m128
__attribute__((__always_inline__
))
226 _mm_or_ps (__m128 __A
, __m128 __B
)
228 return __builtin_ia32_orps (__A
, __B
);
231 static __inline __m128
__attribute__((__always_inline__
))
232 _mm_xor_ps (__m128 __A
, __m128 __B
)
234 return __builtin_ia32_xorps (__A
, __B
);
237 /* Perform a comparison on the lower SPFP values of A and B. If the
238 comparison is true, place a mask of all ones in the result, otherwise a
239 mask of zeros. The upper three SPFP values are passed through from A. */
241 static __inline __m128
__attribute__((__always_inline__
))
242 _mm_cmpeq_ss (__m128 __A
, __m128 __B
)
244 return (__m128
) __builtin_ia32_cmpeqss ((__v4sf
)__A
, (__v4sf
)__B
);
247 static __inline __m128
__attribute__((__always_inline__
))
248 _mm_cmplt_ss (__m128 __A
, __m128 __B
)
250 return (__m128
) __builtin_ia32_cmpltss ((__v4sf
)__A
, (__v4sf
)__B
);
253 static __inline __m128
__attribute__((__always_inline__
))
254 _mm_cmple_ss (__m128 __A
, __m128 __B
)
256 return (__m128
) __builtin_ia32_cmpless ((__v4sf
)__A
, (__v4sf
)__B
);
259 static __inline __m128
__attribute__((__always_inline__
))
260 _mm_cmpgt_ss (__m128 __A
, __m128 __B
)
262 return (__m128
) __builtin_ia32_movss ((__v4sf
) __A
,
264 __builtin_ia32_cmpltss ((__v4sf
) __B
,
269 static __inline __m128
__attribute__((__always_inline__
))
270 _mm_cmpge_ss (__m128 __A
, __m128 __B
)
272 return (__m128
) __builtin_ia32_movss ((__v4sf
) __A
,
274 __builtin_ia32_cmpless ((__v4sf
) __B
,
279 static __inline __m128
__attribute__((__always_inline__
))
280 _mm_cmpneq_ss (__m128 __A
, __m128 __B
)
282 return (__m128
) __builtin_ia32_cmpneqss ((__v4sf
)__A
, (__v4sf
)__B
);
285 static __inline __m128
__attribute__((__always_inline__
))
286 _mm_cmpnlt_ss (__m128 __A
, __m128 __B
)
288 return (__m128
) __builtin_ia32_cmpnltss ((__v4sf
)__A
, (__v4sf
)__B
);
291 static __inline __m128
__attribute__((__always_inline__
))
292 _mm_cmpnle_ss (__m128 __A
, __m128 __B
)
294 return (__m128
) __builtin_ia32_cmpnless ((__v4sf
)__A
, (__v4sf
)__B
);
297 static __inline __m128
__attribute__((__always_inline__
))
298 _mm_cmpngt_ss (__m128 __A
, __m128 __B
)
300 return (__m128
) __builtin_ia32_movss ((__v4sf
) __A
,
302 __builtin_ia32_cmpnltss ((__v4sf
) __B
,
307 static __inline __m128
__attribute__((__always_inline__
))
308 _mm_cmpnge_ss (__m128 __A
, __m128 __B
)
310 return (__m128
) __builtin_ia32_movss ((__v4sf
) __A
,
312 __builtin_ia32_cmpnless ((__v4sf
) __B
,
317 static __inline __m128
__attribute__((__always_inline__
))
318 _mm_cmpord_ss (__m128 __A
, __m128 __B
)
320 return (__m128
) __builtin_ia32_cmpordss ((__v4sf
)__A
, (__v4sf
)__B
);
323 static __inline __m128
__attribute__((__always_inline__
))
324 _mm_cmpunord_ss (__m128 __A
, __m128 __B
)
326 return (__m128
) __builtin_ia32_cmpunordss ((__v4sf
)__A
, (__v4sf
)__B
);
329 /* Perform a comparison on the four SPFP values of A and B. For each
330 element, if the comparison is true, place a mask of all ones in the
331 result, otherwise a mask of zeros. */
333 static __inline __m128
__attribute__((__always_inline__
))
334 _mm_cmpeq_ps (__m128 __A
, __m128 __B
)
336 return (__m128
) __builtin_ia32_cmpeqps ((__v4sf
)__A
, (__v4sf
)__B
);
339 static __inline __m128
__attribute__((__always_inline__
))
340 _mm_cmplt_ps (__m128 __A
, __m128 __B
)
342 return (__m128
) __builtin_ia32_cmpltps ((__v4sf
)__A
, (__v4sf
)__B
);
345 static __inline __m128
__attribute__((__always_inline__
))
346 _mm_cmple_ps (__m128 __A
, __m128 __B
)
348 return (__m128
) __builtin_ia32_cmpleps ((__v4sf
)__A
, (__v4sf
)__B
);
351 static __inline __m128
__attribute__((__always_inline__
))
352 _mm_cmpgt_ps (__m128 __A
, __m128 __B
)
354 return (__m128
) __builtin_ia32_cmpgtps ((__v4sf
)__A
, (__v4sf
)__B
);
357 static __inline __m128
__attribute__((__always_inline__
))
358 _mm_cmpge_ps (__m128 __A
, __m128 __B
)
360 return (__m128
) __builtin_ia32_cmpgeps ((__v4sf
)__A
, (__v4sf
)__B
);
363 static __inline __m128
__attribute__((__always_inline__
))
364 _mm_cmpneq_ps (__m128 __A
, __m128 __B
)
366 return (__m128
) __builtin_ia32_cmpneqps ((__v4sf
)__A
, (__v4sf
)__B
);
369 static __inline __m128
__attribute__((__always_inline__
))
370 _mm_cmpnlt_ps (__m128 __A
, __m128 __B
)
372 return (__m128
) __builtin_ia32_cmpnltps ((__v4sf
)__A
, (__v4sf
)__B
);
375 static __inline __m128
__attribute__((__always_inline__
))
376 _mm_cmpnle_ps (__m128 __A
, __m128 __B
)
378 return (__m128
) __builtin_ia32_cmpnleps ((__v4sf
)__A
, (__v4sf
)__B
);
381 static __inline __m128
__attribute__((__always_inline__
))
382 _mm_cmpngt_ps (__m128 __A
, __m128 __B
)
384 return (__m128
) __builtin_ia32_cmpngtps ((__v4sf
)__A
, (__v4sf
)__B
);
387 static __inline __m128
__attribute__((__always_inline__
))
388 _mm_cmpnge_ps (__m128 __A
, __m128 __B
)
390 return (__m128
) __builtin_ia32_cmpngeps ((__v4sf
)__A
, (__v4sf
)__B
);
393 static __inline __m128
__attribute__((__always_inline__
))
394 _mm_cmpord_ps (__m128 __A
, __m128 __B
)
396 return (__m128
) __builtin_ia32_cmpordps ((__v4sf
)__A
, (__v4sf
)__B
);
399 static __inline __m128
__attribute__((__always_inline__
))
400 _mm_cmpunord_ps (__m128 __A
, __m128 __B
)
402 return (__m128
) __builtin_ia32_cmpunordps ((__v4sf
)__A
, (__v4sf
)__B
);
405 /* Compare the lower SPFP values of A and B and return 1 if true
408 static __inline
int __attribute__((__always_inline__
))
409 _mm_comieq_ss (__m128 __A
, __m128 __B
)
411 return __builtin_ia32_comieq ((__v4sf
)__A
, (__v4sf
)__B
);
414 static __inline
int __attribute__((__always_inline__
))
415 _mm_comilt_ss (__m128 __A
, __m128 __B
)
417 return __builtin_ia32_comilt ((__v4sf
)__A
, (__v4sf
)__B
);
420 static __inline
int __attribute__((__always_inline__
))
421 _mm_comile_ss (__m128 __A
, __m128 __B
)
423 return __builtin_ia32_comile ((__v4sf
)__A
, (__v4sf
)__B
);
426 static __inline
int __attribute__((__always_inline__
))
427 _mm_comigt_ss (__m128 __A
, __m128 __B
)
429 return __builtin_ia32_comigt ((__v4sf
)__A
, (__v4sf
)__B
);
432 static __inline
int __attribute__((__always_inline__
))
433 _mm_comige_ss (__m128 __A
, __m128 __B
)
435 return __builtin_ia32_comige ((__v4sf
)__A
, (__v4sf
)__B
);
438 static __inline
int __attribute__((__always_inline__
))
439 _mm_comineq_ss (__m128 __A
, __m128 __B
)
441 return __builtin_ia32_comineq ((__v4sf
)__A
, (__v4sf
)__B
);
444 static __inline
int __attribute__((__always_inline__
))
445 _mm_ucomieq_ss (__m128 __A
, __m128 __B
)
447 return __builtin_ia32_ucomieq ((__v4sf
)__A
, (__v4sf
)__B
);
450 static __inline
int __attribute__((__always_inline__
))
451 _mm_ucomilt_ss (__m128 __A
, __m128 __B
)
453 return __builtin_ia32_ucomilt ((__v4sf
)__A
, (__v4sf
)__B
);
456 static __inline
int __attribute__((__always_inline__
))
457 _mm_ucomile_ss (__m128 __A
, __m128 __B
)
459 return __builtin_ia32_ucomile ((__v4sf
)__A
, (__v4sf
)__B
);
462 static __inline
int __attribute__((__always_inline__
))
463 _mm_ucomigt_ss (__m128 __A
, __m128 __B
)
465 return __builtin_ia32_ucomigt ((__v4sf
)__A
, (__v4sf
)__B
);
468 static __inline
int __attribute__((__always_inline__
))
469 _mm_ucomige_ss (__m128 __A
, __m128 __B
)
471 return __builtin_ia32_ucomige ((__v4sf
)__A
, (__v4sf
)__B
);
474 static __inline
int __attribute__((__always_inline__
))
475 _mm_ucomineq_ss (__m128 __A
, __m128 __B
)
477 return __builtin_ia32_ucomineq ((__v4sf
)__A
, (__v4sf
)__B
);
480 /* Convert the lower SPFP value to a 32-bit integer according to the current
482 static __inline
int __attribute__((__always_inline__
))
483 _mm_cvtss_si32 (__m128 __A
)
485 return __builtin_ia32_cvtss2si ((__v4sf
) __A
);
488 static __inline
int __attribute__((__always_inline__
))
489 _mm_cvt_ss2si (__m128 __A
)
491 return _mm_cvtss_si32 (__A
);
495 /* Convert the lower SPFP value to a 32-bit integer according to the
496 current rounding mode. */
498 /* Intel intrinsic. */
499 static __inline
long long __attribute__((__always_inline__
))
500 _mm_cvtss_si64 (__m128 __A
)
502 return __builtin_ia32_cvtss2si64 ((__v4sf
) __A
);
505 /* Microsoft intrinsic. */
506 static __inline
long long __attribute__((__always_inline__
))
507 _mm_cvtss_si64x (__m128 __A
)
509 return __builtin_ia32_cvtss2si64 ((__v4sf
) __A
);
513 /* Convert the two lower SPFP values to 32-bit integers according to the
514 current rounding mode. Return the integers in packed form. */
515 static __inline __m64
__attribute__((__always_inline__
))
516 _mm_cvtps_pi32 (__m128 __A
)
518 return (__m64
) __builtin_ia32_cvtps2pi ((__v4sf
) __A
);
521 static __inline __m64
__attribute__((__always_inline__
))
522 _mm_cvt_ps2pi (__m128 __A
)
524 return _mm_cvtps_pi32 (__A
);
527 /* Truncate the lower SPFP value to a 32-bit integer. */
528 static __inline
int __attribute__((__always_inline__
))
529 _mm_cvttss_si32 (__m128 __A
)
531 return __builtin_ia32_cvttss2si ((__v4sf
) __A
);
534 static __inline
int __attribute__((__always_inline__
))
535 _mm_cvtt_ss2si (__m128 __A
)
537 return _mm_cvttss_si32 (__A
);
541 /* Truncate the lower SPFP value to a 32-bit integer. */
543 /* Intel intrinsic. */
544 static __inline
long long __attribute__((__always_inline__
))
545 _mm_cvttss_si64 (__m128 __A
)
547 return __builtin_ia32_cvttss2si64 ((__v4sf
) __A
);
550 /* Microsoft intrinsic. */
551 static __inline
long long __attribute__((__always_inline__
))
552 _mm_cvttss_si64x (__m128 __A
)
554 return __builtin_ia32_cvttss2si64 ((__v4sf
) __A
);
558 /* Truncate the two lower SPFP values to 32-bit integers. Return the
559 integers in packed form. */
560 static __inline __m64
__attribute__((__always_inline__
))
561 _mm_cvttps_pi32 (__m128 __A
)
563 return (__m64
) __builtin_ia32_cvttps2pi ((__v4sf
) __A
);
566 static __inline __m64
__attribute__((__always_inline__
))
567 _mm_cvtt_ps2pi (__m128 __A
)
569 return _mm_cvttps_pi32 (__A
);
572 /* Convert B to a SPFP value and insert it as element zero in A. */
573 static __inline __m128
__attribute__((__always_inline__
))
574 _mm_cvtsi32_ss (__m128 __A
, int __B
)
576 return (__m128
) __builtin_ia32_cvtsi2ss ((__v4sf
) __A
, __B
);
579 static __inline __m128
__attribute__((__always_inline__
))
580 _mm_cvt_si2ss (__m128 __A
, int __B
)
582 return _mm_cvtsi32_ss (__A
, __B
);
586 /* Convert B to a SPFP value and insert it as element zero in A. */
588 /* Intel intrinsic. */
589 static __inline __m128
__attribute__((__always_inline__
))
590 _mm_cvtsi64_ss (__m128 __A
, long long __B
)
592 return (__m128
) __builtin_ia32_cvtsi642ss ((__v4sf
) __A
, __B
);
595 /* Microsoft intrinsic. */
596 static __inline __m128
__attribute__((__always_inline__
))
597 _mm_cvtsi64x_ss (__m128 __A
, long long __B
)
599 return (__m128
) __builtin_ia32_cvtsi642ss ((__v4sf
) __A
, __B
);
603 /* Convert the two 32-bit values in B to SPFP form and insert them
604 as the two lower elements in A. */
605 static __inline __m128
__attribute__((__always_inline__
))
606 _mm_cvtpi32_ps (__m128 __A
, __m64 __B
)
608 return (__m128
) __builtin_ia32_cvtpi2ps ((__v4sf
) __A
, (__v2si
)__B
);
611 static __inline __m128
__attribute__((__always_inline__
))
612 _mm_cvt_pi2ps (__m128 __A
, __m64 __B
)
614 return _mm_cvtpi32_ps (__A
, __B
);
617 /* Convert the four signed 16-bit values in A to SPFP form. */
618 static __inline __m128
__attribute__((__always_inline__
))
619 _mm_cvtpi16_ps (__m64 __A
)
622 __v2si __hisi
, __losi
;
625 /* This comparison against zero gives us a mask that can be used to
626 fill in the missing sign bits in the unpack operations below, so
627 that we get signed values after unpacking. */
628 __sign
= __builtin_ia32_pcmpgtw ((__v4hi
)0LL, (__v4hi
)__A
);
630 /* Convert the four words to doublewords. */
631 __hisi
= (__v2si
) __builtin_ia32_punpckhwd ((__v4hi
)__A
, __sign
);
632 __losi
= (__v2si
) __builtin_ia32_punpcklwd ((__v4hi
)__A
, __sign
);
634 /* Convert the doublewords to floating point two at a time. */
635 __r
= (__v4sf
) _mm_setzero_ps ();
636 __r
= __builtin_ia32_cvtpi2ps (__r
, __hisi
);
637 __r
= __builtin_ia32_movlhps (__r
, __r
);
638 __r
= __builtin_ia32_cvtpi2ps (__r
, __losi
);
643 /* Convert the four unsigned 16-bit values in A to SPFP form. */
644 static __inline __m128
__attribute__((__always_inline__
))
645 _mm_cvtpu16_ps (__m64 __A
)
647 __v2si __hisi
, __losi
;
650 /* Convert the four words to doublewords. */
651 __hisi
= (__v2si
) __builtin_ia32_punpckhwd ((__v4hi
)__A
, (__v4hi
)0LL);
652 __losi
= (__v2si
) __builtin_ia32_punpcklwd ((__v4hi
)__A
, (__v4hi
)0LL);
654 /* Convert the doublewords to floating point two at a time. */
655 __r
= (__v4sf
) _mm_setzero_ps ();
656 __r
= __builtin_ia32_cvtpi2ps (__r
, __hisi
);
657 __r
= __builtin_ia32_movlhps (__r
, __r
);
658 __r
= __builtin_ia32_cvtpi2ps (__r
, __losi
);
663 /* Convert the low four signed 8-bit values in A to SPFP form. */
664 static __inline __m128
__attribute__((__always_inline__
))
665 _mm_cvtpi8_ps (__m64 __A
)
669 /* This comparison against zero gives us a mask that can be used to
670 fill in the missing sign bits in the unpack operations below, so
671 that we get signed values after unpacking. */
672 __sign
= __builtin_ia32_pcmpgtb ((__v8qi
)0LL, (__v8qi
)__A
);
674 /* Convert the four low bytes to words. */
675 __A
= (__m64
) __builtin_ia32_punpcklbw ((__v8qi
)__A
, __sign
);
677 return _mm_cvtpi16_ps(__A
);
680 /* Convert the low four unsigned 8-bit values in A to SPFP form. */
681 static __inline __m128
__attribute__((__always_inline__
))
682 _mm_cvtpu8_ps(__m64 __A
)
684 __A
= (__m64
) __builtin_ia32_punpcklbw ((__v8qi
)__A
, (__v8qi
)0LL);
685 return _mm_cvtpu16_ps(__A
);
688 /* Convert the four signed 32-bit values in A and B to SPFP form. */
689 static __inline __m128
__attribute__((__always_inline__
))
690 _mm_cvtpi32x2_ps(__m64 __A
, __m64 __B
)
692 __v4sf __zero
= (__v4sf
) _mm_setzero_ps ();
693 __v4sf __sfa
= __builtin_ia32_cvtpi2ps (__zero
, (__v2si
)__A
);
694 __v4sf __sfb
= __builtin_ia32_cvtpi2ps (__zero
, (__v2si
)__B
);
695 return (__m128
) __builtin_ia32_movlhps (__sfa
, __sfb
);
698 /* Convert the four SPFP values in A to four signed 16-bit integers. */
699 static __inline __m64
__attribute__((__always_inline__
))
700 _mm_cvtps_pi16(__m128 __A
)
702 __v4sf __hisf
= (__v4sf
)__A
;
703 __v4sf __losf
= __builtin_ia32_movhlps (__hisf
, __hisf
);
704 __v2si __hisi
= __builtin_ia32_cvtps2pi (__hisf
);
705 __v2si __losi
= __builtin_ia32_cvtps2pi (__losf
);
706 return (__m64
) __builtin_ia32_packssdw (__hisi
, __losi
);
709 /* Convert the four SPFP values in A to four signed 8-bit integers. */
710 static __inline __m64
__attribute__((__always_inline__
))
711 _mm_cvtps_pi8(__m128 __A
)
713 __v4hi __tmp
= (__v4hi
) _mm_cvtps_pi16 (__A
);
714 return (__m64
) __builtin_ia32_packsswb (__tmp
, (__v4hi
)0LL);
717 /* Selects four specific SPFP values from A and B based on MASK. */
719 static __inline __m128
__attribute__((__always_inline__
))
720 _mm_shuffle_ps (__m128 __A
, __m128 __B
, int __mask
)
722 return (__m128
) __builtin_ia32_shufps ((__v4sf
)__A
, (__v4sf
)__B
, __mask
);
725 #define _mm_shuffle_ps(A, B, MASK) \
726 ((__m128) __builtin_ia32_shufps ((__v4sf)(A), (__v4sf)(B), (MASK)))
730 /* Selects and interleaves the upper two SPFP values from A and B. */
731 static __inline __m128
__attribute__((__always_inline__
))
732 _mm_unpackhi_ps (__m128 __A
, __m128 __B
)
734 return (__m128
) __builtin_ia32_unpckhps ((__v4sf
)__A
, (__v4sf
)__B
);
737 /* Selects and interleaves the lower two SPFP values from A and B. */
738 static __inline __m128
__attribute__((__always_inline__
))
739 _mm_unpacklo_ps (__m128 __A
, __m128 __B
)
741 return (__m128
) __builtin_ia32_unpcklps ((__v4sf
)__A
, (__v4sf
)__B
);
744 /* Sets the upper two SPFP values with 64-bits of data loaded from P;
745 the lower two values are passed through from A. */
746 static __inline __m128
__attribute__((__always_inline__
))
747 _mm_loadh_pi (__m128 __A
, __m64
const *__P
)
749 return (__m128
) __builtin_ia32_loadhps ((__v4sf
)__A
, (__v2si
*)__P
);
752 /* Stores the upper two SPFP values of A into P. */
753 static __inline
void __attribute__((__always_inline__
))
754 _mm_storeh_pi (__m64
*__P
, __m128 __A
)
756 __builtin_ia32_storehps ((__v2si
*)__P
, (__v4sf
)__A
);
759 /* Moves the upper two values of B into the lower two values of A. */
760 static __inline __m128
__attribute__((__always_inline__
))
761 _mm_movehl_ps (__m128 __A
, __m128 __B
)
763 return (__m128
) __builtin_ia32_movhlps ((__v4sf
)__A
, (__v4sf
)__B
);
766 /* Moves the lower two values of B into the upper two values of A. */
767 static __inline __m128
__attribute__((__always_inline__
))
768 _mm_movelh_ps (__m128 __A
, __m128 __B
)
770 return (__m128
) __builtin_ia32_movlhps ((__v4sf
)__A
, (__v4sf
)__B
);
773 /* Sets the lower two SPFP values with 64-bits of data loaded from P;
774 the upper two values are passed through from A. */
775 static __inline __m128
__attribute__((__always_inline__
))
776 _mm_loadl_pi (__m128 __A
, __m64
const *__P
)
778 return (__m128
) __builtin_ia32_loadlps ((__v4sf
)__A
, (__v2si
*)__P
);
781 /* Stores the lower two SPFP values of A into P. */
782 static __inline
void __attribute__((__always_inline__
))
783 _mm_storel_pi (__m64
*__P
, __m128 __A
)
785 __builtin_ia32_storelps ((__v2si
*)__P
, (__v4sf
)__A
);
788 /* Creates a 4-bit mask from the most significant bits of the SPFP values. */
789 static __inline
int __attribute__((__always_inline__
))
790 _mm_movemask_ps (__m128 __A
)
792 return __builtin_ia32_movmskps ((__v4sf
)__A
);
795 /* Return the contents of the control register. */
796 static __inline
unsigned int __attribute__((__always_inline__
))
799 return __builtin_ia32_stmxcsr ();
802 /* Read exception bits from the control register. */
803 static __inline
unsigned int __attribute__((__always_inline__
))
804 _MM_GET_EXCEPTION_STATE (void)
806 return _mm_getcsr() & _MM_EXCEPT_MASK
;
809 static __inline
unsigned int __attribute__((__always_inline__
))
810 _MM_GET_EXCEPTION_MASK (void)
812 return _mm_getcsr() & _MM_MASK_MASK
;
815 static __inline
unsigned int __attribute__((__always_inline__
))
816 _MM_GET_ROUNDING_MODE (void)
818 return _mm_getcsr() & _MM_ROUND_MASK
;
821 static __inline
unsigned int __attribute__((__always_inline__
))
822 _MM_GET_FLUSH_ZERO_MODE (void)
824 return _mm_getcsr() & _MM_FLUSH_ZERO_MASK
;
827 /* Set the control register to I. */
828 static __inline
void __attribute__((__always_inline__
))
829 _mm_setcsr (unsigned int __I
)
831 __builtin_ia32_ldmxcsr (__I
);
834 /* Set exception bits in the control register. */
835 static __inline
void __attribute__((__always_inline__
))
836 _MM_SET_EXCEPTION_STATE(unsigned int __mask
)
838 _mm_setcsr((_mm_getcsr() & ~_MM_EXCEPT_MASK
) | __mask
);
841 static __inline
void __attribute__((__always_inline__
))
842 _MM_SET_EXCEPTION_MASK (unsigned int __mask
)
844 _mm_setcsr((_mm_getcsr() & ~_MM_MASK_MASK
) | __mask
);
847 static __inline
void __attribute__((__always_inline__
))
848 _MM_SET_ROUNDING_MODE (unsigned int __mode
)
850 _mm_setcsr((_mm_getcsr() & ~_MM_ROUND_MASK
) | __mode
);
853 static __inline
void __attribute__((__always_inline__
))
854 _MM_SET_FLUSH_ZERO_MODE (unsigned int __mode
)
856 _mm_setcsr((_mm_getcsr() & ~_MM_FLUSH_ZERO_MASK
) | __mode
);
859 /* Create a vector with element 0 as F and the rest zero. */
860 static __inline __m128
__attribute__((__always_inline__
))
861 _mm_set_ss (float __F
)
863 return __extension__ (__m128
)(__v4sf
){ __F
, 0, 0, 0 };
866 /* Create a vector with all four elements equal to F. */
867 static __inline __m128
__attribute__((__always_inline__
))
868 _mm_set1_ps (float __F
)
870 return __extension__ (__m128
)(__v4sf
){ __F
, __F
, __F
, __F
};
873 static __inline __m128
__attribute__((__always_inline__
))
874 _mm_set_ps1 (float __F
)
876 return _mm_set1_ps (__F
);
879 /* Create a vector with element 0 as *P and the rest zero. */
880 static __inline __m128
__attribute__((__always_inline__
))
881 _mm_load_ss (float const *__P
)
883 return _mm_set_ss (*__P
);
886 /* Create a vector with all four elements equal to *P. */
887 static __inline __m128
__attribute__((__always_inline__
))
888 _mm_load1_ps (float const *__P
)
890 return _mm_set1_ps (*__P
);
893 static __inline __m128
__attribute__((__always_inline__
))
894 _mm_load_ps1 (float const *__P
)
896 return _mm_load1_ps (__P
);
899 /* Load four SPFP values from P. The address must be 16-byte aligned. */
900 static __inline __m128
__attribute__((__always_inline__
))
901 _mm_load_ps (float const *__P
)
903 return (__m128
) *(__v4sf
*)__P
;
906 /* Load four SPFP values from P. The address need not be 16-byte aligned. */
907 static __inline __m128
__attribute__((__always_inline__
))
908 _mm_loadu_ps (float const *__P
)
910 return (__m128
) __builtin_ia32_loadups (__P
);
913 /* Load four SPFP values in reverse order. The address must be aligned. */
914 static __inline __m128
__attribute__((__always_inline__
))
915 _mm_loadr_ps (float const *__P
)
917 __v4sf __tmp
= *(__v4sf
*)__P
;
918 return (__m128
) __builtin_ia32_shufps (__tmp
, __tmp
, _MM_SHUFFLE (0,1,2,3));
921 /* Create the vector [Z Y X W]. */
922 static __inline __m128
__attribute__((__always_inline__
))
923 _mm_set_ps (const float __Z
, const float __Y
, const float __X
, const float __W
)
925 return __extension__ (__m128
)(__v4sf
){ __W
, __X
, __Y
, __Z
};
928 /* Create the vector [W X Y Z]. */
929 static __inline __m128
__attribute__((__always_inline__
))
930 _mm_setr_ps (float __Z
, float __Y
, float __X
, float __W
)
932 return __extension__ (__m128
)(__v4sf
){ __Z
, __Y
, __X
, __W
};
935 /* Stores the lower SPFP value. */
936 static __inline
void __attribute__((__always_inline__
))
937 _mm_store_ss (float *__P
, __m128 __A
)
939 *__P
= __builtin_ia32_vec_ext_v4sf ((__v4sf
)__A
, 0);
942 static __inline
float __attribute__((__always_inline__
))
943 _mm_cvtss_f32 (__m128 __A
)
945 return __builtin_ia32_vec_ext_v4sf ((__v4sf
)__A
, 0);
948 /* Store four SPFP values. The address must be 16-byte aligned. */
949 static __inline
void __attribute__((__always_inline__
))
950 _mm_store_ps (float *__P
, __m128 __A
)
952 *(__v4sf
*)__P
= (__v4sf
)__A
;
955 /* Store four SPFP values. The address need not be 16-byte aligned. */
956 static __inline
void __attribute__((__always_inline__
))
957 _mm_storeu_ps (float *__P
, __m128 __A
)
959 __builtin_ia32_storeups (__P
, (__v4sf
)__A
);
962 /* Store the lower SPFP value across four words. */
963 static __inline
void __attribute__((__always_inline__
))
964 _mm_store1_ps (float *__P
, __m128 __A
)
966 __v4sf __va
= (__v4sf
)__A
;
967 __v4sf __tmp
= __builtin_ia32_shufps (__va
, __va
, _MM_SHUFFLE (0,0,0,0));
968 _mm_storeu_ps (__P
, __tmp
);
971 static __inline
void __attribute__((__always_inline__
))
972 _mm_store_ps1 (float *__P
, __m128 __A
)
974 _mm_store1_ps (__P
, __A
);
977 /* Store four SPFP values in reverse order. The address must be aligned. */
978 static __inline
void __attribute__((__always_inline__
))
979 _mm_storer_ps (float *__P
, __m128 __A
)
981 __v4sf __va
= (__v4sf
)__A
;
982 __v4sf __tmp
= __builtin_ia32_shufps (__va
, __va
, _MM_SHUFFLE (0,1,2,3));
983 _mm_store_ps (__P
, __tmp
);
986 /* Sets the low SPFP value of A from the low value of B. */
987 static __inline __m128
__attribute__((__always_inline__
))
988 _mm_move_ss (__m128 __A
, __m128 __B
)
990 return (__m128
) __builtin_ia32_movss ((__v4sf
)__A
, (__v4sf
)__B
);
993 /* Extracts one of the four words of A. The selector N must be immediate. */
995 static __inline
int __attribute__((__always_inline__
))
996 _mm_extract_pi16 (__m64
const __A
, int const __N
)
998 return __builtin_ia32_vec_ext_v4hi ((__v4hi
)__A
, __N
);
1001 static __inline
int __attribute__((__always_inline__
))
1002 _m_pextrw (__m64
const __A
, int const __N
)
1004 return _mm_extract_pi16 (__A
, __N
);
1007 #define _mm_extract_pi16(A, N) __builtin_ia32_vec_ext_v4hi ((__v4hi)(A), (N))
1008 #define _m_pextrw(A, N) _mm_extract_pi16((A), (N))
1011 /* Inserts word D into one of four words of A. The selector N must be
1014 static __inline __m64
__attribute__((__always_inline__
))
1015 _mm_insert_pi16 (__m64
const __A
, int const __D
, int const __N
)
1017 return (__m64
) __builtin_ia32_vec_set_v4hi ((__v4hi
)__A
, __D
, __N
);
1020 static __inline __m64
__attribute__((__always_inline__
))
1021 _m_pinsrw (__m64
const __A
, int const __D
, int const __N
)
1023 return _mm_insert_pi16 (__A
, __D
, __N
);
1026 #define _mm_insert_pi16(A, D, N) \
1027 ((__m64) __builtin_ia32_vec_set_v4hi ((__v4hi)(A), (D), (N)))
1028 #define _m_pinsrw(A, D, N) _mm_insert_pi16((A), (D), (N))
1031 /* Compute the element-wise maximum of signed 16-bit values. */
1032 static __inline __m64
__attribute__((__always_inline__
))
1033 _mm_max_pi16 (__m64 __A
, __m64 __B
)
1035 return (__m64
) __builtin_ia32_pmaxsw ((__v4hi
)__A
, (__v4hi
)__B
);
1038 static __inline __m64
__attribute__((__always_inline__
))
1039 _m_pmaxsw (__m64 __A
, __m64 __B
)
1041 return _mm_max_pi16 (__A
, __B
);
1044 /* Compute the element-wise maximum of unsigned 8-bit values. */
1045 static __inline __m64
__attribute__((__always_inline__
))
1046 _mm_max_pu8 (__m64 __A
, __m64 __B
)
1048 return (__m64
) __builtin_ia32_pmaxub ((__v8qi
)__A
, (__v8qi
)__B
);
1051 static __inline __m64
__attribute__((__always_inline__
))
1052 _m_pmaxub (__m64 __A
, __m64 __B
)
1054 return _mm_max_pu8 (__A
, __B
);
1057 /* Compute the element-wise minimum of signed 16-bit values. */
1058 static __inline __m64
__attribute__((__always_inline__
))
1059 _mm_min_pi16 (__m64 __A
, __m64 __B
)
1061 return (__m64
) __builtin_ia32_pminsw ((__v4hi
)__A
, (__v4hi
)__B
);
1064 static __inline __m64
__attribute__((__always_inline__
))
1065 _m_pminsw (__m64 __A
, __m64 __B
)
1067 return _mm_min_pi16 (__A
, __B
);
1070 /* Compute the element-wise minimum of unsigned 8-bit values. */
1071 static __inline __m64
__attribute__((__always_inline__
))
1072 _mm_min_pu8 (__m64 __A
, __m64 __B
)
1074 return (__m64
) __builtin_ia32_pminub ((__v8qi
)__A
, (__v8qi
)__B
);
1077 static __inline __m64
__attribute__((__always_inline__
))
1078 _m_pminub (__m64 __A
, __m64 __B
)
1080 return _mm_min_pu8 (__A
, __B
);
1083 /* Create an 8-bit mask of the signs of 8-bit values. */
1084 static __inline
int __attribute__((__always_inline__
))
1085 _mm_movemask_pi8 (__m64 __A
)
1087 return __builtin_ia32_pmovmskb ((__v8qi
)__A
);
1090 static __inline
int __attribute__((__always_inline__
))
1091 _m_pmovmskb (__m64 __A
)
1093 return _mm_movemask_pi8 (__A
);
1096 /* Multiply four unsigned 16-bit values in A by four unsigned 16-bit values
1097 in B and produce the high 16 bits of the 32-bit results. */
1098 static __inline __m64
__attribute__((__always_inline__
))
1099 _mm_mulhi_pu16 (__m64 __A
, __m64 __B
)
1101 return (__m64
) __builtin_ia32_pmulhuw ((__v4hi
)__A
, (__v4hi
)__B
);
1104 static __inline __m64
__attribute__((__always_inline__
))
1105 _m_pmulhuw (__m64 __A
, __m64 __B
)
1107 return _mm_mulhi_pu16 (__A
, __B
);
1110 /* Return a combination of the four 16-bit values in A. The selector
1111 must be an immediate. */
1114 static __inline __m64
__attribute__((__always_inline__
))
1115 _mm_shuffle_pi16 (__m64 __A
, int __N
)
1117 return (__m64
) __builtin_ia32_pshufw ((__v4hi
)__A
, __N
);
1120 static __inline __m64
__attribute__((__always_inline__
))
1121 _m_pshufw (__m64 __A
, int __N
)
1123 return _mm_shuffle_pi16 (__A
, __N
);
1126 #define _mm_shuffle_pi16(A, N) \
1127 ((__m64) __builtin_ia32_pshufw ((__v4hi)(A), (N)))
1128 #define _m_pshufw(A, N) _mm_shuffle_pi16 ((A), (N))
1132 /* Conditionally store byte elements of A into P. The high bit of each
1133 byte in the selector N determines whether the corresponding byte from
1135 static __inline
void __attribute__((__always_inline__
))
1136 _mm_maskmove_si64 (__m64 __A
, __m64 __N
, char *__P
)
1138 __builtin_ia32_maskmovq ((__v8qi
)__A
, (__v8qi
)__N
, __P
);
1141 static __inline
void __attribute__((__always_inline__
))
1142 _m_maskmovq (__m64 __A
, __m64 __N
, char *__P
)
1144 _mm_maskmove_si64 (__A
, __N
, __P
);
1147 /* Compute the rounded averages of the unsigned 8-bit values in A and B. */
1148 static __inline __m64
__attribute__((__always_inline__
))
1149 _mm_avg_pu8 (__m64 __A
, __m64 __B
)
1151 return (__m64
) __builtin_ia32_pavgb ((__v8qi
)__A
, (__v8qi
)__B
);
1154 static __inline __m64
__attribute__((__always_inline__
))
1155 _m_pavgb (__m64 __A
, __m64 __B
)
1157 return _mm_avg_pu8 (__A
, __B
);
1160 /* Compute the rounded averages of the unsigned 16-bit values in A and B. */
1161 static __inline __m64
__attribute__((__always_inline__
))
1162 _mm_avg_pu16 (__m64 __A
, __m64 __B
)
1164 return (__m64
) __builtin_ia32_pavgw ((__v4hi
)__A
, (__v4hi
)__B
);
1167 static __inline __m64
__attribute__((__always_inline__
))
1168 _m_pavgw (__m64 __A
, __m64 __B
)
1170 return _mm_avg_pu16 (__A
, __B
);
1173 /* Compute the sum of the absolute differences of the unsigned 8-bit
1174 values in A and B. Return the value in the lower 16-bit word; the
1175 upper words are cleared. */
1176 static __inline __m64
__attribute__((__always_inline__
))
1177 _mm_sad_pu8 (__m64 __A
, __m64 __B
)
1179 return (__m64
) __builtin_ia32_psadbw ((__v8qi
)__A
, (__v8qi
)__B
);
1182 static __inline __m64
__attribute__((__always_inline__
))
1183 _m_psadbw (__m64 __A
, __m64 __B
)
1185 return _mm_sad_pu8 (__A
, __B
);
1188 /* Loads one cache line from address P to a location "closer" to the
1189 processor. The selector I specifies the type of prefetch operation. */
1191 static __inline
void __attribute__((__always_inline__
))
1192 _mm_prefetch (void *__P
, enum _mm_hint __I
)
1194 __builtin_prefetch (__P
, 0, __I
);
1197 #define _mm_prefetch(P, I) \
1198 __builtin_prefetch ((P), 0, (I))
1201 /* Stores the data in A to the address P without polluting the caches. */
1202 static __inline
void __attribute__((__always_inline__
))
1203 _mm_stream_pi (__m64
*__P
, __m64 __A
)
1205 __builtin_ia32_movntq ((unsigned long long *)__P
, (unsigned long long)__A
);
1208 /* Likewise. The address must be 16-byte aligned. */
1209 static __inline
void __attribute__((__always_inline__
))
1210 _mm_stream_ps (float *__P
, __m128 __A
)
1212 __builtin_ia32_movntps (__P
, (__v4sf
)__A
);
1215 /* Guarantees that every preceding store is globally visible before
1216 any subsequent store. */
1217 static __inline
void __attribute__((__always_inline__
))
1220 __builtin_ia32_sfence ();
1223 /* The execution of the next instruction is delayed by an implementation
1224 specific amount of time. The instruction does not modify the
1225 architectural state. */
1226 static __inline
void __attribute__((__always_inline__
))
1229 __asm__
__volatile__ ("rep; nop" : : );
1232 /* Transpose the 4x4 matrix composed of row[0-3]. */
1233 #define _MM_TRANSPOSE4_PS(row0, row1, row2, row3) \
1235 __v4sf __r0 = (row0), __r1 = (row1), __r2 = (row2), __r3 = (row3); \
1236 __v4sf __t0 = __builtin_ia32_unpcklps (__r0, __r1); \
1237 __v4sf __t1 = __builtin_ia32_unpcklps (__r2, __r3); \
1238 __v4sf __t2 = __builtin_ia32_unpckhps (__r0, __r1); \
1239 __v4sf __t3 = __builtin_ia32_unpckhps (__r2, __r3); \
1240 (row0) = __builtin_ia32_movlhps (__t0, __t1); \
1241 (row1) = __builtin_ia32_movhlps (__t1, __t0); \
1242 (row2) = __builtin_ia32_movlhps (__t2, __t3); \
1243 (row3) = __builtin_ia32_movhlps (__t3, __t2); \
1246 /* For backward source compatibility. */
1247 #include <emmintrin.h>
1249 #endif /* __SSE__ */
1250 #endif /* _XMMINTRIN_H_INCLUDED */
