1 /*===---- xmmintrin.h - SSE intrinsics -------------------------------------===
3 * Permission is hereby granted, free of charge, to any person obtaining a copy
4 * of this software and associated documentation files (the "Software"), to deal
5 * in the Software without restriction, including without limitation the rights
6 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7 * copies of the Software, and to permit persons to whom the Software is
8 * furnished to do so, subject to the following conditions:
10 * The above copyright notice and this permission notice shall be included in
11 * all copies or substantial portions of the Software.
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21 *===-----------------------------------------------------------------------===
29 typedef int __v4si
__attribute__((__vector_size__(16)));
30 typedef float __v4sf
__attribute__((__vector_size__(16)));
31 typedef float __m128
__attribute__((__vector_size__(16)));
33 /* This header should only be included in a hosted environment as it depends on
34 * a standard library to provide allocation routines. */
36 #include <mm_malloc.h>
39 /* Define the default attributes for the functions in this file. */
40 #define __DEFAULT_FN_ATTRS __attribute__((__always_inline__, __nodebug__, __target__("sse")))
42 static __inline__ __m128 __DEFAULT_FN_ATTRS
43 _mm_add_ss(__m128 __a
, __m128 __b
)
49 static __inline__ __m128 __DEFAULT_FN_ATTRS
50 _mm_add_ps(__m128 __a
, __m128 __b
)
55 static __inline__ __m128 __DEFAULT_FN_ATTRS
56 _mm_sub_ss(__m128 __a
, __m128 __b
)
62 static __inline__ __m128 __DEFAULT_FN_ATTRS
63 _mm_sub_ps(__m128 __a
, __m128 __b
)
68 static __inline__ __m128 __DEFAULT_FN_ATTRS
69 _mm_mul_ss(__m128 __a
, __m128 __b
)
75 static __inline__ __m128 __DEFAULT_FN_ATTRS
76 _mm_mul_ps(__m128 __a
, __m128 __b
)
81 static __inline__ __m128 __DEFAULT_FN_ATTRS
82 _mm_div_ss(__m128 __a
, __m128 __b
)
88 static __inline__ __m128 __DEFAULT_FN_ATTRS
89 _mm_div_ps(__m128 __a
, __m128 __b
)
94 static __inline__ __m128 __DEFAULT_FN_ATTRS
95 _mm_sqrt_ss(__m128 __a
)
97 __m128 __c
= __builtin_ia32_sqrtss(__a
);
98 return (__m128
) { __c
[0], __a
[1], __a
[2], __a
[3] };
101 static __inline__ __m128 __DEFAULT_FN_ATTRS
102 _mm_sqrt_ps(__m128 __a
)
104 return __builtin_ia32_sqrtps(__a
);
107 static __inline__ __m128 __DEFAULT_FN_ATTRS
108 _mm_rcp_ss(__m128 __a
)
110 __m128 __c
= __builtin_ia32_rcpss(__a
);
111 return (__m128
) { __c
[0], __a
[1], __a
[2], __a
[3] };
114 static __inline__ __m128 __DEFAULT_FN_ATTRS
115 _mm_rcp_ps(__m128 __a
)
117 return __builtin_ia32_rcpps(__a
);
120 static __inline__ __m128 __DEFAULT_FN_ATTRS
121 _mm_rsqrt_ss(__m128 __a
)
123 __m128 __c
= __builtin_ia32_rsqrtss(__a
);
124 return (__m128
) { __c
[0], __a
[1], __a
[2], __a
[3] };
127 static __inline__ __m128 __DEFAULT_FN_ATTRS
128 _mm_rsqrt_ps(__m128 __a
)
130 return __builtin_ia32_rsqrtps(__a
);
133 static __inline__ __m128 __DEFAULT_FN_ATTRS
134 _mm_min_ss(__m128 __a
, __m128 __b
)
136 return __builtin_ia32_minss(__a
, __b
);
139 static __inline__ __m128 __DEFAULT_FN_ATTRS
140 _mm_min_ps(__m128 __a
, __m128 __b
)
142 return __builtin_ia32_minps(__a
, __b
);
145 static __inline__ __m128 __DEFAULT_FN_ATTRS
146 _mm_max_ss(__m128 __a
, __m128 __b
)
148 return __builtin_ia32_maxss(__a
, __b
);
151 static __inline__ __m128 __DEFAULT_FN_ATTRS
152 _mm_max_ps(__m128 __a
, __m128 __b
)
154 return __builtin_ia32_maxps(__a
, __b
);
157 static __inline__ __m128 __DEFAULT_FN_ATTRS
158 _mm_and_ps(__m128 __a
, __m128 __b
)
160 return (__m128
)((__v4si
)__a
& (__v4si
)__b
);
163 static __inline__ __m128 __DEFAULT_FN_ATTRS
164 _mm_andnot_ps(__m128 __a
, __m128 __b
)
166 return (__m128
)(~(__v4si
)__a
& (__v4si
)__b
);
169 static __inline__ __m128 __DEFAULT_FN_ATTRS
170 _mm_or_ps(__m128 __a
, __m128 __b
)
172 return (__m128
)((__v4si
)__a
| (__v4si
)__b
);
175 static __inline__ __m128 __DEFAULT_FN_ATTRS
176 _mm_xor_ps(__m128 __a
, __m128 __b
)
178 return (__m128
)((__v4si
)__a
^ (__v4si
)__b
);
181 static __inline__ __m128 __DEFAULT_FN_ATTRS
182 _mm_cmpeq_ss(__m128 __a
, __m128 __b
)
184 return (__m128
)__builtin_ia32_cmpeqss(__a
, __b
);
187 static __inline__ __m128 __DEFAULT_FN_ATTRS
188 _mm_cmpeq_ps(__m128 __a
, __m128 __b
)
190 return (__m128
)__builtin_ia32_cmpeqps(__a
, __b
);
193 static __inline__ __m128 __DEFAULT_FN_ATTRS
194 _mm_cmplt_ss(__m128 __a
, __m128 __b
)
196 return (__m128
)__builtin_ia32_cmpltss(__a
, __b
);
199 static __inline__ __m128 __DEFAULT_FN_ATTRS
200 _mm_cmplt_ps(__m128 __a
, __m128 __b
)
202 return (__m128
)__builtin_ia32_cmpltps(__a
, __b
);
205 static __inline__ __m128 __DEFAULT_FN_ATTRS
206 _mm_cmple_ss(__m128 __a
, __m128 __b
)
208 return (__m128
)__builtin_ia32_cmpless(__a
, __b
);
211 static __inline__ __m128 __DEFAULT_FN_ATTRS
212 _mm_cmple_ps(__m128 __a
, __m128 __b
)
214 return (__m128
)__builtin_ia32_cmpleps(__a
, __b
);
217 static __inline__ __m128 __DEFAULT_FN_ATTRS
218 _mm_cmpgt_ss(__m128 __a
, __m128 __b
)
220 return (__m128
)__builtin_shufflevector(__a
,
221 __builtin_ia32_cmpltss(__b
, __a
),
225 static __inline__ __m128 __DEFAULT_FN_ATTRS
226 _mm_cmpgt_ps(__m128 __a
, __m128 __b
)
228 return (__m128
)__builtin_ia32_cmpltps(__b
, __a
);
231 static __inline__ __m128 __DEFAULT_FN_ATTRS
232 _mm_cmpge_ss(__m128 __a
, __m128 __b
)
234 return (__m128
)__builtin_shufflevector(__a
,
235 __builtin_ia32_cmpless(__b
, __a
),
239 static __inline__ __m128 __DEFAULT_FN_ATTRS
240 _mm_cmpge_ps(__m128 __a
, __m128 __b
)
242 return (__m128
)__builtin_ia32_cmpleps(__b
, __a
);
245 static __inline__ __m128 __DEFAULT_FN_ATTRS
246 _mm_cmpneq_ss(__m128 __a
, __m128 __b
)
248 return (__m128
)__builtin_ia32_cmpneqss(__a
, __b
);
251 static __inline__ __m128 __DEFAULT_FN_ATTRS
252 _mm_cmpneq_ps(__m128 __a
, __m128 __b
)
254 return (__m128
)__builtin_ia32_cmpneqps(__a
, __b
);
257 static __inline__ __m128 __DEFAULT_FN_ATTRS
258 _mm_cmpnlt_ss(__m128 __a
, __m128 __b
)
260 return (__m128
)__builtin_ia32_cmpnltss(__a
, __b
);
263 static __inline__ __m128 __DEFAULT_FN_ATTRS
264 _mm_cmpnlt_ps(__m128 __a
, __m128 __b
)
266 return (__m128
)__builtin_ia32_cmpnltps(__a
, __b
);
269 static __inline__ __m128 __DEFAULT_FN_ATTRS
270 _mm_cmpnle_ss(__m128 __a
, __m128 __b
)
272 return (__m128
)__builtin_ia32_cmpnless(__a
, __b
);
275 static __inline__ __m128 __DEFAULT_FN_ATTRS
276 _mm_cmpnle_ps(__m128 __a
, __m128 __b
)
278 return (__m128
)__builtin_ia32_cmpnleps(__a
, __b
);
281 static __inline__ __m128 __DEFAULT_FN_ATTRS
282 _mm_cmpngt_ss(__m128 __a
, __m128 __b
)
284 return (__m128
)__builtin_shufflevector(__a
,
285 __builtin_ia32_cmpnltss(__b
, __a
),
289 static __inline__ __m128 __DEFAULT_FN_ATTRS
290 _mm_cmpngt_ps(__m128 __a
, __m128 __b
)
292 return (__m128
)__builtin_ia32_cmpnltps(__b
, __a
);
295 static __inline__ __m128 __DEFAULT_FN_ATTRS
296 _mm_cmpnge_ss(__m128 __a
, __m128 __b
)
298 return (__m128
)__builtin_shufflevector(__a
,
299 __builtin_ia32_cmpnless(__b
, __a
),
303 static __inline__ __m128 __DEFAULT_FN_ATTRS
304 _mm_cmpnge_ps(__m128 __a
, __m128 __b
)
306 return (__m128
)__builtin_ia32_cmpnleps(__b
, __a
);
309 static __inline__ __m128 __DEFAULT_FN_ATTRS
310 _mm_cmpord_ss(__m128 __a
, __m128 __b
)
312 return (__m128
)__builtin_ia32_cmpordss(__a
, __b
);
315 static __inline__ __m128 __DEFAULT_FN_ATTRS
316 _mm_cmpord_ps(__m128 __a
, __m128 __b
)
318 return (__m128
)__builtin_ia32_cmpordps(__a
, __b
);
321 static __inline__ __m128 __DEFAULT_FN_ATTRS
322 _mm_cmpunord_ss(__m128 __a
, __m128 __b
)
324 return (__m128
)__builtin_ia32_cmpunordss(__a
, __b
);
327 static __inline__ __m128 __DEFAULT_FN_ATTRS
328 _mm_cmpunord_ps(__m128 __a
, __m128 __b
)
330 return (__m128
)__builtin_ia32_cmpunordps(__a
, __b
);
333 static __inline__
int __DEFAULT_FN_ATTRS
334 _mm_comieq_ss(__m128 __a
, __m128 __b
)
336 return __builtin_ia32_comieq(__a
, __b
);
339 static __inline__
int __DEFAULT_FN_ATTRS
340 _mm_comilt_ss(__m128 __a
, __m128 __b
)
342 return __builtin_ia32_comilt(__a
, __b
);
345 static __inline__
int __DEFAULT_FN_ATTRS
346 _mm_comile_ss(__m128 __a
, __m128 __b
)
348 return __builtin_ia32_comile(__a
, __b
);
351 static __inline__
int __DEFAULT_FN_ATTRS
352 _mm_comigt_ss(__m128 __a
, __m128 __b
)
354 return __builtin_ia32_comigt(__a
, __b
);
357 static __inline__
int __DEFAULT_FN_ATTRS
358 _mm_comige_ss(__m128 __a
, __m128 __b
)
360 return __builtin_ia32_comige(__a
, __b
);
363 static __inline__
int __DEFAULT_FN_ATTRS
364 _mm_comineq_ss(__m128 __a
, __m128 __b
)
366 return __builtin_ia32_comineq(__a
, __b
);
369 static __inline__
int __DEFAULT_FN_ATTRS
370 _mm_ucomieq_ss(__m128 __a
, __m128 __b
)
372 return __builtin_ia32_ucomieq(__a
, __b
);
375 static __inline__
int __DEFAULT_FN_ATTRS
376 _mm_ucomilt_ss(__m128 __a
, __m128 __b
)
378 return __builtin_ia32_ucomilt(__a
, __b
);
381 static __inline__
int __DEFAULT_FN_ATTRS
382 _mm_ucomile_ss(__m128 __a
, __m128 __b
)
384 return __builtin_ia32_ucomile(__a
, __b
);
387 static __inline__
int __DEFAULT_FN_ATTRS
388 _mm_ucomigt_ss(__m128 __a
, __m128 __b
)
390 return __builtin_ia32_ucomigt(__a
, __b
);
393 static __inline__
int __DEFAULT_FN_ATTRS
394 _mm_ucomige_ss(__m128 __a
, __m128 __b
)
396 return __builtin_ia32_ucomige(__a
, __b
);
399 static __inline__
int __DEFAULT_FN_ATTRS
400 _mm_ucomineq_ss(__m128 __a
, __m128 __b
)
402 return __builtin_ia32_ucomineq(__a
, __b
);
405 static __inline__
int __DEFAULT_FN_ATTRS
406 _mm_cvtss_si32(__m128 __a
)
408 return __builtin_ia32_cvtss2si(__a
);
411 static __inline__
int __DEFAULT_FN_ATTRS
412 _mm_cvt_ss2si(__m128 __a
)
414 return _mm_cvtss_si32(__a
);
419 static __inline__
long long __DEFAULT_FN_ATTRS
420 _mm_cvtss_si64(__m128 __a
)
422 return __builtin_ia32_cvtss2si64(__a
);
427 static __inline__ __m64 __DEFAULT_FN_ATTRS
428 _mm_cvtps_pi32(__m128 __a
)
430 return (__m64
)__builtin_ia32_cvtps2pi(__a
);
433 static __inline__ __m64 __DEFAULT_FN_ATTRS
434 _mm_cvt_ps2pi(__m128 __a
)
436 return _mm_cvtps_pi32(__a
);
439 static __inline__
int __DEFAULT_FN_ATTRS
440 _mm_cvttss_si32(__m128 __a
)
445 static __inline__
int __DEFAULT_FN_ATTRS
446 _mm_cvtt_ss2si(__m128 __a
)
448 return _mm_cvttss_si32(__a
);
451 static __inline__
long long __DEFAULT_FN_ATTRS
452 _mm_cvttss_si64(__m128 __a
)
457 static __inline__ __m64 __DEFAULT_FN_ATTRS
458 _mm_cvttps_pi32(__m128 __a
)
460 return (__m64
)__builtin_ia32_cvttps2pi(__a
);
463 static __inline__ __m64 __DEFAULT_FN_ATTRS
464 _mm_cvtt_ps2pi(__m128 __a
)
466 return _mm_cvttps_pi32(__a
);
469 static __inline__ __m128 __DEFAULT_FN_ATTRS
470 _mm_cvtsi32_ss(__m128 __a
, int __b
)
476 static __inline__ __m128 __DEFAULT_FN_ATTRS
477 _mm_cvt_si2ss(__m128 __a
, int __b
)
479 return _mm_cvtsi32_ss(__a
, __b
);
484 static __inline__ __m128 __DEFAULT_FN_ATTRS
485 _mm_cvtsi64_ss(__m128 __a
, long long __b
)
493 static __inline__ __m128 __DEFAULT_FN_ATTRS
494 _mm_cvtpi32_ps(__m128 __a
, __m64 __b
)
496 return __builtin_ia32_cvtpi2ps(__a
, (__v2si
)__b
);
499 static __inline__ __m128 __DEFAULT_FN_ATTRS
500 _mm_cvt_pi2ps(__m128 __a
, __m64 __b
)
502 return _mm_cvtpi32_ps(__a
, __b
);
505 static __inline__
float __DEFAULT_FN_ATTRS
506 _mm_cvtss_f32(__m128 __a
)
511 static __inline__ __m128 __DEFAULT_FN_ATTRS
512 _mm_loadh_pi(__m128 __a
, const __m64
*__p
)
514 typedef float __mm_loadh_pi_v2f32
__attribute__((__vector_size__(8)));
515 struct __mm_loadh_pi_struct
{
516 __mm_loadh_pi_v2f32 __u
;
517 } __attribute__((__packed__
, __may_alias__
));
518 __mm_loadh_pi_v2f32 __b
= ((struct __mm_loadh_pi_struct
*)__p
)->__u
;
519 __m128 __bb
= __builtin_shufflevector(__b
, __b
, 0, 1, 0, 1);
520 return __builtin_shufflevector(__a
, __bb
, 0, 1, 4, 5);
523 static __inline__ __m128 __DEFAULT_FN_ATTRS
524 _mm_loadl_pi(__m128 __a
, const __m64
*__p
)
526 typedef float __mm_loadl_pi_v2f32
__attribute__((__vector_size__(8)));
527 struct __mm_loadl_pi_struct
{
528 __mm_loadl_pi_v2f32 __u
;
529 } __attribute__((__packed__
, __may_alias__
));
530 __mm_loadl_pi_v2f32 __b
= ((struct __mm_loadl_pi_struct
*)__p
)->__u
;
531 __m128 __bb
= __builtin_shufflevector(__b
, __b
, 0, 1, 0, 1);
532 return __builtin_shufflevector(__a
, __bb
, 4, 5, 2, 3);
535 static __inline__ __m128 __DEFAULT_FN_ATTRS
536 _mm_load_ss(const float *__p
)
538 struct __mm_load_ss_struct
{
540 } __attribute__((__packed__
, __may_alias__
));
541 float __u
= ((struct __mm_load_ss_struct
*)__p
)->__u
;
542 return (__m128
){ __u
, 0, 0, 0 };
545 static __inline__ __m128 __DEFAULT_FN_ATTRS
546 _mm_load1_ps(const float *__p
)
548 struct __mm_load1_ps_struct
{
550 } __attribute__((__packed__
, __may_alias__
));
551 float __u
= ((struct __mm_load1_ps_struct
*)__p
)->__u
;
552 return (__m128
){ __u
, __u
, __u
, __u
};
555 #define _mm_load_ps1(p) _mm_load1_ps(p)
557 static __inline__ __m128 __DEFAULT_FN_ATTRS
558 _mm_load_ps(const float *__p
)
560 return *(__m128
*)__p
;
563 static __inline__ __m128 __DEFAULT_FN_ATTRS
564 _mm_loadu_ps(const float *__p
)
568 } __attribute__((__packed__
, __may_alias__
));
569 return ((struct __loadu_ps
*)__p
)->__v
;
572 static __inline__ __m128 __DEFAULT_FN_ATTRS
573 _mm_loadr_ps(const float *__p
)
575 __m128 __a
= _mm_load_ps(__p
);
576 return __builtin_shufflevector(__a
, __a
, 3, 2, 1, 0);
579 static __inline__ __m128 __DEFAULT_FN_ATTRS
582 return (__m128
)__builtin_ia32_undef128();
585 static __inline__ __m128 __DEFAULT_FN_ATTRS
586 _mm_set_ss(float __w
)
588 return (__m128
){ __w
, 0, 0, 0 };
591 static __inline__ __m128 __DEFAULT_FN_ATTRS
592 _mm_set1_ps(float __w
)
594 return (__m128
){ __w
, __w
, __w
, __w
};
597 /* Microsoft specific. */
598 static __inline__ __m128 __DEFAULT_FN_ATTRS
599 _mm_set_ps1(float __w
)
601 return _mm_set1_ps(__w
);
604 static __inline__ __m128 __DEFAULT_FN_ATTRS
605 _mm_set_ps(float __z
, float __y
, float __x
, float __w
)
607 return (__m128
){ __w
, __x
, __y
, __z
};
610 static __inline__ __m128 __DEFAULT_FN_ATTRS
611 _mm_setr_ps(float __z
, float __y
, float __x
, float __w
)
613 return (__m128
){ __z
, __y
, __x
, __w
};
616 static __inline__ __m128 __DEFAULT_FN_ATTRS
619 return (__m128
){ 0, 0, 0, 0 };
622 static __inline__
void __DEFAULT_FN_ATTRS
623 _mm_storeh_pi(__m64
*__p
, __m128 __a
)
625 __builtin_ia32_storehps((__v2si
*)__p
, __a
);
628 static __inline__
void __DEFAULT_FN_ATTRS
629 _mm_storel_pi(__m64
*__p
, __m128 __a
)
631 __builtin_ia32_storelps((__v2si
*)__p
, __a
);
634 static __inline__
void __DEFAULT_FN_ATTRS
635 _mm_store_ss(float *__p
, __m128 __a
)
637 struct __mm_store_ss_struct
{
639 } __attribute__((__packed__
, __may_alias__
));
640 ((struct __mm_store_ss_struct
*)__p
)->__u
= __a
[0];
643 static __inline__
void __DEFAULT_FN_ATTRS
644 _mm_storeu_ps(float *__p
, __m128 __a
)
646 __builtin_ia32_storeups(__p
, __a
);
649 static __inline__
void __DEFAULT_FN_ATTRS
650 _mm_store1_ps(float *__p
, __m128 __a
)
652 __a
= __builtin_shufflevector(__a
, __a
, 0, 0, 0, 0);
653 _mm_storeu_ps(__p
, __a
);
656 static __inline__
void __DEFAULT_FN_ATTRS
657 _mm_store_ps1(float *__p
, __m128 __a
)
659 return _mm_store1_ps(__p
, __a
);
662 static __inline__
void __DEFAULT_FN_ATTRS
663 _mm_store_ps(float *__p
, __m128 __a
)
665 *(__m128
*)__p
= __a
;
668 static __inline__
void __DEFAULT_FN_ATTRS
669 _mm_storer_ps(float *__p
, __m128 __a
)
671 __a
= __builtin_shufflevector(__a
, __a
, 3, 2, 1, 0);
672 _mm_store_ps(__p
, __a
);
675 #define _MM_HINT_T0 3
676 #define _MM_HINT_T1 2
677 #define _MM_HINT_T2 1
678 #define _MM_HINT_NTA 0
681 /* FIXME: We have to #define this because "sel" must be a constant integer, and
682 Sema doesn't do any form of constant propagation yet. */
684 #define _mm_prefetch(a, sel) (__builtin_prefetch((void *)(a), 0, (sel)))
687 static __inline__
void __DEFAULT_FN_ATTRS
688 _mm_stream_pi(__m64
*__p
, __m64 __a
)
690 __builtin_ia32_movntq(__p
, __a
);
693 static __inline__
void __DEFAULT_FN_ATTRS
694 _mm_stream_ps(float *__p
, __m128 __a
)
696 __builtin_ia32_movntps(__p
, __a
);
699 static __inline__
void __DEFAULT_FN_ATTRS
702 __builtin_ia32_sfence();
705 static __inline__
int __DEFAULT_FN_ATTRS
706 _mm_extract_pi16(__m64 __a
, int __n
)
708 __v4hi __b
= (__v4hi
)__a
;
709 return (unsigned short)__b
[__n
& 3];
712 static __inline__ __m64 __DEFAULT_FN_ATTRS
713 _mm_insert_pi16(__m64 __a
, int __d
, int __n
)
715 __v4hi __b
= (__v4hi
)__a
;
720 static __inline__ __m64 __DEFAULT_FN_ATTRS
721 _mm_max_pi16(__m64 __a
, __m64 __b
)
723 return (__m64
)__builtin_ia32_pmaxsw((__v4hi
)__a
, (__v4hi
)__b
);
726 static __inline__ __m64 __DEFAULT_FN_ATTRS
727 _mm_max_pu8(__m64 __a
, __m64 __b
)
729 return (__m64
)__builtin_ia32_pmaxub((__v8qi
)__a
, (__v8qi
)__b
);
732 static __inline__ __m64 __DEFAULT_FN_ATTRS
733 _mm_min_pi16(__m64 __a
, __m64 __b
)
735 return (__m64
)__builtin_ia32_pminsw((__v4hi
)__a
, (__v4hi
)__b
);
738 static __inline__ __m64 __DEFAULT_FN_ATTRS
739 _mm_min_pu8(__m64 __a
, __m64 __b
)
741 return (__m64
)__builtin_ia32_pminub((__v8qi
)__a
, (__v8qi
)__b
);
744 static __inline__
int __DEFAULT_FN_ATTRS
745 _mm_movemask_pi8(__m64 __a
)
747 return __builtin_ia32_pmovmskb((__v8qi
)__a
);
750 static __inline__ __m64 __DEFAULT_FN_ATTRS
751 _mm_mulhi_pu16(__m64 __a
, __m64 __b
)
753 return (__m64
)__builtin_ia32_pmulhuw((__v4hi
)__a
, (__v4hi
)__b
);
756 #define _mm_shuffle_pi16(a, n) __extension__ ({ \
757 (__m64)__builtin_ia32_pshufw((__v4hi)(__m64)(a), (n)); })
759 static __inline__
void __DEFAULT_FN_ATTRS
760 _mm_maskmove_si64(__m64 __d
, __m64 __n
, char *__p
)
762 __builtin_ia32_maskmovq((__v8qi
)__d
, (__v8qi
)__n
, __p
);
765 static __inline__ __m64 __DEFAULT_FN_ATTRS
766 _mm_avg_pu8(__m64 __a
, __m64 __b
)
768 return (__m64
)__builtin_ia32_pavgb((__v8qi
)__a
, (__v8qi
)__b
);
771 static __inline__ __m64 __DEFAULT_FN_ATTRS
772 _mm_avg_pu16(__m64 __a
, __m64 __b
)
774 return (__m64
)__builtin_ia32_pavgw((__v4hi
)__a
, (__v4hi
)__b
);
777 static __inline__ __m64 __DEFAULT_FN_ATTRS
778 _mm_sad_pu8(__m64 __a
, __m64 __b
)
780 return (__m64
)__builtin_ia32_psadbw((__v8qi
)__a
, (__v8qi
)__b
);
783 static __inline__
unsigned int __DEFAULT_FN_ATTRS
786 return __builtin_ia32_stmxcsr();
789 static __inline__
void __DEFAULT_FN_ATTRS
790 _mm_setcsr(unsigned int __i
)
792 __builtin_ia32_ldmxcsr(__i
);
795 #define _mm_shuffle_ps(a, b, mask) __extension__ ({ \
796 (__m128)__builtin_shufflevector((__v4sf)(__m128)(a), (__v4sf)(__m128)(b), \
797 (mask) & 0x3, ((mask) & 0xc) >> 2, \
798 (((mask) & 0x30) >> 4) + 4, \
799 (((mask) & 0xc0) >> 6) + 4); })
801 static __inline__ __m128 __DEFAULT_FN_ATTRS
802 _mm_unpackhi_ps(__m128 __a
, __m128 __b
)
804 return __builtin_shufflevector(__a
, __b
, 2, 6, 3, 7);
807 static __inline__ __m128 __DEFAULT_FN_ATTRS
808 _mm_unpacklo_ps(__m128 __a
, __m128 __b
)
810 return __builtin_shufflevector(__a
, __b
, 0, 4, 1, 5);
813 static __inline__ __m128 __DEFAULT_FN_ATTRS
814 _mm_move_ss(__m128 __a
, __m128 __b
)
816 return __builtin_shufflevector(__a
, __b
, 4, 1, 2, 3);
819 static __inline__ __m128 __DEFAULT_FN_ATTRS
820 _mm_movehl_ps(__m128 __a
, __m128 __b
)
822 return __builtin_shufflevector(__a
, __b
, 6, 7, 2, 3);
825 static __inline__ __m128 __DEFAULT_FN_ATTRS
826 _mm_movelh_ps(__m128 __a
, __m128 __b
)
828 return __builtin_shufflevector(__a
, __b
, 0, 1, 4, 5);
831 static __inline__ __m128 __DEFAULT_FN_ATTRS
832 _mm_cvtpi16_ps(__m64 __a
)
837 __b
= _mm_setzero_si64();
838 __b
= _mm_cmpgt_pi16(__b
, __a
);
839 __c
= _mm_unpackhi_pi16(__a
, __b
);
840 __r
= _mm_setzero_ps();
841 __r
= _mm_cvtpi32_ps(__r
, __c
);
842 __r
= _mm_movelh_ps(__r
, __r
);
843 __c
= _mm_unpacklo_pi16(__a
, __b
);
844 __r
= _mm_cvtpi32_ps(__r
, __c
);
849 static __inline__ __m128 __DEFAULT_FN_ATTRS
850 _mm_cvtpu16_ps(__m64 __a
)
855 __b
= _mm_setzero_si64();
856 __c
= _mm_unpackhi_pi16(__a
, __b
);
857 __r
= _mm_setzero_ps();
858 __r
= _mm_cvtpi32_ps(__r
, __c
);
859 __r
= _mm_movelh_ps(__r
, __r
);
860 __c
= _mm_unpacklo_pi16(__a
, __b
);
861 __r
= _mm_cvtpi32_ps(__r
, __c
);
866 static __inline__ __m128 __DEFAULT_FN_ATTRS
867 _mm_cvtpi8_ps(__m64 __a
)
871 __b
= _mm_setzero_si64();
872 __b
= _mm_cmpgt_pi8(__b
, __a
);
873 __b
= _mm_unpacklo_pi8(__a
, __b
);
875 return _mm_cvtpi16_ps(__b
);
878 static __inline__ __m128 __DEFAULT_FN_ATTRS
879 _mm_cvtpu8_ps(__m64 __a
)
883 __b
= _mm_setzero_si64();
884 __b
= _mm_unpacklo_pi8(__a
, __b
);
886 return _mm_cvtpi16_ps(__b
);
889 static __inline__ __m128 __DEFAULT_FN_ATTRS
890 _mm_cvtpi32x2_ps(__m64 __a
, __m64 __b
)
894 __c
= _mm_setzero_ps();
895 __c
= _mm_cvtpi32_ps(__c
, __b
);
896 __c
= _mm_movelh_ps(__c
, __c
);
898 return _mm_cvtpi32_ps(__c
, __a
);
901 static __inline__ __m64 __DEFAULT_FN_ATTRS
902 _mm_cvtps_pi16(__m128 __a
)
906 __b
= _mm_cvtps_pi32(__a
);
907 __a
= _mm_movehl_ps(__a
, __a
);
908 __c
= _mm_cvtps_pi32(__a
);
910 return _mm_packs_pi32(__b
, __c
);
913 static __inline__ __m64 __DEFAULT_FN_ATTRS
914 _mm_cvtps_pi8(__m128 __a
)
918 __b
= _mm_cvtps_pi16(__a
);
919 __c
= _mm_setzero_si64();
921 return _mm_packs_pi16(__b
, __c
);
924 static __inline__
int __DEFAULT_FN_ATTRS
925 _mm_movemask_ps(__m128 __a
)
927 return __builtin_ia32_movmskps(__a
);
932 #define _MM_ALIGN16 __declspec(align(16))
935 #define _MM_SHUFFLE(z, y, x, w) (((z) << 6) | ((y) << 4) | ((x) << 2) | (w))
937 #define _MM_EXCEPT_INVALID (0x0001)
938 #define _MM_EXCEPT_DENORM (0x0002)
939 #define _MM_EXCEPT_DIV_ZERO (0x0004)
940 #define _MM_EXCEPT_OVERFLOW (0x0008)
941 #define _MM_EXCEPT_UNDERFLOW (0x0010)
942 #define _MM_EXCEPT_INEXACT (0x0020)
943 #define _MM_EXCEPT_MASK (0x003f)
945 #define _MM_MASK_INVALID (0x0080)
946 #define _MM_MASK_DENORM (0x0100)
947 #define _MM_MASK_DIV_ZERO (0x0200)
948 #define _MM_MASK_OVERFLOW (0x0400)
949 #define _MM_MASK_UNDERFLOW (0x0800)
950 #define _MM_MASK_INEXACT (0x1000)
951 #define _MM_MASK_MASK (0x1f80)
953 #define _MM_ROUND_NEAREST (0x0000)
954 #define _MM_ROUND_DOWN (0x2000)
955 #define _MM_ROUND_UP (0x4000)
956 #define _MM_ROUND_TOWARD_ZERO (0x6000)
957 #define _MM_ROUND_MASK (0x6000)
959 #define _MM_FLUSH_ZERO_MASK (0x8000)
960 #define _MM_FLUSH_ZERO_ON (0x8000)
961 #define _MM_FLUSH_ZERO_OFF (0x0000)
963 #define _MM_GET_EXCEPTION_MASK() (_mm_getcsr() & _MM_MASK_MASK)
964 #define _MM_GET_EXCEPTION_STATE() (_mm_getcsr() & _MM_EXCEPT_MASK)
965 #define _MM_GET_FLUSH_ZERO_MODE() (_mm_getcsr() & _MM_FLUSH_ZERO_MASK)
966 #define _MM_GET_ROUNDING_MODE() (_mm_getcsr() & _MM_ROUND_MASK)
968 #define _MM_SET_EXCEPTION_MASK(x) (_mm_setcsr((_mm_getcsr() & ~_MM_MASK_MASK) | (x)))
969 #define _MM_SET_EXCEPTION_STATE(x) (_mm_setcsr((_mm_getcsr() & ~_MM_EXCEPT_MASK) | (x)))
970 #define _MM_SET_FLUSH_ZERO_MODE(x) (_mm_setcsr((_mm_getcsr() & ~_MM_FLUSH_ZERO_MASK) | (x)))
971 #define _MM_SET_ROUNDING_MODE(x) (_mm_setcsr((_mm_getcsr() & ~_MM_ROUND_MASK) | (x)))
973 #define _MM_TRANSPOSE4_PS(row0, row1, row2, row3) \
975 __m128 tmp3, tmp2, tmp1, tmp0; \
976 tmp0 = _mm_unpacklo_ps((row0), (row1)); \
977 tmp2 = _mm_unpacklo_ps((row2), (row3)); \
978 tmp1 = _mm_unpackhi_ps((row0), (row1)); \
979 tmp3 = _mm_unpackhi_ps((row2), (row3)); \
980 (row0) = _mm_movelh_ps(tmp0, tmp2); \
981 (row1) = _mm_movehl_ps(tmp2, tmp0); \
982 (row2) = _mm_movelh_ps(tmp1, tmp3); \
983 (row3) = _mm_movehl_ps(tmp3, tmp1); \
986 /* Aliases for compatibility. */
987 #define _m_pextrw _mm_extract_pi16
988 #define _m_pinsrw _mm_insert_pi16
989 #define _m_pmaxsw _mm_max_pi16
990 #define _m_pmaxub _mm_max_pu8
991 #define _m_pminsw _mm_min_pi16
992 #define _m_pminub _mm_min_pu8
993 #define _m_pmovmskb _mm_movemask_pi8
994 #define _m_pmulhuw _mm_mulhi_pu16
995 #define _m_pshufw _mm_shuffle_pi16
996 #define _m_maskmovq _mm_maskmove_si64
997 #define _m_pavgb _mm_avg_pu8
998 #define _m_pavgw _mm_avg_pu16
999 #define _m_psadbw _mm_sad_pu8
1003 #undef __DEFAULT_FN_ATTRS
1005 /* Ugly hack for backwards-compatibility (compatible with gcc) */
1006 #if defined(__SSE2__) && !__has_feature(modules)
1007 #include <emmintrin.h>
1010 #endif /* __XMMINTRIN_H */