sdk/build-tools/25.0.2/renderscript/clang-include/smmintrin.h

   1 /*===---- smmintrin.h - SSE4 intrinsics ------------------------------------===
   2  *
   3  * Permission is hereby granted, free of charge, to any person obtaining a copy
   4  * of this software and associated documentation files (the "Software"), to deal
   5  * in the Software without restriction, including without limitation the rights
   6  * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
   7  * copies of the Software, and to permit persons to whom the Software is
   8  * furnished to do so, subject to the following conditions:
   9  *
  10  * The above copyright notice and this permission notice shall be included in
  11  * all copies or substantial portions of the Software.
  12  *
  13  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  14  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  15  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  16  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  17  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  18  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  19  * THE SOFTWARE.
  20  *
  21  *===-----------------------------------------------------------------------===
  22  */
  23
  24 #ifndef _SMMINTRIN_H
  25 #define _SMMINTRIN_H
  26
  27 #include <tmmintrin.h>
  28
  29 /* Define the default attributes for the functions in this file. */
  30 #define __DEFAULT_FN_ATTRS __attribute__((__always_inline__, __nodebug__, __target__("sse4.1")))
  31
  32 /* SSE4 Rounding macros. */
  33 #define _MM_FROUND_TO_NEAREST_INT    0x00
  34 #define _MM_FROUND_TO_NEG_INF        0x01
  35 #define _MM_FROUND_TO_POS_INF        0x02
  36 #define _MM_FROUND_TO_ZERO           0x03
  37 #define _MM_FROUND_CUR_DIRECTION     0x04
  38
  39 #define _MM_FROUND_RAISE_EXC         0x00
  40 #define _MM_FROUND_NO_EXC            0x08
  41
  42 #define _MM_FROUND_NINT      (_MM_FROUND_RAISE_EXC | _MM_FROUND_TO_NEAREST_INT)
  43 #define _MM_FROUND_FLOOR     (_MM_FROUND_RAISE_EXC | _MM_FROUND_TO_NEG_INF)
  44 #define _MM_FROUND_CEIL      (_MM_FROUND_RAISE_EXC | _MM_FROUND_TO_POS_INF)
  45 #define _MM_FROUND_TRUNC     (_MM_FROUND_RAISE_EXC | _MM_FROUND_TO_ZERO)
  46 #define _MM_FROUND_RINT      (_MM_FROUND_RAISE_EXC | _MM_FROUND_CUR_DIRECTION)
  47 #define _MM_FROUND_NEARBYINT (_MM_FROUND_NO_EXC | _MM_FROUND_CUR_DIRECTION)
  48
  49 #define _mm_ceil_ps(X)       _mm_round_ps((X), _MM_FROUND_CEIL)
  50 #define _mm_ceil_pd(X)       _mm_round_pd((X), _MM_FROUND_CEIL)
  51 #define _mm_ceil_ss(X, Y)    _mm_round_ss((X), (Y), _MM_FROUND_CEIL)
  52 #define _mm_ceil_sd(X, Y)    _mm_round_sd((X), (Y), _MM_FROUND_CEIL)
  53
  54 #define _mm_floor_ps(X)      _mm_round_ps((X), _MM_FROUND_FLOOR)
  55 #define _mm_floor_pd(X)      _mm_round_pd((X), _MM_FROUND_FLOOR)
  56 #define _mm_floor_ss(X, Y)   _mm_round_ss((X), (Y), _MM_FROUND_FLOOR)
  57 #define _mm_floor_sd(X, Y)   _mm_round_sd((X), (Y), _MM_FROUND_FLOOR)
  58
  59 #define _mm_round_ps(X, M) __extension__ ({ \
  60   (__m128)__builtin_ia32_roundps((__v4sf)(__m128)(X), (M)); })
  61
  62 #define _mm_round_ss(X, Y, M) __extension__ ({ \
  63   (__m128)__builtin_ia32_roundss((__v4sf)(__m128)(X), \
  64                                  (__v4sf)(__m128)(Y), (M)); })
  65
  66 #define _mm_round_pd(X, M) __extension__ ({ \
  67   (__m128d)__builtin_ia32_roundpd((__v2df)(__m128d)(X), (M)); })
  68
  69 #define _mm_round_sd(X, Y, M) __extension__ ({ \
  70   (__m128d)__builtin_ia32_roundsd((__v2df)(__m128d)(X), \
  71                                   (__v2df)(__m128d)(Y), (M)); })
  72
  73 /* SSE4 Packed Blending Intrinsics.  */
  74 #define _mm_blend_pd(V1, V2, M) __extension__ ({ \
  75   (__m128d)__builtin_shufflevector((__v2df)(__m128d)(V1), \
  76                                    (__v2df)(__m128d)(V2), \
  77                                    (((M) & 0x01) ? 2 : 0), \
  78                                    (((M) & 0x02) ? 3 : 1)); })
  79
  80 #define _mm_blend_ps(V1, V2, M) __extension__ ({ \
  81   (__m128)__builtin_shufflevector((__v4sf)(__m128)(V1), (__v4sf)(__m128)(V2), \
  82                                   (((M) & 0x01) ? 4 : 0), \
  83                                   (((M) & 0x02) ? 5 : 1), \
  84                                   (((M) & 0x04) ? 6 : 2), \
  85                                   (((M) & 0x08) ? 7 : 3)); })
  86
  87 static __inline__ __m128d __DEFAULT_FN_ATTRS
  88 _mm_blendv_pd (__m128d __V1, __m128d __V2, __m128d __M)
  89 {
  90   return (__m128d) __builtin_ia32_blendvpd ((__v2df)__V1, (__v2df)__V2,
  91                                             (__v2df)__M);
  92 }
  93
  94 static __inline__ __m128 __DEFAULT_FN_ATTRS
  95 _mm_blendv_ps (__m128 __V1, __m128 __V2, __m128 __M)
  96 {
  97   return (__m128) __builtin_ia32_blendvps ((__v4sf)__V1, (__v4sf)__V2,
  98                                            (__v4sf)__M);
  99 }
 100
 101 static __inline__ __m128i __DEFAULT_FN_ATTRS
 102 _mm_blendv_epi8 (__m128i __V1, __m128i __V2, __m128i __M)
 103 {
 104   return (__m128i) __builtin_ia32_pblendvb128 ((__v16qi)__V1, (__v16qi)__V2,
 105                                                (__v16qi)__M);
 106 }
 107
 108 #define _mm_blend_epi16(V1, V2, M) __extension__ ({ \
 109   (__m128i)__builtin_shufflevector((__v8hi)(__m128i)(V1), \
 110                                    (__v8hi)(__m128i)(V2), \
 111                                    (((M) & 0x01) ?  8 : 0), \
 112                                    (((M) & 0x02) ?  9 : 1), \
 113                                    (((M) & 0x04) ? 10 : 2), \
 114                                    (((M) & 0x08) ? 11 : 3), \
 115                                    (((M) & 0x10) ? 12 : 4), \
 116                                    (((M) & 0x20) ? 13 : 5), \
 117                                    (((M) & 0x40) ? 14 : 6), \
 118                                    (((M) & 0x80) ? 15 : 7)); })
 119
 120 /* SSE4 Dword Multiply Instructions.  */
 121 static __inline__  __m128i __DEFAULT_FN_ATTRS
 122 _mm_mullo_epi32 (__m128i __V1, __m128i __V2)
 123 {
 124   return (__m128i) ((__v4si)__V1 * (__v4si)__V2);
 125 }
 126
 127 static __inline__  __m128i __DEFAULT_FN_ATTRS
 128 _mm_mul_epi32 (__m128i __V1, __m128i __V2)
 129 {
 130   return (__m128i) __builtin_ia32_pmuldq128 ((__v4si)__V1, (__v4si)__V2);
 131 }
 132
 133 /* SSE4 Floating Point Dot Product Instructions.  */
 134 #define _mm_dp_ps(X, Y, M) __extension__ ({ \
 135   (__m128) __builtin_ia32_dpps((__v4sf)(__m128)(X), \
 136                                (__v4sf)(__m128)(Y), (M)); })
 137
 138 #define _mm_dp_pd(X, Y, M) __extension__ ({\
 139   (__m128d) __builtin_ia32_dppd((__v2df)(__m128d)(X), \
 140                                 (__v2df)(__m128d)(Y), (M)); })
 141
 142 /* SSE4 Streaming Load Hint Instruction.  */
 143 static __inline__  __m128i __DEFAULT_FN_ATTRS
 144 _mm_stream_load_si128 (__m128i const *__V)
 145 {
 146   return (__m128i) __builtin_ia32_movntdqa ((const __v2di *) __V);
 147 }
 148
 149 /* SSE4 Packed Integer Min/Max Instructions.  */
 150 static __inline__  __m128i __DEFAULT_FN_ATTRS
 151 _mm_min_epi8 (__m128i __V1, __m128i __V2)
 152 {
 153   return (__m128i) __builtin_ia32_pminsb128 ((__v16qi) __V1, (__v16qi) __V2);
 154 }
 155
 156 static __inline__  __m128i __DEFAULT_FN_ATTRS
 157 _mm_max_epi8 (__m128i __V1, __m128i __V2)
 158 {
 159   return (__m128i) __builtin_ia32_pmaxsb128 ((__v16qi) __V1, (__v16qi) __V2);
 160 }
 161
 162 static __inline__  __m128i __DEFAULT_FN_ATTRS
 163 _mm_min_epu16 (__m128i __V1, __m128i __V2)
 164 {
 165   return (__m128i) __builtin_ia32_pminuw128 ((__v8hi) __V1, (__v8hi) __V2);
 166 }
 167
 168 static __inline__  __m128i __DEFAULT_FN_ATTRS
 169 _mm_max_epu16 (__m128i __V1, __m128i __V2)
 170 {
 171   return (__m128i) __builtin_ia32_pmaxuw128 ((__v8hi) __V1, (__v8hi) __V2);
 172 }
 173
 174 static __inline__  __m128i __DEFAULT_FN_ATTRS
 175 _mm_min_epi32 (__m128i __V1, __m128i __V2)
 176 {
 177   return (__m128i) __builtin_ia32_pminsd128 ((__v4si) __V1, (__v4si) __V2);
 178 }
 179
 180 static __inline__  __m128i __DEFAULT_FN_ATTRS
 181 _mm_max_epi32 (__m128i __V1, __m128i __V2)
 182 {
 183   return (__m128i) __builtin_ia32_pmaxsd128 ((__v4si) __V1, (__v4si) __V2);
 184 }
 185
 186 static __inline__  __m128i __DEFAULT_FN_ATTRS
 187 _mm_min_epu32 (__m128i __V1, __m128i __V2)
 188 {
 189   return (__m128i) __builtin_ia32_pminud128((__v4si) __V1, (__v4si) __V2);
 190 }
 191
 192 static __inline__  __m128i __DEFAULT_FN_ATTRS
 193 _mm_max_epu32 (__m128i __V1, __m128i __V2)
 194 {
 195   return (__m128i) __builtin_ia32_pmaxud128((__v4si) __V1, (__v4si) __V2);
 196 }
 197
 198 /* SSE4 Insertion and Extraction from XMM Register Instructions.  */
 199 #define _mm_insert_ps(X, Y, N) __builtin_ia32_insertps128((X), (Y), (N))
 200 #define _mm_extract_ps(X, N) (__extension__                      \
 201                               ({ union { int __i; float __f; } __t;  \
 202                                  __v4sf __a = (__v4sf)(__m128)(X);       \
 203                                  __t.__f = __a[(N) & 3];                 \
 204                                  __t.__i;}))
 205
 206 /* Miscellaneous insert and extract macros.  */
 207 /* Extract a single-precision float from X at index N into D.  */
 208 #define _MM_EXTRACT_FLOAT(D, X, N) (__extension__ ({ __v4sf __a = (__v4sf)(X); \
 209                                                     (D) = __a[N]; }))
 210
 211 /* Or together 2 sets of indexes (X and Y) with the zeroing bits (Z) to create
 212    an index suitable for _mm_insert_ps.  */
 213 #define _MM_MK_INSERTPS_NDX(X, Y, Z) (((X) << 6) | ((Y) << 4) | (Z))
 214
 215 /* Extract a float from X at index N into the first index of the return.  */
 216 #define _MM_PICK_OUT_PS(X, N) _mm_insert_ps (_mm_setzero_ps(), (X),   \
 217                                              _MM_MK_INSERTPS_NDX((N), 0, 0x0e))
 218
 219 /* Insert int into packed integer array at index.  */
 220 #define _mm_insert_epi8(X, I, N) (__extension__                           \
 221                                   ({ __v16qi __a = (__v16qi)(__m128i)(X); \
 222                                      __a[(N) & 15] = (I);                 \
 223                                      __a;}))
 224 #define _mm_insert_epi32(X, I, N) (__extension__                         \
 225                                    ({ __v4si __a = (__v4si)(__m128i)(X); \
 226                                       __a[(N) & 3] = (I);                \
 227                                       __a;}))
 228 #ifdef __x86_64__
 229 #define _mm_insert_epi64(X, I, N) (__extension__                         \
 230                                    ({ __v2di __a = (__v2di)(__m128i)(X); \
 231                                       __a[(N) & 1] = (I);                \
 232                                       __a;}))
 233 #endif /* __x86_64__ */
 234
 235 /* Extract int from packed integer array at index.  This returns the element
 236  * as a zero extended value, so it is unsigned.
 237  */
 238 #define _mm_extract_epi8(X, N) (__extension__                           \
 239                                 ({ __v16qi __a = (__v16qi)(__m128i)(X); \
 240                                    (int)(unsigned char) __a[(N) & 15];}))
 241 #define _mm_extract_epi32(X, N) (__extension__                         \
 242                                  ({ __v4si __a = (__v4si)(__m128i)(X); \
 243                                     (int)__a[(N) & 3];}))
 244 #ifdef __x86_64__
 245 #define _mm_extract_epi64(X, N) (__extension__                         \
 246                                  ({ __v2di __a = (__v2di)(__m128i)(X); \
 247                                     (long long)__a[(N) & 1];}))
 248 #endif /* __x86_64 */
 249
 250 /* SSE4 128-bit Packed Integer Comparisons.  */
 251 static __inline__ int __DEFAULT_FN_ATTRS
 252 _mm_testz_si128(__m128i __M, __m128i __V)
 253 {
 254   return __builtin_ia32_ptestz128((__v2di)__M, (__v2di)__V);
 255 }
 256
 257 static __inline__ int __DEFAULT_FN_ATTRS
 258 _mm_testc_si128(__m128i __M, __m128i __V)
 259 {
 260   return __builtin_ia32_ptestc128((__v2di)__M, (__v2di)__V);
 261 }
 262
 263 static __inline__ int __DEFAULT_FN_ATTRS
 264 _mm_testnzc_si128(__m128i __M, __m128i __V)
 265 {
 266   return __builtin_ia32_ptestnzc128((__v2di)__M, (__v2di)__V);
 267 }
 268
 269 #define _mm_test_all_ones(V) _mm_testc_si128((V), _mm_cmpeq_epi32((V), (V)))
 270 #define _mm_test_mix_ones_zeros(M, V) _mm_testnzc_si128((M), (V))
 271 #define _mm_test_all_zeros(M, V) _mm_testz_si128 ((M), (V))
 272
 273 /* SSE4 64-bit Packed Integer Comparisons.  */
 274 static __inline__ __m128i __DEFAULT_FN_ATTRS
 275 _mm_cmpeq_epi64(__m128i __V1, __m128i __V2)
 276 {
 277   return (__m128i)((__v2di)__V1 == (__v2di)__V2);
 278 }
 279
 280 /* SSE4 Packed Integer Sign-Extension.  */
 281 static __inline__ __m128i __DEFAULT_FN_ATTRS
 282 _mm_cvtepi8_epi16(__m128i __V)
 283 {
 284   /* This function always performs a signed extension, but __v16qi is a char
 285      which may be signed or unsigned, so use __v16qs. */
 286   return (__m128i)__builtin_convertvector(__builtin_shufflevector((__v16qs)__V, (__v16qs)__V, 0, 1, 2, 3, 4, 5, 6, 7), __v8hi);
 287 }
 288
 289 static __inline__ __m128i __DEFAULT_FN_ATTRS
 290 _mm_cvtepi8_epi32(__m128i __V)
 291 {
 292   /* This function always performs a signed extension, but __v16qi is a char
 293      which may be signed or unsigned, so use __v16qs. */
 294   return (__m128i)__builtin_convertvector(__builtin_shufflevector((__v16qs)__V, (__v16qs)__V, 0, 1, 2, 3), __v4si);
 295 }
 296
 297 static __inline__ __m128i __DEFAULT_FN_ATTRS
 298 _mm_cvtepi8_epi64(__m128i __V)
 299 {
 300   /* This function always performs a signed extension, but __v16qi is a char
 301      which may be signed or unsigned, so use __v16qs. */
 302   typedef signed char __v16qs __attribute__((__vector_size__(16)));
 303   return (__m128i)__builtin_convertvector(__builtin_shufflevector((__v16qs)__V, (__v16qs)__V, 0, 1), __v2di);
 304 }
 305
 306 static __inline__ __m128i __DEFAULT_FN_ATTRS
 307 _mm_cvtepi16_epi32(__m128i __V)
 308 {
 309   return (__m128i)__builtin_convertvector(__builtin_shufflevector((__v8hi)__V, (__v8hi)__V, 0, 1, 2, 3), __v4si);
 310 }
 311
 312 static __inline__ __m128i __DEFAULT_FN_ATTRS
 313 _mm_cvtepi16_epi64(__m128i __V)
 314 {
 315   return (__m128i)__builtin_convertvector(__builtin_shufflevector((__v8hi)__V, (__v8hi)__V, 0, 1), __v2di);
 316 }
 317
 318 static __inline__ __m128i __DEFAULT_FN_ATTRS
 319 _mm_cvtepi32_epi64(__m128i __V)
 320 {
 321   return (__m128i)__builtin_convertvector(__builtin_shufflevector((__v4si)__V, (__v4si)__V, 0, 1), __v2di);
 322 }
 323
 324 /* SSE4 Packed Integer Zero-Extension.  */
 325 static __inline__ __m128i __DEFAULT_FN_ATTRS
 326 _mm_cvtepu8_epi16(__m128i __V)
 327 {
 328   return (__m128i) __builtin_ia32_pmovzxbw128((__v16qi) __V);
 329 }
 330
 331 static __inline__ __m128i __DEFAULT_FN_ATTRS
 332 _mm_cvtepu8_epi32(__m128i __V)
 333 {
 334   return (__m128i) __builtin_ia32_pmovzxbd128((__v16qi)__V);
 335 }
 336
 337 static __inline__ __m128i __DEFAULT_FN_ATTRS
 338 _mm_cvtepu8_epi64(__m128i __V)
 339 {
 340   return (__m128i) __builtin_ia32_pmovzxbq128((__v16qi)__V);
 341 }
 342
 343 static __inline__ __m128i __DEFAULT_FN_ATTRS
 344 _mm_cvtepu16_epi32(__m128i __V)
 345 {
 346   return (__m128i) __builtin_ia32_pmovzxwd128((__v8hi)__V);
 347 }
 348
 349 static __inline__ __m128i __DEFAULT_FN_ATTRS
 350 _mm_cvtepu16_epi64(__m128i __V)
 351 {
 352   return (__m128i) __builtin_ia32_pmovzxwq128((__v8hi)__V);
 353 }
 354
 355 static __inline__ __m128i __DEFAULT_FN_ATTRS
 356 _mm_cvtepu32_epi64(__m128i __V)
 357 {
 358   return (__m128i) __builtin_ia32_pmovzxdq128((__v4si)__V);
 359 }
 360
 361 /* SSE4 Pack with Unsigned Saturation.  */
 362 static __inline__ __m128i __DEFAULT_FN_ATTRS
 363 _mm_packus_epi32(__m128i __V1, __m128i __V2)
 364 {
 365   return (__m128i) __builtin_ia32_packusdw128((__v4si)__V1, (__v4si)__V2);
 366 }
 367
 368 /* SSE4 Multiple Packed Sums of Absolute Difference.  */
 369 #define _mm_mpsadbw_epu8(X, Y, M) __extension__ ({ \
 370   (__m128i) __builtin_ia32_mpsadbw128((__v16qi)(__m128i)(X), \
 371                                       (__v16qi)(__m128i)(Y), (M)); })
 372
 373 static __inline__ __m128i __DEFAULT_FN_ATTRS
 374 _mm_minpos_epu16(__m128i __V)
 375 {
 376   return (__m128i) __builtin_ia32_phminposuw128((__v8hi)__V);
 377 }
 378
 379 /* Handle the sse4.2 definitions here. */
 380
 381 /* These definitions are normally in nmmintrin.h, but gcc puts them in here
 382    so we'll do the same.  */
 383
 384 #undef __DEFAULT_FN_ATTRS
 385 #define __DEFAULT_FN_ATTRS __attribute__((__always_inline__, __nodebug__, __target__("sse4.2")))
 386
 387 /* These specify the type of data that we're comparing.  */
 388 #define _SIDD_UBYTE_OPS                 0x00
 389 #define _SIDD_UWORD_OPS                 0x01
 390 #define _SIDD_SBYTE_OPS                 0x02
 391 #define _SIDD_SWORD_OPS                 0x03
 392
 393 /* These specify the type of comparison operation.  */
 394 #define _SIDD_CMP_EQUAL_ANY             0x00
 395 #define _SIDD_CMP_RANGES                0x04
 396 #define _SIDD_CMP_EQUAL_EACH            0x08
 397 #define _SIDD_CMP_EQUAL_ORDERED         0x0c
 398
 399 /* These macros specify the polarity of the operation.  */
 400 #define _SIDD_POSITIVE_POLARITY         0x00
 401 #define _SIDD_NEGATIVE_POLARITY         0x10
 402 #define _SIDD_MASKED_POSITIVE_POLARITY  0x20
 403 #define _SIDD_MASKED_NEGATIVE_POLARITY  0x30
 404
 405 /* These macros are used in _mm_cmpXstri() to specify the return.  */
 406 #define _SIDD_LEAST_SIGNIFICANT         0x00
 407 #define _SIDD_MOST_SIGNIFICANT          0x40
 408
 409 /* These macros are used in _mm_cmpXstri() to specify the return.  */
 410 #define _SIDD_BIT_MASK                  0x00
 411 #define _SIDD_UNIT_MASK                 0x40
 412
 413 /* SSE4.2 Packed Comparison Intrinsics.  */
 414 #define _mm_cmpistrm(A, B, M) \
 415   (__m128i)__builtin_ia32_pcmpistrm128((__v16qi)(__m128i)(A), \
 416                                        (__v16qi)(__m128i)(B), (int)(M))
 417 #define _mm_cmpistri(A, B, M) \
 418   (int)__builtin_ia32_pcmpistri128((__v16qi)(__m128i)(A), \
 419                                    (__v16qi)(__m128i)(B), (int)(M))
 420
 421 #define _mm_cmpestrm(A, LA, B, LB, M) \
 422   (__m128i)__builtin_ia32_pcmpestrm128((__v16qi)(__m128i)(A), (int)(LA), \
 423                                        (__v16qi)(__m128i)(B), (int)(LB), \
 424                                        (int)(M))
 425 #define _mm_cmpestri(A, LA, B, LB, M) \
 426   (int)__builtin_ia32_pcmpestri128((__v16qi)(__m128i)(A), (int)(LA), \
 427                                    (__v16qi)(__m128i)(B), (int)(LB), \
 428                                    (int)(M))
 429
 430 /* SSE4.2 Packed Comparison Intrinsics and EFlag Reading.  */
 431 #define _mm_cmpistra(A, B, M) \
 432   (int)__builtin_ia32_pcmpistria128((__v16qi)(__m128i)(A), \
 433                                     (__v16qi)(__m128i)(B), (int)(M))
 434 #define _mm_cmpistrc(A, B, M) \
 435   (int)__builtin_ia32_pcmpistric128((__v16qi)(__m128i)(A), \
 436                                     (__v16qi)(__m128i)(B), (int)(M))
 437 #define _mm_cmpistro(A, B, M) \
 438   (int)__builtin_ia32_pcmpistrio128((__v16qi)(__m128i)(A), \
 439                                     (__v16qi)(__m128i)(B), (int)(M))
 440 #define _mm_cmpistrs(A, B, M) \
 441   (int)__builtin_ia32_pcmpistris128((__v16qi)(__m128i)(A), \
 442                                     (__v16qi)(__m128i)(B), (int)(M))
 443 #define _mm_cmpistrz(A, B, M) \
 444   (int)__builtin_ia32_pcmpistriz128((__v16qi)(__m128i)(A), \
 445                                     (__v16qi)(__m128i)(B), (int)(M))
 446
 447 #define _mm_cmpestra(A, LA, B, LB, M) \
 448   (int)__builtin_ia32_pcmpestria128((__v16qi)(__m128i)(A), (int)(LA), \
 449                                     (__v16qi)(__m128i)(B), (int)(LB), \
 450                                     (int)(M))
 451 #define _mm_cmpestrc(A, LA, B, LB, M) \
 452   (int)__builtin_ia32_pcmpestric128((__v16qi)(__m128i)(A), (int)(LA), \
 453                                     (__v16qi)(__m128i)(B), (int)(LB), \
 454                                     (int)(M))
 455 #define _mm_cmpestro(A, LA, B, LB, M) \
 456   (int)__builtin_ia32_pcmpestrio128((__v16qi)(__m128i)(A), (int)(LA), \
 457                                     (__v16qi)(__m128i)(B), (int)(LB), \
 458                                     (int)(M))
 459 #define _mm_cmpestrs(A, LA, B, LB, M) \
 460   (int)__builtin_ia32_pcmpestris128((__v16qi)(__m128i)(A), (int)(LA), \
 461                                     (__v16qi)(__m128i)(B), (int)(LB), \
 462                                     (int)(M))
 463 #define _mm_cmpestrz(A, LA, B, LB, M) \
 464   (int)__builtin_ia32_pcmpestriz128((__v16qi)(__m128i)(A), (int)(LA), \
 465                                     (__v16qi)(__m128i)(B), (int)(LB), \
 466                                     (int)(M))
 467
 468 /* SSE4.2 Compare Packed Data -- Greater Than.  */
 469 static __inline__ __m128i __DEFAULT_FN_ATTRS
 470 _mm_cmpgt_epi64(__m128i __V1, __m128i __V2)
 471 {
 472   return (__m128i)((__v2di)__V1 > (__v2di)__V2);
 473 }
 474
 475 /* SSE4.2 Accumulate CRC32.  */
 476 static __inline__ unsigned int __DEFAULT_FN_ATTRS
 477 _mm_crc32_u8(unsigned int __C, unsigned char __D)
 478 {
 479   return __builtin_ia32_crc32qi(__C, __D);
 480 }
 481
 482 static __inline__ unsigned int __DEFAULT_FN_ATTRS
 483 _mm_crc32_u16(unsigned int __C, unsigned short __D)
 484 {
 485   return __builtin_ia32_crc32hi(__C, __D);
 486 }
 487
 488 static __inline__ unsigned int __DEFAULT_FN_ATTRS
 489 _mm_crc32_u32(unsigned int __C, unsigned int __D)
 490 {
 491   return __builtin_ia32_crc32si(__C, __D);
 492 }
 493
 494 #ifdef __x86_64__
 495 static __inline__ unsigned long long __DEFAULT_FN_ATTRS
 496 _mm_crc32_u64(unsigned long long __C, unsigned long long __D)
 497 {
 498   return __builtin_ia32_crc32di(__C, __D);
 499 }
 500 #endif /* __x86_64__ */
 501
 502 #undef __DEFAULT_FN_ATTRS
 503
 504 #ifdef __POPCNT__
 505 #include <popcntintrin.h>
 506 #endif
 507
 508 #endif /* _SMMINTRIN_H */