gcc/config/i386/smmintrin.h

   1 /* Copyright (C) 2007, 2008, 2009, 2010, 2012 Free Software Foundation, Inc.
   2
   3    This file is part of GCC.
   4
   5    GCC is free software; you can redistribute it and/or modify
   6    it under the terms of the GNU General Public License as published by
   7    the Free Software Foundation; either version 3, or (at your option)
   8    any later version.
   9
  10    GCC is distributed in the hope that it will be useful,
  11    but WITHOUT ANY WARRANTY; without even the implied warranty of
  12    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13    GNU General Public License for more details.
  14
  15    Under Section 7 of GPL version 3, you are granted additional
  16    permissions described in the GCC Runtime Library Exception, version
  17    3.1, as published by the Free Software Foundation.
  18
  19    You should have received a copy of the GNU General Public License and
  20    a copy of the GCC Runtime Library Exception along with this program;
  21    see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
  22    <http://www.gnu.org/licenses/>.  */
  23
  24 /* Implemented from the specification included in the Intel C++ Compiler
  25    User Guide and Reference, version 10.0.  */
  26
  27 #ifndef _SMMINTRIN_H_INCLUDED
  28 #define _SMMINTRIN_H_INCLUDED
  29
  30 #ifndef __SSE4_1__
  31 # error "SSE4.1 instruction set not enabled"
  32 #else
  33
  34 /* We need definitions from the SSSE3, SSE3, SSE2 and SSE header
  35    files.  */
  36 #include <tmmintrin.h>
  37
  38 /* Rounding mode macros. */
  39 #define _MM_FROUND_TO_NEAREST_INT       0x00
  40 #define _MM_FROUND_TO_NEG_INF           0x01
  41 #define _MM_FROUND_TO_POS_INF           0x02
  42 #define _MM_FROUND_TO_ZERO              0x03
  43 #define _MM_FROUND_CUR_DIRECTION        0x04
  44
  45 #define _MM_FROUND_RAISE_EXC            0x00
  46 #define _MM_FROUND_NO_EXC               0x08
  47
  48 #define _MM_FROUND_NINT         \
  49   (_MM_FROUND_TO_NEAREST_INT | _MM_FROUND_RAISE_EXC)
  50 #define _MM_FROUND_FLOOR        \
  51   (_MM_FROUND_TO_NEG_INF | _MM_FROUND_RAISE_EXC)
  52 #define _MM_FROUND_CEIL         \
  53   (_MM_FROUND_TO_POS_INF | _MM_FROUND_RAISE_EXC)
  54 #define _MM_FROUND_TRUNC        \
  55   (_MM_FROUND_TO_ZERO | _MM_FROUND_RAISE_EXC)
  56 #define _MM_FROUND_RINT         \
  57   (_MM_FROUND_CUR_DIRECTION | _MM_FROUND_RAISE_EXC)
  58 #define _MM_FROUND_NEARBYINT    \
  59   (_MM_FROUND_CUR_DIRECTION | _MM_FROUND_NO_EXC)
  60
  61 /* Test Instruction */
  62 /* Packed integer 128-bit bitwise comparison. Return 1 if
  63    (__V & __M) == 0.  */
  64 extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  65 _mm_testz_si128 (__m128i __M, __m128i __V)
  66 {
  67   return __builtin_ia32_ptestz128 ((__v2di)__M, (__v2di)__V);
  68 }
  69
  70 /* Packed integer 128-bit bitwise comparison. Return 1 if
  71    (__V & ~__M) == 0.  */
  72 extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  73 _mm_testc_si128 (__m128i __M, __m128i __V)
  74 {
  75   return __builtin_ia32_ptestc128 ((__v2di)__M, (__v2di)__V);
  76 }
  77
  78 /* Packed integer 128-bit bitwise comparison. Return 1 if
  79    (__V & __M) != 0 && (__V & ~__M) != 0.  */
  80 extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  81 _mm_testnzc_si128 (__m128i __M, __m128i __V)
  82 {
  83   return __builtin_ia32_ptestnzc128 ((__v2di)__M, (__v2di)__V);
  84 }
  85
  86 /* Macros for packed integer 128-bit comparison intrinsics.  */
  87 #define _mm_test_all_zeros(M, V) _mm_testz_si128 ((M), (V))
  88
  89 #define _mm_test_all_ones(V) \
  90   _mm_testc_si128 ((V), _mm_cmpeq_epi32 ((V), (V)))
  91
  92 #define _mm_test_mix_ones_zeros(M, V) _mm_testnzc_si128 ((M), (V))
  93
  94 /* Packed/scalar double precision floating point rounding.  */
  95
  96 #ifdef __OPTIMIZE__
  97 extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  98 _mm_round_pd (__m128d __V, const int __M)
  99 {
 100   return (__m128d) __builtin_ia32_roundpd ((__v2df)__V, __M);
 101 }
 102
 103 extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 104 _mm_round_sd(__m128d __D, __m128d __V, const int __M)
 105 {
 106   return (__m128d) __builtin_ia32_roundsd ((__v2df)__D,
 107                                            (__v2df)__V,
 108                                            __M);
 109 }
 110 #else
 111 #define _mm_round_pd(V, M) \
 112   ((__m128d) __builtin_ia32_roundpd ((__v2df)(__m128d)(V), (int)(M)))
 113
 114 #define _mm_round_sd(D, V, M)                                           \
 115   ((__m128d) __builtin_ia32_roundsd ((__v2df)(__m128d)(D),              \
 116                                      (__v2df)(__m128d)(V), (int)(M)))
 117 #endif
 118
 119 /* Packed/scalar single precision floating point rounding.  */
 120
 121 #ifdef __OPTIMIZE__
 122 extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 123 _mm_round_ps (__m128 __V, const int __M)
 124 {
 125   return (__m128) __builtin_ia32_roundps ((__v4sf)__V, __M);
 126 }
 127
 128 extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 129 _mm_round_ss (__m128 __D, __m128 __V, const int __M)
 130 {
 131   return (__m128) __builtin_ia32_roundss ((__v4sf)__D,
 132                                           (__v4sf)__V,
 133                                           __M);
 134 }
 135 #else
 136 #define _mm_round_ps(V, M) \
 137   ((__m128) __builtin_ia32_roundps ((__v4sf)(__m128)(V), (int)(M)))
 138
 139 #define _mm_round_ss(D, V, M)                                           \
 140   ((__m128) __builtin_ia32_roundss ((__v4sf)(__m128)(D),                \
 141                                     (__v4sf)(__m128)(V), (int)(M)))
 142 #endif
 143
 144 /* Macros for ceil/floor intrinsics.  */
 145 #define _mm_ceil_pd(V)     _mm_round_pd ((V), _MM_FROUND_CEIL)
 146 #define _mm_ceil_sd(D, V)  _mm_round_sd ((D), (V), _MM_FROUND_CEIL)
 147
 148 #define _mm_floor_pd(V)    _mm_round_pd((V), _MM_FROUND_FLOOR)
 149 #define _mm_floor_sd(D, V) _mm_round_sd ((D), (V), _MM_FROUND_FLOOR)
 150
 151 #define _mm_ceil_ps(V)     _mm_round_ps ((V), _MM_FROUND_CEIL)
 152 #define _mm_ceil_ss(D, V)  _mm_round_ss ((D), (V), _MM_FROUND_CEIL)
 153
 154 #define _mm_floor_ps(V)    _mm_round_ps ((V), _MM_FROUND_FLOOR)
 155 #define _mm_floor_ss(D, V) _mm_round_ss ((D), (V), _MM_FROUND_FLOOR)
 156
 157 /* SSE4.1 */
 158
 159 /* Integer blend instructions - select data from 2 sources using
 160    constant/variable mask.  */
 161
 162 #ifdef __OPTIMIZE__
 163 extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 164 _mm_blend_epi16 (__m128i __X, __m128i __Y, const int __M)
 165 {
 166   return (__m128i) __builtin_ia32_pblendw128 ((__v8hi)__X,
 167                                               (__v8hi)__Y,
 168                                               __M);
 169 }
 170 #else
 171 #define _mm_blend_epi16(X, Y, M)                                        \
 172   ((__m128i) __builtin_ia32_pblendw128 ((__v8hi)(__m128i)(X),           \
 173                                         (__v8hi)(__m128i)(Y), (int)(M)))
 174 #endif
 175
 176 extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 177 _mm_blendv_epi8 (__m128i __X, __m128i __Y, __m128i __M)
 178 {
 179   return (__m128i) __builtin_ia32_pblendvb128 ((__v16qi)__X,
 180                                                (__v16qi)__Y,
 181                                                (__v16qi)__M);
 182 }
 183
 184 /* Single precision floating point blend instructions - select data
 185    from 2 sources using constant/variable mask.  */
 186
 187 #ifdef __OPTIMIZE__
 188 extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 189 _mm_blend_ps (__m128 __X, __m128 __Y, const int __M)
 190 {
 191   return (__m128) __builtin_ia32_blendps ((__v4sf)__X,
 192                                           (__v4sf)__Y,
 193                                           __M);
 194 }
 195 #else
 196 #define _mm_blend_ps(X, Y, M)                                           \
 197   ((__m128) __builtin_ia32_blendps ((__v4sf)(__m128)(X),                \
 198                                     (__v4sf)(__m128)(Y), (int)(M)))
 199 #endif
 200
 201 extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 202 _mm_blendv_ps (__m128 __X, __m128 __Y, __m128 __M)
 203 {
 204   return (__m128) __builtin_ia32_blendvps ((__v4sf)__X,
 205                                            (__v4sf)__Y,
 206                                            (__v4sf)__M);
 207 }
 208
 209 /* Double precision floating point blend instructions - select data
 210    from 2 sources using constant/variable mask.  */
 211
 212 #ifdef __OPTIMIZE__
 213 extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 214 _mm_blend_pd (__m128d __X, __m128d __Y, const int __M)
 215 {
 216   return (__m128d) __builtin_ia32_blendpd ((__v2df)__X,
 217                                            (__v2df)__Y,
 218                                            __M);
 219 }
 220 #else
 221 #define _mm_blend_pd(X, Y, M)                                           \
 222   ((__m128d) __builtin_ia32_blendpd ((__v2df)(__m128d)(X),              \
 223                                      (__v2df)(__m128d)(Y), (int)(M)))
 224 #endif
 225
 226 extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 227 _mm_blendv_pd (__m128d __X, __m128d __Y, __m128d __M)
 228 {
 229   return (__m128d) __builtin_ia32_blendvpd ((__v2df)__X,
 230                                             (__v2df)__Y,
 231                                             (__v2df)__M);
 232 }
 233
 234 /* Dot product instructions with mask-defined summing and zeroing parts
 235    of result.  */
 236
 237 #ifdef __OPTIMIZE__
 238 extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 239 _mm_dp_ps (__m128 __X, __m128 __Y, const int __M)
 240 {
 241   return (__m128) __builtin_ia32_dpps ((__v4sf)__X,
 242                                        (__v4sf)__Y,
 243                                        __M);
 244 }
 245
 246 extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 247 _mm_dp_pd (__m128d __X, __m128d __Y, const int __M)
 248 {
 249   return (__m128d) __builtin_ia32_dppd ((__v2df)__X,
 250                                         (__v2df)__Y,
 251                                         __M);
 252 }
 253 #else
 254 #define _mm_dp_ps(X, Y, M)                                              \
 255   ((__m128) __builtin_ia32_dpps ((__v4sf)(__m128)(X),                   \
 256                                  (__v4sf)(__m128)(Y), (int)(M)))
 257
 258 #define _mm_dp_pd(X, Y, M)                                              \
 259   ((__m128d) __builtin_ia32_dppd ((__v2df)(__m128d)(X),                 \
 260                                   (__v2df)(__m128d)(Y), (int)(M)))
 261 #endif
 262
 263 /* Packed integer 64-bit comparison, zeroing or filling with ones
 264    corresponding parts of result.  */
 265 extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 266 _mm_cmpeq_epi64 (__m128i __X, __m128i __Y)
 267 {
 268   return (__m128i) __builtin_ia32_pcmpeqq ((__v2di)__X, (__v2di)__Y);
 269 }
 270
 271 /*  Min/max packed integer instructions.  */
 272
 273 extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 274 _mm_min_epi8 (__m128i __X, __m128i __Y)
 275 {
 276   return (__m128i) __builtin_ia32_pminsb128 ((__v16qi)__X, (__v16qi)__Y);
 277 }
 278
 279 extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 280 _mm_max_epi8 (__m128i __X, __m128i __Y)
 281 {
 282   return (__m128i) __builtin_ia32_pmaxsb128 ((__v16qi)__X, (__v16qi)__Y);
 283 }
 284
 285 extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 286 _mm_min_epu16 (__m128i __X, __m128i __Y)
 287 {
 288   return (__m128i) __builtin_ia32_pminuw128 ((__v8hi)__X, (__v8hi)__Y);
 289 }
 290
 291 extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 292 _mm_max_epu16 (__m128i __X, __m128i __Y)
 293 {
 294   return (__m128i) __builtin_ia32_pmaxuw128 ((__v8hi)__X, (__v8hi)__Y);
 295 }
 296
 297 extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 298 _mm_min_epi32 (__m128i __X, __m128i __Y)
 299 {
 300   return (__m128i) __builtin_ia32_pminsd128 ((__v4si)__X, (__v4si)__Y);
 301 }
 302
 303 extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 304 _mm_max_epi32 (__m128i __X, __m128i __Y)
 305 {
 306   return (__m128i) __builtin_ia32_pmaxsd128 ((__v4si)__X, (__v4si)__Y);
 307 }
 308
 309 extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 310 _mm_min_epu32 (__m128i __X, __m128i __Y)
 311 {
 312   return (__m128i) __builtin_ia32_pminud128 ((__v4si)__X, (__v4si)__Y);
 313 }
 314
 315 extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 316 _mm_max_epu32 (__m128i __X, __m128i __Y)
 317 {
 318   return (__m128i) __builtin_ia32_pmaxud128 ((__v4si)__X, (__v4si)__Y);
 319 }
 320
 321 /* Packed integer 32-bit multiplication with truncation of upper
 322    halves of results.  */
 323 extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 324 _mm_mullo_epi32 (__m128i __X, __m128i __Y)
 325 {
 326   return (__m128i) __builtin_ia32_pmulld128 ((__v4si)__X, (__v4si)__Y);
 327 }
 328
 329 /* Packed integer 32-bit multiplication of 2 pairs of operands
 330    with two 64-bit results.  */
 331 extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 332 _mm_mul_epi32 (__m128i __X, __m128i __Y)
 333 {
 334   return (__m128i) __builtin_ia32_pmuldq128 ((__v4si)__X, (__v4si)__Y);
 335 }
 336
 337 /* Insert single precision float into packed single precision array
 338    element selected by index N.  The bits [7-6] of N define S
 339    index, the bits [5-4] define D index, and bits [3-0] define
 340    zeroing mask for D.  */
 341
 342 #ifdef __OPTIMIZE__
 343 extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 344 _mm_insert_ps (__m128 __D, __m128 __S, const int __N)
 345 {
 346   return (__m128) __builtin_ia32_insertps128 ((__v4sf)__D,
 347                                               (__v4sf)__S,
 348                                               __N);
 349 }
 350 #else
 351 #define _mm_insert_ps(D, S, N)                                          \
 352   ((__m128) __builtin_ia32_insertps128 ((__v4sf)(__m128)(D),            \
 353                                         (__v4sf)(__m128)(S), (int)(N)))
 354 #endif
 355
 356 /* Helper macro to create the N value for _mm_insert_ps.  */
 357 #define _MM_MK_INSERTPS_NDX(S, D, M) (((S) << 6) | ((D) << 4) | (M))
 358
 359 /* Extract binary representation of single precision float from packed
 360    single precision array element of X selected by index N.  */
 361
 362 #ifdef __OPTIMIZE__
 363 extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 364 _mm_extract_ps (__m128 __X, const int __N)
 365 {
 366   union { int i; float f; } __tmp;
 367   __tmp.f = __builtin_ia32_vec_ext_v4sf ((__v4sf)__X, __N);
 368   return __tmp.i;
 369 }
 370 #else
 371 #define _mm_extract_ps(X, N)                                            \
 372   (__extension__                                                        \
 373    ({                                                                   \
 374      union { int i; float f; } __tmp;                                   \
 375      __tmp.f = __builtin_ia32_vec_ext_v4sf ((__v4sf)(__m128)(X), (int)(N)); \
 376      __tmp.i;                                                           \
 377    }))
 378 #endif
 379
 380 /* Extract binary representation of single precision float into
 381    D from packed single precision array element of S selected
 382    by index N.  */
 383 #define _MM_EXTRACT_FLOAT(D, S, N) \
 384   { (D) = __builtin_ia32_vec_ext_v4sf ((__v4sf)(S), (N)); }
 385
 386 /* Extract specified single precision float element into the lower
 387    part of __m128.  */
 388 #define _MM_PICK_OUT_PS(X, N)                           \
 389   _mm_insert_ps (_mm_setzero_ps (), (X),                \
 390                  _MM_MK_INSERTPS_NDX ((N), 0, 0x0e))
 391
 392 /* Insert integer, S, into packed integer array element of D
 393    selected by index N.  */
 394
 395 #ifdef __OPTIMIZE__
 396 extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 397 _mm_insert_epi8 (__m128i __D, int __S, const int __N)
 398 {
 399   return (__m128i) __builtin_ia32_vec_set_v16qi ((__v16qi)__D,
 400                                                  __S, __N);
 401 }
 402
 403 extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 404 _mm_insert_epi32 (__m128i __D, int __S, const int __N)
 405 {
 406   return (__m128i) __builtin_ia32_vec_set_v4si ((__v4si)__D,
 407                                                  __S, __N);
 408 }
 409
 410 #ifdef __x86_64__
 411 extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 412 _mm_insert_epi64 (__m128i __D, long long __S, const int __N)
 413 {
 414   return (__m128i) __builtin_ia32_vec_set_v2di ((__v2di)__D,
 415                                                  __S, __N);
 416 }
 417 #endif
 418 #else
 419 #define _mm_insert_epi8(D, S, N)                                        \
 420   ((__m128i) __builtin_ia32_vec_set_v16qi ((__v16qi)(__m128i)(D),       \
 421                                            (int)(S), (int)(N)))
 422
 423 #define _mm_insert_epi32(D, S, N)                               \
 424   ((__m128i) __builtin_ia32_vec_set_v4si ((__v4si)(__m128i)(D), \
 425                                           (int)(S), (int)(N)))
 426
 427 #ifdef __x86_64__
 428 #define _mm_insert_epi64(D, S, N)                                       \
 429   ((__m128i) __builtin_ia32_vec_set_v2di ((__v2di)(__m128i)(D),         \
 430                                           (long long)(S), (int)(N)))
 431 #endif
 432 #endif
 433
 434 /* Extract integer from packed integer array element of X selected by
 435    index N.  */
 436
 437 #ifdef __OPTIMIZE__
 438 extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 439 _mm_extract_epi8 (__m128i __X, const int __N)
 440 {
 441    return (unsigned char) __builtin_ia32_vec_ext_v16qi ((__v16qi)__X, __N);
 442 }
 443
 444 extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 445 _mm_extract_epi32 (__m128i __X, const int __N)
 446 {
 447    return __builtin_ia32_vec_ext_v4si ((__v4si)__X, __N);
 448 }
 449
 450 #ifdef __x86_64__
 451 extern __inline long long  __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 452 _mm_extract_epi64 (__m128i __X, const int __N)
 453 {
 454   return __builtin_ia32_vec_ext_v2di ((__v2di)__X, __N);
 455 }
 456 #endif
 457 #else
 458 #define _mm_extract_epi8(X, N) \
 459   ((int) (unsigned char) __builtin_ia32_vec_ext_v16qi ((__v16qi)(__m128i)(X), (int)(N)))
 460 #define _mm_extract_epi32(X, N) \
 461   ((int) __builtin_ia32_vec_ext_v4si ((__v4si)(__m128i)(X), (int)(N)))
 462
 463 #ifdef __x86_64__
 464 #define _mm_extract_epi64(X, N) \
 465   ((long long) __builtin_ia32_vec_ext_v2di ((__v2di)(__m128i)(X), (int)(N)))
 466 #endif
 467 #endif
 468
 469 /* Return horizontal packed word minimum and its index in bits [15:0]
 470    and bits [18:16] respectively.  */
 471 extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 472 _mm_minpos_epu16 (__m128i __X)
 473 {
 474   return (__m128i) __builtin_ia32_phminposuw128 ((__v8hi)__X);
 475 }
 476
 477 /* Packed integer sign-extension.  */
 478
 479 extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 480 _mm_cvtepi8_epi32 (__m128i __X)
 481 {
 482   return (__m128i) __builtin_ia32_pmovsxbd128 ((__v16qi)__X);
 483 }
 484
 485 extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 486 _mm_cvtepi16_epi32 (__m128i __X)
 487 {
 488   return (__m128i) __builtin_ia32_pmovsxwd128 ((__v8hi)__X);
 489 }
 490
 491 extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 492 _mm_cvtepi8_epi64 (__m128i __X)
 493 {
 494   return (__m128i) __builtin_ia32_pmovsxbq128 ((__v16qi)__X);
 495 }
 496
 497 extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 498 _mm_cvtepi32_epi64 (__m128i __X)
 499 {
 500   return (__m128i) __builtin_ia32_pmovsxdq128 ((__v4si)__X);
 501 }
 502
 503 extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 504 _mm_cvtepi16_epi64 (__m128i __X)
 505 {
 506   return (__m128i) __builtin_ia32_pmovsxwq128 ((__v8hi)__X);
 507 }
 508
 509 extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 510 _mm_cvtepi8_epi16 (__m128i __X)
 511 {
 512   return (__m128i) __builtin_ia32_pmovsxbw128 ((__v16qi)__X);
 513 }
 514
 515 /* Packed integer zero-extension. */
 516
 517 extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 518 _mm_cvtepu8_epi32 (__m128i __X)
 519 {
 520   return (__m128i) __builtin_ia32_pmovzxbd128 ((__v16qi)__X);
 521 }
 522
 523 extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 524 _mm_cvtepu16_epi32 (__m128i __X)
 525 {
 526   return (__m128i) __builtin_ia32_pmovzxwd128 ((__v8hi)__X);
 527 }
 528
 529 extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 530 _mm_cvtepu8_epi64 (__m128i __X)
 531 {
 532   return (__m128i) __builtin_ia32_pmovzxbq128 ((__v16qi)__X);
 533 }
 534
 535 extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 536 _mm_cvtepu32_epi64 (__m128i __X)
 537 {
 538   return (__m128i) __builtin_ia32_pmovzxdq128 ((__v4si)__X);
 539 }
 540
 541 extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 542 _mm_cvtepu16_epi64 (__m128i __X)
 543 {
 544   return (__m128i) __builtin_ia32_pmovzxwq128 ((__v8hi)__X);
 545 }
 546
 547 extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 548 _mm_cvtepu8_epi16 (__m128i __X)
 549 {
 550   return (__m128i) __builtin_ia32_pmovzxbw128 ((__v16qi)__X);
 551 }
 552
 553 /* Pack 8 double words from 2 operands into 8 words of result with
 554    unsigned saturation. */
 555 extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 556 _mm_packus_epi32 (__m128i __X, __m128i __Y)
 557 {
 558   return (__m128i) __builtin_ia32_packusdw128 ((__v4si)__X, (__v4si)__Y);
 559 }
 560
 561 /* Sum absolute 8-bit integer difference of adjacent groups of 4
 562    byte integers in the first 2 operands.  Starting offsets within
 563    operands are determined by the 3rd mask operand.  */
 564
 565 #ifdef __OPTIMIZE__
 566 extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 567 _mm_mpsadbw_epu8 (__m128i __X, __m128i __Y, const int __M)
 568 {
 569   return (__m128i) __builtin_ia32_mpsadbw128 ((__v16qi)__X,
 570                                               (__v16qi)__Y, __M);
 571 }
 572 #else
 573 #define _mm_mpsadbw_epu8(X, Y, M)                                       \
 574   ((__m128i) __builtin_ia32_mpsadbw128 ((__v16qi)(__m128i)(X),          \
 575                                         (__v16qi)(__m128i)(Y), (int)(M)))
 576 #endif
 577
 578 /* Load double quadword using non-temporal aligned hint.  */
 579 extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 580 _mm_stream_load_si128 (__m128i *__X)
 581 {
 582   return (__m128i) __builtin_ia32_movntdqa ((__v2di *) __X);
 583 }
 584
 585 #ifdef __SSE4_2__
 586
 587 /* These macros specify the source data format.  */
 588 #define _SIDD_UBYTE_OPS                 0x00
 589 #define _SIDD_UWORD_OPS                 0x01
 590 #define _SIDD_SBYTE_OPS                 0x02
 591 #define _SIDD_SWORD_OPS                 0x03
 592
 593 /* These macros specify the comparison operation.  */
 594 #define _SIDD_CMP_EQUAL_ANY             0x00
 595 #define _SIDD_CMP_RANGES                0x04
 596 #define _SIDD_CMP_EQUAL_EACH            0x08
 597 #define _SIDD_CMP_EQUAL_ORDERED         0x0c
 598
 599 /* These macros specify the polarity.  */
 600 #define _SIDD_POSITIVE_POLARITY         0x00
 601 #define _SIDD_NEGATIVE_POLARITY         0x10
 602 #define _SIDD_MASKED_POSITIVE_POLARITY  0x20
 603 #define _SIDD_MASKED_NEGATIVE_POLARITY  0x30
 604
 605 /* These macros specify the output selection in _mm_cmpXstri ().  */
 606 #define _SIDD_LEAST_SIGNIFICANT         0x00
 607 #define _SIDD_MOST_SIGNIFICANT          0x40
 608
 609 /* These macros specify the output selection in _mm_cmpXstrm ().  */
 610 #define _SIDD_BIT_MASK                  0x00
 611 #define _SIDD_UNIT_MASK                 0x40
 612
 613 /* Intrinsics for text/string processing.  */
 614
 615 #ifdef __OPTIMIZE__
 616 extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 617 _mm_cmpistrm (__m128i __X, __m128i __Y, const int __M)
 618 {
 619   return (__m128i) __builtin_ia32_pcmpistrm128 ((__v16qi)__X,
 620                                                 (__v16qi)__Y,
 621                                                 __M);
 622 }
 623
 624 extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 625 _mm_cmpistri (__m128i __X, __m128i __Y, const int __M)
 626 {
 627   return __builtin_ia32_pcmpistri128 ((__v16qi)__X,
 628                                       (__v16qi)__Y,
 629                                       __M);
 630 }
 631
 632 extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 633 _mm_cmpestrm (__m128i __X, int __LX, __m128i __Y, int __LY, const int __M)
 634 {
 635   return (__m128i) __builtin_ia32_pcmpestrm128 ((__v16qi)__X, __LX,
 636                                                 (__v16qi)__Y, __LY,
 637                                                 __M);
 638 }
 639
 640 extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 641 _mm_cmpestri (__m128i __X, int __LX, __m128i __Y, int __LY, const int __M)
 642 {
 643   return __builtin_ia32_pcmpestri128 ((__v16qi)__X, __LX,
 644                                       (__v16qi)__Y, __LY,
 645                                       __M);
 646 }
 647 #else
 648 #define _mm_cmpistrm(X, Y, M)                                           \
 649   ((__m128i) __builtin_ia32_pcmpistrm128 ((__v16qi)(__m128i)(X),        \
 650                                           (__v16qi)(__m128i)(Y), (int)(M)))
 651 #define _mm_cmpistri(X, Y, M)                                           \
 652   ((int) __builtin_ia32_pcmpistri128 ((__v16qi)(__m128i)(X),            \
 653                                       (__v16qi)(__m128i)(Y), (int)(M)))
 654
 655 #define _mm_cmpestrm(X, LX, Y, LY, M)                                   \
 656   ((__m128i) __builtin_ia32_pcmpestrm128 ((__v16qi)(__m128i)(X),        \
 657                                           (int)(LX), (__v16qi)(__m128i)(Y), \
 658                                           (int)(LY), (int)(M)))
 659 #define _mm_cmpestri(X, LX, Y, LY, M)                                   \
 660   ((int) __builtin_ia32_pcmpestri128 ((__v16qi)(__m128i)(X), (int)(LX), \
 661                                       (__v16qi)(__m128i)(Y), (int)(LY), \
 662                                       (int)(M)))
 663 #endif
 664
 665 /* Intrinsics for text/string processing and reading values of
 666    EFlags.  */
 667
 668 #ifdef __OPTIMIZE__
 669 extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 670 _mm_cmpistra (__m128i __X, __m128i __Y, const int __M)
 671 {
 672   return __builtin_ia32_pcmpistria128 ((__v16qi)__X,
 673                                        (__v16qi)__Y,
 674                                        __M);
 675 }
 676
 677 extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 678 _mm_cmpistrc (__m128i __X, __m128i __Y, const int __M)
 679 {
 680   return __builtin_ia32_pcmpistric128 ((__v16qi)__X,
 681                                        (__v16qi)__Y,
 682                                        __M);
 683 }
 684
 685 extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 686 _mm_cmpistro (__m128i __X, __m128i __Y, const int __M)
 687 {
 688   return __builtin_ia32_pcmpistrio128 ((__v16qi)__X,
 689                                        (__v16qi)__Y,
 690                                        __M);
 691 }
 692
 693 extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 694 _mm_cmpistrs (__m128i __X, __m128i __Y, const int __M)
 695 {
 696   return __builtin_ia32_pcmpistris128 ((__v16qi)__X,
 697                                        (__v16qi)__Y,
 698                                        __M);
 699 }
 700
 701 extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 702 _mm_cmpistrz (__m128i __X, __m128i __Y, const int __M)
 703 {
 704   return __builtin_ia32_pcmpistriz128 ((__v16qi)__X,
 705                                        (__v16qi)__Y,
 706                                        __M);
 707 }
 708
 709 extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 710 _mm_cmpestra (__m128i __X, int __LX, __m128i __Y, int __LY, const int __M)
 711 {
 712   return __builtin_ia32_pcmpestria128 ((__v16qi)__X, __LX,
 713                                        (__v16qi)__Y, __LY,
 714                                        __M);
 715 }
 716
 717 extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 718 _mm_cmpestrc (__m128i __X, int __LX, __m128i __Y, int __LY, const int __M)
 719 {
 720   return __builtin_ia32_pcmpestric128 ((__v16qi)__X, __LX,
 721                                        (__v16qi)__Y, __LY,
 722                                        __M);
 723 }
 724
 725 extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 726 _mm_cmpestro (__m128i __X, int __LX, __m128i __Y, int __LY, const int __M)
 727 {
 728   return __builtin_ia32_pcmpestrio128 ((__v16qi)__X, __LX,
 729                                        (__v16qi)__Y, __LY,
 730                                        __M);
 731 }
 732
 733 extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 734 _mm_cmpestrs (__m128i __X, int __LX, __m128i __Y, int __LY, const int __M)
 735 {
 736   return __builtin_ia32_pcmpestris128 ((__v16qi)__X, __LX,
 737                                        (__v16qi)__Y, __LY,
 738                                        __M);
 739 }
 740
 741 extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 742 _mm_cmpestrz (__m128i __X, int __LX, __m128i __Y, int __LY, const int __M)
 743 {
 744   return __builtin_ia32_pcmpestriz128 ((__v16qi)__X, __LX,
 745                                        (__v16qi)__Y, __LY,
 746                                        __M);
 747 }
 748 #else
 749 #define _mm_cmpistra(X, Y, M)                                           \
 750   ((int) __builtin_ia32_pcmpistria128 ((__v16qi)(__m128i)(X),           \
 751                                        (__v16qi)(__m128i)(Y), (int)(M)))
 752 #define _mm_cmpistrc(X, Y, M)                                           \
 753   ((int) __builtin_ia32_pcmpistric128 ((__v16qi)(__m128i)(X),           \
 754                                        (__v16qi)(__m128i)(Y), (int)(M)))
 755 #define _mm_cmpistro(X, Y, M)                                           \
 756   ((int) __builtin_ia32_pcmpistrio128 ((__v16qi)(__m128i)(X),           \
 757                                        (__v16qi)(__m128i)(Y), (int)(M)))
 758 #define _mm_cmpistrs(X, Y, M)                                           \
 759   ((int) __builtin_ia32_pcmpistris128 ((__v16qi)(__m128i)(X),           \
 760                                        (__v16qi)(__m128i)(Y), (int)(M)))
 761 #define _mm_cmpistrz(X, Y, M)                                           \
 762   ((int) __builtin_ia32_pcmpistriz128 ((__v16qi)(__m128i)(X),           \
 763                                        (__v16qi)(__m128i)(Y), (int)(M)))
 764
 765 #define _mm_cmpestra(X, LX, Y, LY, M)                                   \
 766   ((int) __builtin_ia32_pcmpestria128 ((__v16qi)(__m128i)(X), (int)(LX), \
 767                                        (__v16qi)(__m128i)(Y), (int)(LY), \
 768                                        (int)(M)))
 769 #define _mm_cmpestrc(X, LX, Y, LY, M)                                   \
 770   ((int) __builtin_ia32_pcmpestric128 ((__v16qi)(__m128i)(X), (int)(LX), \
 771                                        (__v16qi)(__m128i)(Y), (int)(LY), \
 772                                        (int)(M)))
 773 #define _mm_cmpestro(X, LX, Y, LY, M)                                   \
 774   ((int) __builtin_ia32_pcmpestrio128 ((__v16qi)(__m128i)(X), (int)(LX), \
 775                                        (__v16qi)(__m128i)(Y), (int)(LY), \
 776                                        (int)(M)))
 777 #define _mm_cmpestrs(X, LX, Y, LY, M)                                   \
 778   ((int) __builtin_ia32_pcmpestris128 ((__v16qi)(__m128i)(X), (int)(LX), \
 779                                        (__v16qi)(__m128i)(Y), (int)(LY), \
 780                                        (int)(M)))
 781 #define _mm_cmpestrz(X, LX, Y, LY, M)                                   \
 782   ((int) __builtin_ia32_pcmpestriz128 ((__v16qi)(__m128i)(X), (int)(LX), \
 783                                        (__v16qi)(__m128i)(Y), (int)(LY), \
 784                                        (int)(M)))
 785 #endif
 786
 787 /* Packed integer 64-bit comparison, zeroing or filling with ones
 788    corresponding parts of result.  */
 789 extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 790 _mm_cmpgt_epi64 (__m128i __X, __m128i __Y)
 791 {
 792   return (__m128i) __builtin_ia32_pcmpgtq ((__v2di)__X, (__v2di)__Y);
 793 }
 794
 795 #ifdef __POPCNT__
 796 #include <popcntintrin.h>
 797 #endif
 798
 799 /* Accumulate CRC32 (polynomial 0x11EDC6F41) value.  */
 800 extern __inline unsigned int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 801 _mm_crc32_u8 (unsigned int __C, unsigned char __V)
 802 {
 803   return __builtin_ia32_crc32qi (__C, __V);
 804 }
 805
 806 extern __inline unsigned int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 807 _mm_crc32_u16 (unsigned int __C, unsigned short __V)
 808 {
 809   return __builtin_ia32_crc32hi (__C, __V);
 810 }
 811
 812 extern __inline unsigned int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 813 _mm_crc32_u32 (unsigned int __C, unsigned int __V)
 814 {
 815   return __builtin_ia32_crc32si (__C, __V);
 816 }
 817
 818 #ifdef __x86_64__
 819 extern __inline unsigned long long __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 820 _mm_crc32_u64 (unsigned long long __C, unsigned long long __V)
 821 {
 822   return __builtin_ia32_crc32di (__C, __V);
 823 }
 824 #endif
 825
 826 #endif /* __SSE4_2__ */
 827
 828 #endif /* __SSE4_1__ */
 829
 830 #endif /* _SMMINTRIN_H_INCLUDED */