gcc/config/i386/mmintrin.h

   1 /* Copyright (C) 2002-2018 Free Software Foundation, Inc.
   2
   3    This file is part of GCC.
   4
   5    GCC is free software; you can redistribute it and/or modify
   6    it under the terms of the GNU General Public License as published by
   7    the Free Software Foundation; either version 3, or (at your option)
   8    any later version.
   9
  10    GCC is distributed in the hope that it will be useful,
  11    but WITHOUT ANY WARRANTY; without even the implied warranty of
  12    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13    GNU General Public License for more details.
  14
  15    Under Section 7 of GPL version 3, you are granted additional
  16    permissions described in the GCC Runtime Library Exception, version
  17    3.1, as published by the Free Software Foundation.
  18
  19    You should have received a copy of the GNU General Public License and
  20    a copy of the GCC Runtime Library Exception along with this program;
  21    see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
  22    <http://www.gnu.org/licenses/>.  */
  23
  24 /* Implemented from the specification included in the Intel C++ Compiler
  25    User Guide and Reference, version 9.0.  */
  26
  27 #ifndef _MMINTRIN_H_INCLUDED
  28 #define _MMINTRIN_H_INCLUDED
  29
  30 #if defined __x86_64__ && !defined __SSE__ || !defined __MMX__
  31 #pragma GCC push_options
  32 #ifdef __x86_64__
  33 #pragma GCC target("sse,mmx")
  34 #else
  35 #pragma GCC target("mmx")
  36 #endif
  37 #define __DISABLE_MMX__
  38 #endif /* __MMX__ */
  39
  40 /* The Intel API is flexible enough that we must allow aliasing with other
  41    vector types, and their scalar components.  */
  42 typedef int __m64 __attribute__ ((__vector_size__ (8), __may_alias__));
  43
  44 /* Unaligned version of the same type  */
  45 typedef int __m64_u __attribute__ ((__vector_size__ (8), __may_alias__, __aligned__ (1)));
  46
  47 /* Internal data types for implementing the intrinsics.  */
  48 typedef int __v2si __attribute__ ((__vector_size__ (8)));
  49 typedef short __v4hi __attribute__ ((__vector_size__ (8)));
  50 typedef char __v8qi __attribute__ ((__vector_size__ (8)));
  51 typedef long long __v1di __attribute__ ((__vector_size__ (8)));
  52 typedef float __v2sf __attribute__ ((__vector_size__ (8)));
  53
  54 /* Empty the multimedia state.  */
  55 extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  56 _mm_empty (void)
  57 {
  58   __builtin_ia32_emms ();
  59 }
  60
  61 extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  62 _m_empty (void)
  63 {
  64   _mm_empty ();
  65 }
  66
  67 /* Convert I to a __m64 object.  The integer is zero-extended to 64-bits.  */
  68 extern __inline __m64  __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  69 _mm_cvtsi32_si64 (int __i)
  70 {
  71   return (__m64) __builtin_ia32_vec_init_v2si (__i, 0);
  72 }
  73
  74 extern __inline __m64  __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  75 _m_from_int (int __i)
  76 {
  77   return _mm_cvtsi32_si64 (__i);
  78 }
  79
  80 #ifdef __x86_64__
  81 /* Convert I to a __m64 object.  */
  82
  83 /* Intel intrinsic.  */
  84 extern __inline __m64  __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  85 _m_from_int64 (long long __i)
  86 {
  87   return (__m64) __i;
  88 }
  89
  90 extern __inline __m64  __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  91 _mm_cvtsi64_m64 (long long __i)
  92 {
  93   return (__m64) __i;
  94 }
  95
  96 /* Microsoft intrinsic.  */
  97 extern __inline __m64  __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  98 _mm_cvtsi64x_si64 (long long __i)
  99 {
 100   return (__m64) __i;
 101 }
 102
 103 extern __inline __m64  __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 104 _mm_set_pi64x (long long __i)
 105 {
 106   return (__m64) __i;
 107 }
 108 #endif
 109
 110 /* Convert the lower 32 bits of the __m64 object into an integer.  */
 111 extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 112 _mm_cvtsi64_si32 (__m64 __i)
 113 {
 114   return __builtin_ia32_vec_ext_v2si ((__v2si)__i, 0);
 115 }
 116
 117 extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 118 _m_to_int (__m64 __i)
 119 {
 120   return _mm_cvtsi64_si32 (__i);
 121 }
 122
 123 #ifdef __x86_64__
 124 /* Convert the __m64 object to a 64bit integer.  */
 125
 126 /* Intel intrinsic.  */
 127 extern __inline long long __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 128 _m_to_int64 (__m64 __i)
 129 {
 130   return (long long)__i;
 131 }
 132
 133 extern __inline long long __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 134 _mm_cvtm64_si64 (__m64 __i)
 135 {
 136   return (long long)__i;
 137 }
 138
 139 /* Microsoft intrinsic.  */
 140 extern __inline long long __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 141 _mm_cvtsi64_si64x (__m64 __i)
 142 {
 143   return (long long)__i;
 144 }
 145 #endif
 146
 147 /* Pack the four 16-bit values from M1 into the lower four 8-bit values of
 148    the result, and the four 16-bit values from M2 into the upper four 8-bit
 149    values of the result, all with signed saturation.  */
 150 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 151 _mm_packs_pi16 (__m64 __m1, __m64 __m2)
 152 {
 153   return (__m64) __builtin_ia32_packsswb ((__v4hi)__m1, (__v4hi)__m2);
 154 }
 155
 156 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 157 _m_packsswb (__m64 __m1, __m64 __m2)
 158 {
 159   return _mm_packs_pi16 (__m1, __m2);
 160 }
 161
 162 /* Pack the two 32-bit values from M1 in to the lower two 16-bit values of
 163    the result, and the two 32-bit values from M2 into the upper two 16-bit
 164    values of the result, all with signed saturation.  */
 165 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 166 _mm_packs_pi32 (__m64 __m1, __m64 __m2)
 167 {
 168   return (__m64) __builtin_ia32_packssdw ((__v2si)__m1, (__v2si)__m2);
 169 }
 170
 171 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 172 _m_packssdw (__m64 __m1, __m64 __m2)
 173 {
 174   return _mm_packs_pi32 (__m1, __m2);
 175 }
 176
 177 /* Pack the four 16-bit values from M1 into the lower four 8-bit values of
 178    the result, and the four 16-bit values from M2 into the upper four 8-bit
 179    values of the result, all with unsigned saturation.  */
 180 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 181 _mm_packs_pu16 (__m64 __m1, __m64 __m2)
 182 {
 183   return (__m64) __builtin_ia32_packuswb ((__v4hi)__m1, (__v4hi)__m2);
 184 }
 185
 186 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 187 _m_packuswb (__m64 __m1, __m64 __m2)
 188 {
 189   return _mm_packs_pu16 (__m1, __m2);
 190 }
 191
 192 /* Interleave the four 8-bit values from the high half of M1 with the four
 193    8-bit values from the high half of M2.  */
 194 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 195 _mm_unpackhi_pi8 (__m64 __m1, __m64 __m2)
 196 {
 197   return (__m64) __builtin_ia32_punpckhbw ((__v8qi)__m1, (__v8qi)__m2);
 198 }
 199
 200 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 201 _m_punpckhbw (__m64 __m1, __m64 __m2)
 202 {
 203   return _mm_unpackhi_pi8 (__m1, __m2);
 204 }
 205
 206 /* Interleave the two 16-bit values from the high half of M1 with the two
 207    16-bit values from the high half of M2.  */
 208 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 209 _mm_unpackhi_pi16 (__m64 __m1, __m64 __m2)
 210 {
 211   return (__m64) __builtin_ia32_punpckhwd ((__v4hi)__m1, (__v4hi)__m2);
 212 }
 213
 214 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 215 _m_punpckhwd (__m64 __m1, __m64 __m2)
 216 {
 217   return _mm_unpackhi_pi16 (__m1, __m2);
 218 }
 219
 220 /* Interleave the 32-bit value from the high half of M1 with the 32-bit
 221    value from the high half of M2.  */
 222 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 223 _mm_unpackhi_pi32 (__m64 __m1, __m64 __m2)
 224 {
 225   return (__m64) __builtin_ia32_punpckhdq ((__v2si)__m1, (__v2si)__m2);
 226 }
 227
 228 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 229 _m_punpckhdq (__m64 __m1, __m64 __m2)
 230 {
 231   return _mm_unpackhi_pi32 (__m1, __m2);
 232 }
 233
 234 /* Interleave the four 8-bit values from the low half of M1 with the four
 235    8-bit values from the low half of M2.  */
 236 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 237 _mm_unpacklo_pi8 (__m64 __m1, __m64 __m2)
 238 {
 239   return (__m64) __builtin_ia32_punpcklbw ((__v8qi)__m1, (__v8qi)__m2);
 240 }
 241
 242 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 243 _m_punpcklbw (__m64 __m1, __m64 __m2)
 244 {
 245   return _mm_unpacklo_pi8 (__m1, __m2);
 246 }
 247
 248 /* Interleave the two 16-bit values from the low half of M1 with the two
 249    16-bit values from the low half of M2.  */
 250 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 251 _mm_unpacklo_pi16 (__m64 __m1, __m64 __m2)
 252 {
 253   return (__m64) __builtin_ia32_punpcklwd ((__v4hi)__m1, (__v4hi)__m2);
 254 }
 255
 256 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 257 _m_punpcklwd (__m64 __m1, __m64 __m2)
 258 {
 259   return _mm_unpacklo_pi16 (__m1, __m2);
 260 }
 261
 262 /* Interleave the 32-bit value from the low half of M1 with the 32-bit
 263    value from the low half of M2.  */
 264 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 265 _mm_unpacklo_pi32 (__m64 __m1, __m64 __m2)
 266 {
 267   return (__m64) __builtin_ia32_punpckldq ((__v2si)__m1, (__v2si)__m2);
 268 }
 269
 270 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 271 _m_punpckldq (__m64 __m1, __m64 __m2)
 272 {
 273   return _mm_unpacklo_pi32 (__m1, __m2);
 274 }
 275
 276 /* Add the 8-bit values in M1 to the 8-bit values in M2.  */
 277 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 278 _mm_add_pi8 (__m64 __m1, __m64 __m2)
 279 {
 280   return (__m64) __builtin_ia32_paddb ((__v8qi)__m1, (__v8qi)__m2);
 281 }
 282
 283 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 284 _m_paddb (__m64 __m1, __m64 __m2)
 285 {
 286   return _mm_add_pi8 (__m1, __m2);
 287 }
 288
 289 /* Add the 16-bit values in M1 to the 16-bit values in M2.  */
 290 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 291 _mm_add_pi16 (__m64 __m1, __m64 __m2)
 292 {
 293   return (__m64) __builtin_ia32_paddw ((__v4hi)__m1, (__v4hi)__m2);
 294 }
 295
 296 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 297 _m_paddw (__m64 __m1, __m64 __m2)
 298 {
 299   return _mm_add_pi16 (__m1, __m2);
 300 }
 301
 302 /* Add the 32-bit values in M1 to the 32-bit values in M2.  */
 303 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 304 _mm_add_pi32 (__m64 __m1, __m64 __m2)
 305 {
 306   return (__m64) __builtin_ia32_paddd ((__v2si)__m1, (__v2si)__m2);
 307 }
 308
 309 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 310 _m_paddd (__m64 __m1, __m64 __m2)
 311 {
 312   return _mm_add_pi32 (__m1, __m2);
 313 }
 314
 315 /* Add the 64-bit values in M1 to the 64-bit values in M2.  */
 316 #ifndef __SSE2__
 317 #pragma GCC push_options
 318 #pragma GCC target("sse2,mmx")
 319 #define __DISABLE_SSE2__
 320 #endif /* __SSE2__ */
 321
 322 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 323 _mm_add_si64 (__m64 __m1, __m64 __m2)
 324 {
 325   return (__m64) __builtin_ia32_paddq ((__v1di)__m1, (__v1di)__m2);
 326 }
 327 #ifdef __DISABLE_SSE2__
 328 #undef __DISABLE_SSE2__
 329 #pragma GCC pop_options
 330 #endif /* __DISABLE_SSE2__ */
 331
 332 /* Add the 8-bit values in M1 to the 8-bit values in M2 using signed
 333    saturated arithmetic.  */
 334 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 335 _mm_adds_pi8 (__m64 __m1, __m64 __m2)
 336 {
 337   return (__m64) __builtin_ia32_paddsb ((__v8qi)__m1, (__v8qi)__m2);
 338 }
 339
 340 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 341 _m_paddsb (__m64 __m1, __m64 __m2)
 342 {
 343   return _mm_adds_pi8 (__m1, __m2);
 344 }
 345
 346 /* Add the 16-bit values in M1 to the 16-bit values in M2 using signed
 347    saturated arithmetic.  */
 348 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 349 _mm_adds_pi16 (__m64 __m1, __m64 __m2)
 350 {
 351   return (__m64) __builtin_ia32_paddsw ((__v4hi)__m1, (__v4hi)__m2);
 352 }
 353
 354 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 355 _m_paddsw (__m64 __m1, __m64 __m2)
 356 {
 357   return _mm_adds_pi16 (__m1, __m2);
 358 }
 359
 360 /* Add the 8-bit values in M1 to the 8-bit values in M2 using unsigned
 361    saturated arithmetic.  */
 362 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 363 _mm_adds_pu8 (__m64 __m1, __m64 __m2)
 364 {
 365   return (__m64) __builtin_ia32_paddusb ((__v8qi)__m1, (__v8qi)__m2);
 366 }
 367
 368 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 369 _m_paddusb (__m64 __m1, __m64 __m2)
 370 {
 371   return _mm_adds_pu8 (__m1, __m2);
 372 }
 373
 374 /* Add the 16-bit values in M1 to the 16-bit values in M2 using unsigned
 375    saturated arithmetic.  */
 376 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 377 _mm_adds_pu16 (__m64 __m1, __m64 __m2)
 378 {
 379   return (__m64) __builtin_ia32_paddusw ((__v4hi)__m1, (__v4hi)__m2);
 380 }
 381
 382 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 383 _m_paddusw (__m64 __m1, __m64 __m2)
 384 {
 385   return _mm_adds_pu16 (__m1, __m2);
 386 }
 387
 388 /* Subtract the 8-bit values in M2 from the 8-bit values in M1.  */
 389 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 390 _mm_sub_pi8 (__m64 __m1, __m64 __m2)
 391 {
 392   return (__m64) __builtin_ia32_psubb ((__v8qi)__m1, (__v8qi)__m2);
 393 }
 394
 395 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 396 _m_psubb (__m64 __m1, __m64 __m2)
 397 {
 398   return _mm_sub_pi8 (__m1, __m2);
 399 }
 400
 401 /* Subtract the 16-bit values in M2 from the 16-bit values in M1.  */
 402 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 403 _mm_sub_pi16 (__m64 __m1, __m64 __m2)
 404 {
 405   return (__m64) __builtin_ia32_psubw ((__v4hi)__m1, (__v4hi)__m2);
 406 }
 407
 408 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 409 _m_psubw (__m64 __m1, __m64 __m2)
 410 {
 411   return _mm_sub_pi16 (__m1, __m2);
 412 }
 413
 414 /* Subtract the 32-bit values in M2 from the 32-bit values in M1.  */
 415 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 416 _mm_sub_pi32 (__m64 __m1, __m64 __m2)
 417 {
 418   return (__m64) __builtin_ia32_psubd ((__v2si)__m1, (__v2si)__m2);
 419 }
 420
 421 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 422 _m_psubd (__m64 __m1, __m64 __m2)
 423 {
 424   return _mm_sub_pi32 (__m1, __m2);
 425 }
 426
 427 /* Add the 64-bit values in M1 to the 64-bit values in M2.  */
 428 #ifndef __SSE2__
 429 #pragma GCC push_options
 430 #pragma GCC target("sse2,mmx")
 431 #define __DISABLE_SSE2__
 432 #endif /* __SSE2__ */
 433
 434 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 435 _mm_sub_si64 (__m64 __m1, __m64 __m2)
 436 {
 437   return (__m64) __builtin_ia32_psubq ((__v1di)__m1, (__v1di)__m2);
 438 }
 439 #ifdef __DISABLE_SSE2__
 440 #undef __DISABLE_SSE2__
 441 #pragma GCC pop_options
 442 #endif /* __DISABLE_SSE2__ */
 443
 444 /* Subtract the 8-bit values in M2 from the 8-bit values in M1 using signed
 445    saturating arithmetic.  */
 446 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 447 _mm_subs_pi8 (__m64 __m1, __m64 __m2)
 448 {
 449   return (__m64) __builtin_ia32_psubsb ((__v8qi)__m1, (__v8qi)__m2);
 450 }
 451
 452 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 453 _m_psubsb (__m64 __m1, __m64 __m2)
 454 {
 455   return _mm_subs_pi8 (__m1, __m2);
 456 }
 457
 458 /* Subtract the 16-bit values in M2 from the 16-bit values in M1 using
 459    signed saturating arithmetic.  */
 460 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 461 _mm_subs_pi16 (__m64 __m1, __m64 __m2)
 462 {
 463   return (__m64) __builtin_ia32_psubsw ((__v4hi)__m1, (__v4hi)__m2);
 464 }
 465
 466 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 467 _m_psubsw (__m64 __m1, __m64 __m2)
 468 {
 469   return _mm_subs_pi16 (__m1, __m2);
 470 }
 471
 472 /* Subtract the 8-bit values in M2 from the 8-bit values in M1 using
 473    unsigned saturating arithmetic.  */
 474 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 475 _mm_subs_pu8 (__m64 __m1, __m64 __m2)
 476 {
 477   return (__m64) __builtin_ia32_psubusb ((__v8qi)__m1, (__v8qi)__m2);
 478 }
 479
 480 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 481 _m_psubusb (__m64 __m1, __m64 __m2)
 482 {
 483   return _mm_subs_pu8 (__m1, __m2);
 484 }
 485
 486 /* Subtract the 16-bit values in M2 from the 16-bit values in M1 using
 487    unsigned saturating arithmetic.  */
 488 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 489 _mm_subs_pu16 (__m64 __m1, __m64 __m2)
 490 {
 491   return (__m64) __builtin_ia32_psubusw ((__v4hi)__m1, (__v4hi)__m2);
 492 }
 493
 494 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 495 _m_psubusw (__m64 __m1, __m64 __m2)
 496 {
 497   return _mm_subs_pu16 (__m1, __m2);
 498 }
 499
 500 /* Multiply four 16-bit values in M1 by four 16-bit values in M2 producing
 501    four 32-bit intermediate results, which are then summed by pairs to
 502    produce two 32-bit results.  */
 503 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 504 _mm_madd_pi16 (__m64 __m1, __m64 __m2)
 505 {
 506   return (__m64) __builtin_ia32_pmaddwd ((__v4hi)__m1, (__v4hi)__m2);
 507 }
 508
 509 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 510 _m_pmaddwd (__m64 __m1, __m64 __m2)
 511 {
 512   return _mm_madd_pi16 (__m1, __m2);
 513 }
 514
 515 /* Multiply four signed 16-bit values in M1 by four signed 16-bit values in
 516    M2 and produce the high 16 bits of the 32-bit results.  */
 517 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 518 _mm_mulhi_pi16 (__m64 __m1, __m64 __m2)
 519 {
 520   return (__m64) __builtin_ia32_pmulhw ((__v4hi)__m1, (__v4hi)__m2);
 521 }
 522
 523 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 524 _m_pmulhw (__m64 __m1, __m64 __m2)
 525 {
 526   return _mm_mulhi_pi16 (__m1, __m2);
 527 }
 528
 529 /* Multiply four 16-bit values in M1 by four 16-bit values in M2 and produce
 530    the low 16 bits of the results.  */
 531 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 532 _mm_mullo_pi16 (__m64 __m1, __m64 __m2)
 533 {
 534   return (__m64) __builtin_ia32_pmullw ((__v4hi)__m1, (__v4hi)__m2);
 535 }
 536
 537 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 538 _m_pmullw (__m64 __m1, __m64 __m2)
 539 {
 540   return _mm_mullo_pi16 (__m1, __m2);
 541 }
 542
 543 /* Shift four 16-bit values in M left by COUNT.  */
 544 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 545 _mm_sll_pi16 (__m64 __m, __m64 __count)
 546 {
 547   return (__m64) __builtin_ia32_psllw ((__v4hi)__m, (__v4hi)__count);
 548 }
 549
 550 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 551 _m_psllw (__m64 __m, __m64 __count)
 552 {
 553   return _mm_sll_pi16 (__m, __count);
 554 }
 555
 556 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 557 _mm_slli_pi16 (__m64 __m, int __count)
 558 {
 559   return (__m64) __builtin_ia32_psllwi ((__v4hi)__m, __count);
 560 }
 561
 562 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 563 _m_psllwi (__m64 __m, int __count)
 564 {
 565   return _mm_slli_pi16 (__m, __count);
 566 }
 567
 568 /* Shift two 32-bit values in M left by COUNT.  */
 569 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 570 _mm_sll_pi32 (__m64 __m, __m64 __count)
 571 {
 572   return (__m64) __builtin_ia32_pslld ((__v2si)__m, (__v2si)__count);
 573 }
 574
 575 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 576 _m_pslld (__m64 __m, __m64 __count)
 577 {
 578   return _mm_sll_pi32 (__m, __count);
 579 }
 580
 581 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 582 _mm_slli_pi32 (__m64 __m, int __count)
 583 {
 584   return (__m64) __builtin_ia32_pslldi ((__v2si)__m, __count);
 585 }
 586
 587 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 588 _m_pslldi (__m64 __m, int __count)
 589 {
 590   return _mm_slli_pi32 (__m, __count);
 591 }
 592
 593 /* Shift the 64-bit value in M left by COUNT.  */
 594 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 595 _mm_sll_si64 (__m64 __m, __m64 __count)
 596 {
 597   return (__m64) __builtin_ia32_psllq ((__v1di)__m, (__v1di)__count);
 598 }
 599
 600 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 601 _m_psllq (__m64 __m, __m64 __count)
 602 {
 603   return _mm_sll_si64 (__m, __count);
 604 }
 605
 606 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 607 _mm_slli_si64 (__m64 __m, int __count)
 608 {
 609   return (__m64) __builtin_ia32_psllqi ((__v1di)__m, __count);
 610 }
 611
 612 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 613 _m_psllqi (__m64 __m, int __count)
 614 {
 615   return _mm_slli_si64 (__m, __count);
 616 }
 617
 618 /* Shift four 16-bit values in M right by COUNT; shift in the sign bit.  */
 619 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 620 _mm_sra_pi16 (__m64 __m, __m64 __count)
 621 {
 622   return (__m64) __builtin_ia32_psraw ((__v4hi)__m, (__v4hi)__count);
 623 }
 624
 625 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 626 _m_psraw (__m64 __m, __m64 __count)
 627 {
 628   return _mm_sra_pi16 (__m, __count);
 629 }
 630
 631 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 632 _mm_srai_pi16 (__m64 __m, int __count)
 633 {
 634   return (__m64) __builtin_ia32_psrawi ((__v4hi)__m, __count);
 635 }
 636
 637 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 638 _m_psrawi (__m64 __m, int __count)
 639 {
 640   return _mm_srai_pi16 (__m, __count);
 641 }
 642
 643 /* Shift two 32-bit values in M right by COUNT; shift in the sign bit.  */
 644 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 645 _mm_sra_pi32 (__m64 __m, __m64 __count)
 646 {
 647   return (__m64) __builtin_ia32_psrad ((__v2si)__m, (__v2si)__count);
 648 }
 649
 650 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 651 _m_psrad (__m64 __m, __m64 __count)
 652 {
 653   return _mm_sra_pi32 (__m, __count);
 654 }
 655
 656 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 657 _mm_srai_pi32 (__m64 __m, int __count)
 658 {
 659   return (__m64) __builtin_ia32_psradi ((__v2si)__m, __count);
 660 }
 661
 662 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 663 _m_psradi (__m64 __m, int __count)
 664 {
 665   return _mm_srai_pi32 (__m, __count);
 666 }
 667
 668 /* Shift four 16-bit values in M right by COUNT; shift in zeros.  */
 669 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 670 _mm_srl_pi16 (__m64 __m, __m64 __count)
 671 {
 672   return (__m64) __builtin_ia32_psrlw ((__v4hi)__m, (__v4hi)__count);
 673 }
 674
 675 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 676 _m_psrlw (__m64 __m, __m64 __count)
 677 {
 678   return _mm_srl_pi16 (__m, __count);
 679 }
 680
 681 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 682 _mm_srli_pi16 (__m64 __m, int __count)
 683 {
 684   return (__m64) __builtin_ia32_psrlwi ((__v4hi)__m, __count);
 685 }
 686
 687 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 688 _m_psrlwi (__m64 __m, int __count)
 689 {
 690   return _mm_srli_pi16 (__m, __count);
 691 }
 692
 693 /* Shift two 32-bit values in M right by COUNT; shift in zeros.  */
 694 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 695 _mm_srl_pi32 (__m64 __m, __m64 __count)
 696 {
 697   return (__m64) __builtin_ia32_psrld ((__v2si)__m, (__v2si)__count);
 698 }
 699
 700 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 701 _m_psrld (__m64 __m, __m64 __count)
 702 {
 703   return _mm_srl_pi32 (__m, __count);
 704 }
 705
 706 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 707 _mm_srli_pi32 (__m64 __m, int __count)
 708 {
 709   return (__m64) __builtin_ia32_psrldi ((__v2si)__m, __count);
 710 }
 711
 712 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 713 _m_psrldi (__m64 __m, int __count)
 714 {
 715   return _mm_srli_pi32 (__m, __count);
 716 }
 717
 718 /* Shift the 64-bit value in M left by COUNT; shift in zeros.  */
 719 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 720 _mm_srl_si64 (__m64 __m, __m64 __count)
 721 {
 722   return (__m64) __builtin_ia32_psrlq ((__v1di)__m, (__v1di)__count);
 723 }
 724
 725 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 726 _m_psrlq (__m64 __m, __m64 __count)
 727 {
 728   return _mm_srl_si64 (__m, __count);
 729 }
 730
 731 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 732 _mm_srli_si64 (__m64 __m, int __count)
 733 {
 734   return (__m64) __builtin_ia32_psrlqi ((__v1di)__m, __count);
 735 }
 736
 737 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 738 _m_psrlqi (__m64 __m, int __count)
 739 {
 740   return _mm_srli_si64 (__m, __count);
 741 }
 742
 743 /* Bit-wise AND the 64-bit values in M1 and M2.  */
 744 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 745 _mm_and_si64 (__m64 __m1, __m64 __m2)
 746 {
 747   return __builtin_ia32_pand (__m1, __m2);
 748 }
 749
 750 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 751 _m_pand (__m64 __m1, __m64 __m2)
 752 {
 753   return _mm_and_si64 (__m1, __m2);
 754 }
 755
 756 /* Bit-wise complement the 64-bit value in M1 and bit-wise AND it with the
 757    64-bit value in M2.  */
 758 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 759 _mm_andnot_si64 (__m64 __m1, __m64 __m2)
 760 {
 761   return __builtin_ia32_pandn (__m1, __m2);
 762 }
 763
 764 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 765 _m_pandn (__m64 __m1, __m64 __m2)
 766 {
 767   return _mm_andnot_si64 (__m1, __m2);
 768 }
 769
 770 /* Bit-wise inclusive OR the 64-bit values in M1 and M2.  */
 771 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 772 _mm_or_si64 (__m64 __m1, __m64 __m2)
 773 {
 774   return __builtin_ia32_por (__m1, __m2);
 775 }
 776
 777 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 778 _m_por (__m64 __m1, __m64 __m2)
 779 {
 780   return _mm_or_si64 (__m1, __m2);
 781 }
 782
 783 /* Bit-wise exclusive OR the 64-bit values in M1 and M2.  */
 784 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 785 _mm_xor_si64 (__m64 __m1, __m64 __m2)
 786 {
 787   return __builtin_ia32_pxor (__m1, __m2);
 788 }
 789
 790 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 791 _m_pxor (__m64 __m1, __m64 __m2)
 792 {
 793   return _mm_xor_si64 (__m1, __m2);
 794 }
 795
 796 /* Compare eight 8-bit values.  The result of the comparison is 0xFF if the
 797    test is true and zero if false.  */
 798 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 799 _mm_cmpeq_pi8 (__m64 __m1, __m64 __m2)
 800 {
 801   return (__m64) __builtin_ia32_pcmpeqb ((__v8qi)__m1, (__v8qi)__m2);
 802 }
 803
 804 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 805 _m_pcmpeqb (__m64 __m1, __m64 __m2)
 806 {
 807   return _mm_cmpeq_pi8 (__m1, __m2);
 808 }
 809
 810 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 811 _mm_cmpgt_pi8 (__m64 __m1, __m64 __m2)
 812 {
 813   return (__m64) __builtin_ia32_pcmpgtb ((__v8qi)__m1, (__v8qi)__m2);
 814 }
 815
 816 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 817 _m_pcmpgtb (__m64 __m1, __m64 __m2)
 818 {
 819   return _mm_cmpgt_pi8 (__m1, __m2);
 820 }
 821
 822 /* Compare four 16-bit values.  The result of the comparison is 0xFFFF if
 823    the test is true and zero if false.  */
 824 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 825 _mm_cmpeq_pi16 (__m64 __m1, __m64 __m2)
 826 {
 827   return (__m64) __builtin_ia32_pcmpeqw ((__v4hi)__m1, (__v4hi)__m2);
 828 }
 829
 830 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 831 _m_pcmpeqw (__m64 __m1, __m64 __m2)
 832 {
 833   return _mm_cmpeq_pi16 (__m1, __m2);
 834 }
 835
 836 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 837 _mm_cmpgt_pi16 (__m64 __m1, __m64 __m2)
 838 {
 839   return (__m64) __builtin_ia32_pcmpgtw ((__v4hi)__m1, (__v4hi)__m2);
 840 }
 841
 842 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 843 _m_pcmpgtw (__m64 __m1, __m64 __m2)
 844 {
 845   return _mm_cmpgt_pi16 (__m1, __m2);
 846 }
 847
 848 /* Compare two 32-bit values.  The result of the comparison is 0xFFFFFFFF if
 849    the test is true and zero if false.  */
 850 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 851 _mm_cmpeq_pi32 (__m64 __m1, __m64 __m2)
 852 {
 853   return (__m64) __builtin_ia32_pcmpeqd ((__v2si)__m1, (__v2si)__m2);
 854 }
 855
 856 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 857 _m_pcmpeqd (__m64 __m1, __m64 __m2)
 858 {
 859   return _mm_cmpeq_pi32 (__m1, __m2);
 860 }
 861
 862 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 863 _mm_cmpgt_pi32 (__m64 __m1, __m64 __m2)
 864 {
 865   return (__m64) __builtin_ia32_pcmpgtd ((__v2si)__m1, (__v2si)__m2);
 866 }
 867
 868 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 869 _m_pcmpgtd (__m64 __m1, __m64 __m2)
 870 {
 871   return _mm_cmpgt_pi32 (__m1, __m2);
 872 }
 873
 874 /* Creates a 64-bit zero.  */
 875 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 876 _mm_setzero_si64 (void)
 877 {
 878   return (__m64)0LL;
 879 }
 880
 881 /* Creates a vector of two 32-bit values; I0 is least significant.  */
 882 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 883 _mm_set_pi32 (int __i1, int __i0)
 884 {
 885   return (__m64) __builtin_ia32_vec_init_v2si (__i0, __i1);
 886 }
 887
 888 /* Creates a vector of four 16-bit values; W0 is least significant.  */
 889 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 890 _mm_set_pi16 (short __w3, short __w2, short __w1, short __w0)
 891 {
 892   return (__m64) __builtin_ia32_vec_init_v4hi (__w0, __w1, __w2, __w3);
 893 }
 894
 895 /* Creates a vector of eight 8-bit values; B0 is least significant.  */
 896 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 897 _mm_set_pi8 (char __b7, char __b6, char __b5, char __b4,
 898              char __b3, char __b2, char __b1, char __b0)
 899 {
 900   return (__m64) __builtin_ia32_vec_init_v8qi (__b0, __b1, __b2, __b3,
 901                                                __b4, __b5, __b6, __b7);
 902 }
 903
 904 /* Similar, but with the arguments in reverse order.  */
 905 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 906 _mm_setr_pi32 (int __i0, int __i1)
 907 {
 908   return _mm_set_pi32 (__i1, __i0);
 909 }
 910
 911 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 912 _mm_setr_pi16 (short __w0, short __w1, short __w2, short __w3)
 913 {
 914   return _mm_set_pi16 (__w3, __w2, __w1, __w0);
 915 }
 916
 917 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 918 _mm_setr_pi8 (char __b0, char __b1, char __b2, char __b3,
 919               char __b4, char __b5, char __b6, char __b7)
 920 {
 921   return _mm_set_pi8 (__b7, __b6, __b5, __b4, __b3, __b2, __b1, __b0);
 922 }
 923
 924 /* Creates a vector of two 32-bit values, both elements containing I.  */
 925 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 926 _mm_set1_pi32 (int __i)
 927 {
 928   return _mm_set_pi32 (__i, __i);
 929 }
 930
 931 /* Creates a vector of four 16-bit values, all elements containing W.  */
 932 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 933 _mm_set1_pi16 (short __w)
 934 {
 935   return _mm_set_pi16 (__w, __w, __w, __w);
 936 }
 937
 938 /* Creates a vector of eight 8-bit values, all elements containing B.  */
 939 extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 940 _mm_set1_pi8 (char __b)
 941 {
 942   return _mm_set_pi8 (__b, __b, __b, __b, __b, __b, __b, __b);
 943 }
 944 #ifdef __DISABLE_MMX__
 945 #undef __DISABLE_MMX__
 946 #pragma GCC pop_options
 947 #endif /* __DISABLE_MMX__ */
 948
 949 #endif /* _MMINTRIN_H_INCLUDED */