main/gcc/config/i386/avx2intrin.h

   1 /* Copyright (C) 2011-2014 Free Software Foundation, Inc.
   2
   3    This file is part of GCC.
   4
   5    GCC is free software; you can redistribute it and/or modify
   6    it under the terms of the GNU General Public License as published by
   7    the Free Software Foundation; either version 3, or (at your option)
   8    any later version.
   9
  10    GCC is distributed in the hope that it will be useful,
  11    but WITHOUT ANY WARRANTY; without even the implied warranty of
  12    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13    GNU General Public License for more details.
  14
  15    Under Section 7 of GPL version 3, you are granted additional
  16    permissions described in the GCC Runtime Library Exception, version
  17    3.1, as published by the Free Software Foundation.
  18
  19    You should have received a copy of the GNU General Public License and
  20    a copy of the GCC Runtime Library Exception along with this program;
  21    see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
  22    <http://www.gnu.org/licenses/>.  */
  23
  24 #ifndef _IMMINTRIN_H_INCLUDED
  25 # error "Never use <avx2intrin.h> directly; include <immintrin.h> instead."
  26 #endif
  27
  28 #ifndef _AVX2INTRIN_H_INCLUDED
  29 #define _AVX2INTRIN_H_INCLUDED
  30
  31 #ifndef __AVX2__
  32 #pragma GCC push_options
  33 #pragma GCC target("avx2")
  34 #define __DISABLE_AVX2__
  35 #endif /* __AVX2__ */
  36
  37 /* Sum absolute 8-bit integer difference of adjacent groups of 4
  38    byte integers in the first 2 operands.  Starting offsets within
  39    operands are determined by the 3rd mask operand.  */
  40 #ifdef __OPTIMIZE__
  41 extern __inline __m256i
  42 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
  43 _mm256_mpsadbw_epu8 (__m256i __X, __m256i __Y, const int __M)
  44 {
  45   return (__m256i) __builtin_ia32_mpsadbw256 ((__v32qi)__X,
  46                                               (__v32qi)__Y, __M);
  47 }
  48 #else
  49 #define _mm256_mpsadbw_epu8(X, Y, M)                                    \
  50   ((__m256i) __builtin_ia32_mpsadbw256 ((__v32qi)(__m256i)(X),          \
  51                                         (__v32qi)(__m256i)(Y), (int)(M)))
  52 #endif
  53
  54 extern __inline __m256i
  55 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
  56 _mm256_abs_epi8 (__m256i __A)
  57 {
  58   return (__m256i)__builtin_ia32_pabsb256 ((__v32qi)__A);
  59 }
  60
  61 extern __inline __m256i
  62 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
  63 _mm256_abs_epi16 (__m256i __A)
  64 {
  65   return (__m256i)__builtin_ia32_pabsw256 ((__v16hi)__A);
  66 }
  67
  68 extern __inline __m256i
  69 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
  70 _mm256_abs_epi32 (__m256i __A)
  71 {
  72   return (__m256i)__builtin_ia32_pabsd256 ((__v8si)__A);
  73 }
  74
  75 extern __inline __m256i
  76 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
  77 _mm256_packs_epi32 (__m256i __A, __m256i __B)
  78 {
  79   return (__m256i)__builtin_ia32_packssdw256 ((__v8si)__A, (__v8si)__B);
  80 }
  81
  82 extern __inline __m256i
  83 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
  84 _mm256_packs_epi16 (__m256i __A, __m256i __B)
  85 {
  86   return (__m256i)__builtin_ia32_packsswb256 ((__v16hi)__A, (__v16hi)__B);
  87 }
  88
  89 extern __inline __m256i
  90 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
  91 _mm256_packus_epi32 (__m256i __A, __m256i __B)
  92 {
  93   return (__m256i)__builtin_ia32_packusdw256 ((__v8si)__A, (__v8si)__B);
  94 }
  95
  96 extern __inline __m256i
  97 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
  98 _mm256_packus_epi16 (__m256i __A, __m256i __B)
  99 {
 100   return (__m256i)__builtin_ia32_packuswb256 ((__v16hi)__A, (__v16hi)__B);
 101 }
 102
 103 extern __inline __m256i
 104 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 105 _mm256_add_epi8 (__m256i __A, __m256i __B)
 106 {
 107   return (__m256i)__builtin_ia32_paddb256 ((__v32qi)__A, (__v32qi)__B);
 108 }
 109
 110 extern __inline __m256i
 111 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 112 _mm256_add_epi16 (__m256i __A, __m256i __B)
 113 {
 114   return (__m256i)__builtin_ia32_paddw256 ((__v16hi)__A, (__v16hi)__B);
 115 }
 116
 117 extern __inline __m256i
 118 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 119 _mm256_add_epi32 (__m256i __A, __m256i __B)
 120 {
 121   return (__m256i)__builtin_ia32_paddd256 ((__v8si)__A, (__v8si)__B);
 122 }
 123
 124 extern __inline __m256i
 125 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 126 _mm256_add_epi64 (__m256i __A, __m256i __B)
 127 {
 128   return (__m256i)__builtin_ia32_paddq256 ((__v4di)__A, (__v4di)__B);
 129 }
 130
 131 extern __inline __m256i
 132 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 133 _mm256_adds_epi8 (__m256i __A, __m256i __B)
 134 {
 135   return (__m256i)__builtin_ia32_paddsb256 ((__v32qi)__A, (__v32qi)__B);
 136 }
 137
 138 extern __inline __m256i
 139 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 140 _mm256_adds_epi16 (__m256i __A, __m256i __B)
 141 {
 142   return (__m256i)__builtin_ia32_paddsw256 ((__v16hi)__A, (__v16hi)__B);
 143 }
 144
 145 extern __inline __m256i
 146 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 147 _mm256_adds_epu8 (__m256i __A, __m256i __B)
 148 {
 149   return (__m256i)__builtin_ia32_paddusb256 ((__v32qi)__A, (__v32qi)__B);
 150 }
 151
 152 extern __inline __m256i
 153 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 154 _mm256_adds_epu16 (__m256i __A, __m256i __B)
 155 {
 156   return (__m256i)__builtin_ia32_paddusw256 ((__v16hi)__A, (__v16hi)__B);
 157 }
 158
 159 #ifdef __OPTIMIZE__
 160 extern __inline __m256i
 161 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 162 _mm256_alignr_epi8 (__m256i __A, __m256i __B, const int __N)
 163 {
 164   return (__m256i) __builtin_ia32_palignr256 ((__v4di)__A,
 165                                               (__v4di)__B,
 166                                               __N * 8);
 167 }
 168 #else
 169 /* In that case (__N*8) will be in vreg, and insn will not be matched. */
 170 /* Use define instead */
 171 #define _mm256_alignr_epi8(A, B, N)                                \
 172   ((__m256i) __builtin_ia32_palignr256 ((__v4di)(__m256i)(A),      \
 173                                         (__v4di)(__m256i)(B),      \
 174                                         (int)(N) * 8))
 175 #endif
 176
 177 extern __inline __m256i
 178 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 179 _mm256_and_si256 (__m256i __A, __m256i __B)
 180 {
 181   return (__m256i) __builtin_ia32_andsi256 ((__v4di)__A, (__v4di)__B);
 182 }
 183
 184 extern __inline __m256i
 185 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 186 _mm256_andnot_si256 (__m256i __A, __m256i __B)
 187 {
 188   return (__m256i) __builtin_ia32_andnotsi256 ((__v4di)__A, (__v4di)__B);
 189 }
 190
 191 extern __inline __m256i
 192 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 193 _mm256_avg_epu8 (__m256i __A, __m256i __B)
 194 {
 195   return (__m256i)__builtin_ia32_pavgb256 ((__v32qi)__A, (__v32qi)__B);
 196 }
 197
 198 extern __inline __m256i
 199 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 200 _mm256_avg_epu16 (__m256i __A, __m256i __B)
 201 {
 202   return (__m256i)__builtin_ia32_pavgw256 ((__v16hi)__A, (__v16hi)__B);
 203 }
 204
 205 extern __inline __m256i
 206 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 207 _mm256_blendv_epi8 (__m256i __X, __m256i __Y, __m256i __M)
 208 {
 209   return (__m256i) __builtin_ia32_pblendvb256 ((__v32qi)__X,
 210                                                (__v32qi)__Y,
 211                                                (__v32qi)__M);
 212 }
 213
 214 #ifdef __OPTIMIZE__
 215 extern __inline __m256i
 216 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 217 _mm256_blend_epi16 (__m256i __X, __m256i __Y, const int __M)
 218 {
 219   return (__m256i) __builtin_ia32_pblendw256 ((__v16hi)__X,
 220                                               (__v16hi)__Y,
 221                                                __M);
 222 }
 223 #else
 224 #define _mm256_blend_epi16(X, Y, M)                                     \
 225   ((__m256i) __builtin_ia32_pblendw256 ((__v16hi)(__m256i)(X),          \
 226                                         (__v16hi)(__m256i)(Y), (int)(M)))
 227 #endif
 228
 229 extern __inline __m256i
 230 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 231 _mm256_cmpeq_epi8 (__m256i __A, __m256i __B)
 232 {
 233   return (__m256i)__builtin_ia32_pcmpeqb256 ((__v32qi)__A, (__v32qi)__B);
 234 }
 235
 236 extern __inline __m256i
 237 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 238 _mm256_cmpeq_epi16 (__m256i __A, __m256i __B)
 239 {
 240   return (__m256i)__builtin_ia32_pcmpeqw256 ((__v16hi)__A, (__v16hi)__B);
 241 }
 242
 243 extern __inline __m256i
 244 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 245 _mm256_cmpeq_epi32 (__m256i __A, __m256i __B)
 246 {
 247   return (__m256i)__builtin_ia32_pcmpeqd256 ((__v8si)__A, (__v8si)__B);
 248 }
 249
 250 extern __inline __m256i
 251 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 252 _mm256_cmpeq_epi64 (__m256i __A, __m256i __B)
 253 {
 254   return (__m256i)__builtin_ia32_pcmpeqq256 ((__v4di)__A, (__v4di)__B);
 255 }
 256
 257 extern __inline __m256i
 258 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 259 _mm256_cmpgt_epi8 (__m256i __A, __m256i __B)
 260 {
 261   return (__m256i)__builtin_ia32_pcmpgtb256 ((__v32qi)__A,
 262                                              (__v32qi)__B);
 263 }
 264
 265 extern __inline __m256i
 266 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 267 _mm256_cmpgt_epi16 (__m256i __A, __m256i __B)
 268 {
 269   return (__m256i)__builtin_ia32_pcmpgtw256 ((__v16hi)__A,
 270                                              (__v16hi)__B);
 271 }
 272
 273 extern __inline __m256i
 274 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 275 _mm256_cmpgt_epi32 (__m256i __A, __m256i __B)
 276 {
 277   return (__m256i)__builtin_ia32_pcmpgtd256 ((__v8si)__A,
 278                                              (__v8si)__B);
 279 }
 280
 281 extern __inline __m256i
 282 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 283 _mm256_cmpgt_epi64 (__m256i __A, __m256i __B)
 284 {
 285   return (__m256i)__builtin_ia32_pcmpgtq256 ((__v4di)__A, (__v4di)__B);
 286 }
 287
 288 extern __inline __m256i
 289 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 290 _mm256_hadd_epi16 (__m256i __X, __m256i __Y)
 291 {
 292   return (__m256i) __builtin_ia32_phaddw256 ((__v16hi)__X,
 293                                              (__v16hi)__Y);
 294 }
 295
 296 extern __inline __m256i
 297 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 298 _mm256_hadd_epi32 (__m256i __X, __m256i __Y)
 299 {
 300   return (__m256i) __builtin_ia32_phaddd256 ((__v8si)__X, (__v8si)__Y);
 301 }
 302
 303 extern __inline __m256i
 304 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 305 _mm256_hadds_epi16 (__m256i __X, __m256i __Y)
 306 {
 307   return (__m256i) __builtin_ia32_phaddsw256 ((__v16hi)__X,
 308                                               (__v16hi)__Y);
 309 }
 310
 311 extern __inline __m256i
 312 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 313 _mm256_hsub_epi16 (__m256i __X, __m256i __Y)
 314 {
 315   return (__m256i) __builtin_ia32_phsubw256 ((__v16hi)__X,
 316                                              (__v16hi)__Y);
 317 }
 318
 319 extern __inline __m256i
 320 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 321 _mm256_hsub_epi32 (__m256i __X, __m256i __Y)
 322 {
 323   return (__m256i) __builtin_ia32_phsubd256 ((__v8si)__X, (__v8si)__Y);
 324 }
 325
 326 extern __inline __m256i
 327 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 328 _mm256_hsubs_epi16 (__m256i __X, __m256i __Y)
 329 {
 330   return (__m256i) __builtin_ia32_phsubsw256 ((__v16hi)__X,
 331                                               (__v16hi)__Y);
 332 }
 333
 334 extern __inline __m256i
 335 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 336 _mm256_maddubs_epi16 (__m256i __X, __m256i __Y)
 337 {
 338   return (__m256i) __builtin_ia32_pmaddubsw256 ((__v32qi)__X,
 339                                                 (__v32qi)__Y);
 340 }
 341
 342 extern __inline __m256i
 343 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 344 _mm256_madd_epi16 (__m256i __A, __m256i __B)
 345 {
 346   return (__m256i)__builtin_ia32_pmaddwd256 ((__v16hi)__A,
 347                                              (__v16hi)__B);
 348 }
 349
 350 extern __inline __m256i
 351 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 352 _mm256_max_epi8 (__m256i __A, __m256i __B)
 353 {
 354   return (__m256i)__builtin_ia32_pmaxsb256 ((__v32qi)__A, (__v32qi)__B);
 355 }
 356
 357 extern __inline __m256i
 358 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 359 _mm256_max_epi16 (__m256i __A, __m256i __B)
 360 {
 361   return (__m256i)__builtin_ia32_pmaxsw256 ((__v16hi)__A, (__v16hi)__B);
 362 }
 363
 364 extern __inline __m256i
 365 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 366 _mm256_max_epi32 (__m256i __A, __m256i __B)
 367 {
 368   return (__m256i)__builtin_ia32_pmaxsd256 ((__v8si)__A, (__v8si)__B);
 369 }
 370
 371 extern __inline __m256i
 372 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 373 _mm256_max_epu8 (__m256i __A, __m256i __B)
 374 {
 375   return (__m256i)__builtin_ia32_pmaxub256 ((__v32qi)__A, (__v32qi)__B);
 376 }
 377
 378 extern __inline __m256i
 379 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 380 _mm256_max_epu16 (__m256i __A, __m256i __B)
 381 {
 382   return (__m256i)__builtin_ia32_pmaxuw256 ((__v16hi)__A, (__v16hi)__B);
 383 }
 384
 385 extern __inline __m256i
 386 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 387 _mm256_max_epu32 (__m256i __A, __m256i __B)
 388 {
 389   return (__m256i)__builtin_ia32_pmaxud256 ((__v8si)__A, (__v8si)__B);
 390 }
 391
 392 extern __inline __m256i
 393 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 394 _mm256_min_epi8 (__m256i __A, __m256i __B)
 395 {
 396   return (__m256i)__builtin_ia32_pminsb256 ((__v32qi)__A, (__v32qi)__B);
 397 }
 398
 399 extern __inline __m256i
 400 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 401 _mm256_min_epi16 (__m256i __A, __m256i __B)
 402 {
 403   return (__m256i)__builtin_ia32_pminsw256 ((__v16hi)__A, (__v16hi)__B);
 404 }
 405
 406 extern __inline __m256i
 407 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 408 _mm256_min_epi32 (__m256i __A, __m256i __B)
 409 {
 410   return (__m256i)__builtin_ia32_pminsd256 ((__v8si)__A, (__v8si)__B);
 411 }
 412
 413 extern __inline __m256i
 414 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 415 _mm256_min_epu8 (__m256i __A, __m256i __B)
 416 {
 417   return (__m256i)__builtin_ia32_pminub256 ((__v32qi)__A, (__v32qi)__B);
 418 }
 419
 420 extern __inline __m256i
 421 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 422 _mm256_min_epu16 (__m256i __A, __m256i __B)
 423 {
 424   return (__m256i)__builtin_ia32_pminuw256 ((__v16hi)__A, (__v16hi)__B);
 425 }
 426
 427 extern __inline __m256i
 428 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 429 _mm256_min_epu32 (__m256i __A, __m256i __B)
 430 {
 431   return (__m256i)__builtin_ia32_pminud256 ((__v8si)__A, (__v8si)__B);
 432 }
 433
 434 extern __inline int
 435 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 436 _mm256_movemask_epi8 (__m256i __A)
 437 {
 438   return __builtin_ia32_pmovmskb256 ((__v32qi)__A);
 439 }
 440
 441 extern __inline __m256i
 442 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 443 _mm256_cvtepi8_epi16 (__m128i __X)
 444 {
 445   return (__m256i) __builtin_ia32_pmovsxbw256 ((__v16qi)__X);
 446 }
 447
 448 extern __inline __m256i
 449 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 450 _mm256_cvtepi8_epi32 (__m128i __X)
 451 {
 452   return (__m256i) __builtin_ia32_pmovsxbd256 ((__v16qi)__X);
 453 }
 454
 455 extern __inline __m256i
 456 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 457 _mm256_cvtepi8_epi64 (__m128i __X)
 458 {
 459   return (__m256i) __builtin_ia32_pmovsxbq256 ((__v16qi)__X);
 460 }
 461
 462 extern __inline __m256i
 463 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 464 _mm256_cvtepi16_epi32 (__m128i __X)
 465 {
 466   return (__m256i) __builtin_ia32_pmovsxwd256 ((__v8hi)__X);
 467 }
 468
 469 extern __inline __m256i
 470 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 471 _mm256_cvtepi16_epi64 (__m128i __X)
 472 {
 473   return (__m256i) __builtin_ia32_pmovsxwq256 ((__v8hi)__X);
 474 }
 475
 476 extern __inline __m256i
 477 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 478 _mm256_cvtepi32_epi64 (__m128i __X)
 479 {
 480   return (__m256i) __builtin_ia32_pmovsxdq256 ((__v4si)__X);
 481 }
 482
 483 extern __inline __m256i
 484 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 485 _mm256_cvtepu8_epi16 (__m128i __X)
 486 {
 487   return (__m256i) __builtin_ia32_pmovzxbw256 ((__v16qi)__X);
 488 }
 489
 490 extern __inline __m256i
 491 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 492 _mm256_cvtepu8_epi32 (__m128i __X)
 493 {
 494   return (__m256i) __builtin_ia32_pmovzxbd256 ((__v16qi)__X);
 495 }
 496
 497 extern __inline __m256i
 498 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 499 _mm256_cvtepu8_epi64 (__m128i __X)
 500 {
 501   return (__m256i) __builtin_ia32_pmovzxbq256 ((__v16qi)__X);
 502 }
 503
 504 extern __inline __m256i
 505 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 506 _mm256_cvtepu16_epi32 (__m128i __X)
 507 {
 508   return (__m256i) __builtin_ia32_pmovzxwd256 ((__v8hi)__X);
 509 }
 510
 511 extern __inline __m256i
 512 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 513 _mm256_cvtepu16_epi64 (__m128i __X)
 514 {
 515   return (__m256i) __builtin_ia32_pmovzxwq256 ((__v8hi)__X);
 516 }
 517
 518 extern __inline __m256i
 519 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 520 _mm256_cvtepu32_epi64 (__m128i __X)
 521 {
 522   return (__m256i) __builtin_ia32_pmovzxdq256 ((__v4si)__X);
 523 }
 524
 525 extern __inline __m256i
 526 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 527 _mm256_mul_epi32 (__m256i __X, __m256i __Y)
 528 {
 529   return (__m256i) __builtin_ia32_pmuldq256 ((__v8si)__X, (__v8si)__Y);
 530 }
 531
 532 extern __inline __m256i
 533 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 534 _mm256_mulhrs_epi16 (__m256i __X, __m256i __Y)
 535 {
 536   return (__m256i) __builtin_ia32_pmulhrsw256 ((__v16hi)__X,
 537                                                (__v16hi)__Y);
 538 }
 539
 540 extern __inline __m256i
 541 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 542 _mm256_mulhi_epu16 (__m256i __A, __m256i __B)
 543 {
 544   return (__m256i)__builtin_ia32_pmulhuw256 ((__v16hi)__A, (__v16hi)__B);
 545 }
 546
 547 extern __inline __m256i
 548 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 549 _mm256_mulhi_epi16 (__m256i __A, __m256i __B)
 550 {
 551   return (__m256i)__builtin_ia32_pmulhw256 ((__v16hi)__A, (__v16hi)__B);
 552 }
 553
 554 extern __inline __m256i
 555 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 556 _mm256_mullo_epi16 (__m256i __A, __m256i __B)
 557 {
 558   return (__m256i)__builtin_ia32_pmullw256 ((__v16hi)__A, (__v16hi)__B);
 559 }
 560
 561 extern __inline __m256i
 562 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 563 _mm256_mullo_epi32 (__m256i __A, __m256i __B)
 564 {
 565   return (__m256i)__builtin_ia32_pmulld256 ((__v8si)__A, (__v8si)__B);
 566 }
 567
 568 extern __inline __m256i
 569 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 570 _mm256_mul_epu32 (__m256i __A, __m256i __B)
 571 {
 572   return (__m256i)__builtin_ia32_pmuludq256 ((__v8si)__A, (__v8si)__B);
 573 }
 574
 575 extern __inline __m256i
 576 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 577 _mm256_or_si256 (__m256i __A, __m256i __B)
 578 {
 579   return (__m256i)__builtin_ia32_por256 ((__v4di)__A, (__v4di)__B);
 580 }
 581
 582 extern __inline __m256i
 583 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 584 _mm256_sad_epu8 (__m256i __A, __m256i __B)
 585 {
 586   return (__m256i)__builtin_ia32_psadbw256 ((__v32qi)__A, (__v32qi)__B);
 587 }
 588
 589 extern __inline __m256i
 590 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 591 _mm256_shuffle_epi8 (__m256i __X, __m256i __Y)
 592 {
 593   return (__m256i) __builtin_ia32_pshufb256 ((__v32qi)__X,
 594                                              (__v32qi)__Y);
 595 }
 596
 597 #ifdef __OPTIMIZE__
 598 extern __inline __m256i
 599 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 600 _mm256_shuffle_epi32 (__m256i __A, const int __mask)
 601 {
 602   return (__m256i)__builtin_ia32_pshufd256 ((__v8si)__A, __mask);
 603 }
 604
 605 extern __inline __m256i
 606 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 607 _mm256_shufflehi_epi16 (__m256i __A, const int __mask)
 608 {
 609   return (__m256i)__builtin_ia32_pshufhw256 ((__v16hi)__A, __mask);
 610 }
 611
 612 extern __inline __m256i
 613 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 614 _mm256_shufflelo_epi16 (__m256i __A, const int __mask)
 615 {
 616   return (__m256i)__builtin_ia32_pshuflw256 ((__v16hi)__A, __mask);
 617 }
 618 #else
 619 #define _mm256_shuffle_epi32(A, N) \
 620   ((__m256i)__builtin_ia32_pshufd256 ((__v8si)(__m256i)(A), (int)(N)))
 621 #define _mm256_shufflehi_epi16(A, N) \
 622   ((__m256i)__builtin_ia32_pshufhw256 ((__v16hi)(__m256i)(A), (int)(N)))
 623 #define _mm256_shufflelo_epi16(A, N) \
 624   ((__m256i)__builtin_ia32_pshuflw256 ((__v16hi)(__m256i)(A), (int)(N)))
 625 #endif
 626
 627 extern __inline __m256i
 628 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 629 _mm256_sign_epi8 (__m256i __X, __m256i __Y)
 630 {
 631   return (__m256i) __builtin_ia32_psignb256 ((__v32qi)__X, (__v32qi)__Y);
 632 }
 633
 634 extern __inline __m256i
 635 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 636 _mm256_sign_epi16 (__m256i __X, __m256i __Y)
 637 {
 638   return (__m256i) __builtin_ia32_psignw256 ((__v16hi)__X, (__v16hi)__Y);
 639 }
 640
 641 extern __inline __m256i
 642 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 643 _mm256_sign_epi32 (__m256i __X, __m256i __Y)
 644 {
 645   return (__m256i) __builtin_ia32_psignd256 ((__v8si)__X, (__v8si)__Y);
 646 }
 647
 648 #ifdef __OPTIMIZE__
 649 extern __inline __m256i
 650 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 651 _mm256_slli_si256 (__m256i __A, const int __N)
 652 {
 653   return (__m256i)__builtin_ia32_pslldqi256 (__A, __N * 8);
 654 }
 655 #else
 656 #define _mm256_slli_si256(A, N) \
 657   ((__m256i)__builtin_ia32_pslldqi256 ((__m256i)(A), (int)(N) * 8))
 658 #endif
 659
 660 extern __inline __m256i
 661 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 662 _mm256_slli_epi16 (__m256i __A, int __B)
 663 {
 664   return (__m256i)__builtin_ia32_psllwi256 ((__v16hi)__A, __B);
 665 }
 666
 667 extern __inline __m256i
 668 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 669 _mm256_sll_epi16 (__m256i __A, __m128i __B)
 670 {
 671   return (__m256i)__builtin_ia32_psllw256((__v16hi)__A, (__v8hi)__B);
 672 }
 673
 674 extern __inline __m256i
 675 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 676 _mm256_slli_epi32 (__m256i __A, int __B)
 677 {
 678   return (__m256i)__builtin_ia32_pslldi256 ((__v8si)__A, __B);
 679 }
 680
 681 extern __inline __m256i
 682 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 683 _mm256_sll_epi32 (__m256i __A, __m128i __B)
 684 {
 685   return (__m256i)__builtin_ia32_pslld256((__v8si)__A, (__v4si)__B);
 686 }
 687
 688 extern __inline __m256i
 689 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 690 _mm256_slli_epi64 (__m256i __A, int __B)
 691 {
 692   return (__m256i)__builtin_ia32_psllqi256 ((__v4di)__A, __B);
 693 }
 694
 695 extern __inline __m256i
 696 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 697 _mm256_sll_epi64 (__m256i __A, __m128i __B)
 698 {
 699   return (__m256i)__builtin_ia32_psllq256((__v4di)__A, (__v2di)__B);
 700 }
 701
 702 extern __inline __m256i
 703 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 704 _mm256_srai_epi16 (__m256i __A, int __B)
 705 {
 706   return (__m256i)__builtin_ia32_psrawi256 ((__v16hi)__A, __B);
 707 }
 708
 709 extern __inline __m256i
 710 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 711 _mm256_sra_epi16 (__m256i __A, __m128i __B)
 712 {
 713   return (__m256i)__builtin_ia32_psraw256 ((__v16hi)__A, (__v8hi)__B);
 714 }
 715
 716 extern __inline __m256i
 717 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 718 _mm256_srai_epi32 (__m256i __A, int __B)
 719 {
 720   return (__m256i)__builtin_ia32_psradi256 ((__v8si)__A, __B);
 721 }
 722
 723 extern __inline __m256i
 724 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 725 _mm256_sra_epi32 (__m256i __A, __m128i __B)
 726 {
 727   return (__m256i)__builtin_ia32_psrad256 ((__v8si)__A, (__v4si)__B);
 728 }
 729
 730 #ifdef __OPTIMIZE__
 731 extern __inline __m256i
 732 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 733 _mm256_srli_si256 (__m256i __A, const int __N)
 734 {
 735   return (__m256i)__builtin_ia32_psrldqi256 (__A, __N * 8);
 736 }
 737 #else
 738 #define _mm256_srli_si256(A, N) \
 739   ((__m256i)__builtin_ia32_psrldqi256 ((__m256i)(A), (int)(N) * 8))
 740 #endif
 741
 742 extern __inline __m256i
 743 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 744 _mm256_srli_epi16 (__m256i __A, int __B)
 745 {
 746   return (__m256i)__builtin_ia32_psrlwi256 ((__v16hi)__A, __B);
 747 }
 748
 749 extern __inline __m256i
 750 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 751 _mm256_srl_epi16 (__m256i __A, __m128i __B)
 752 {
 753   return (__m256i)__builtin_ia32_psrlw256((__v16hi)__A, (__v8hi)__B);
 754 }
 755
 756 extern __inline __m256i
 757 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 758 _mm256_srli_epi32 (__m256i __A, int __B)
 759 {
 760   return (__m256i)__builtin_ia32_psrldi256 ((__v8si)__A, __B);
 761 }
 762
 763 extern __inline __m256i
 764 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 765 _mm256_srl_epi32 (__m256i __A, __m128i __B)
 766 {
 767   return (__m256i)__builtin_ia32_psrld256((__v8si)__A, (__v4si)__B);
 768 }
 769
 770 extern __inline __m256i
 771 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 772 _mm256_srli_epi64 (__m256i __A, int __B)
 773 {
 774   return (__m256i)__builtin_ia32_psrlqi256 ((__v4di)__A, __B);
 775 }
 776
 777 extern __inline __m256i
 778 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 779 _mm256_srl_epi64 (__m256i __A, __m128i __B)
 780 {
 781   return (__m256i)__builtin_ia32_psrlq256((__v4di)__A, (__v2di)__B);
 782 }
 783
 784 extern __inline __m256i
 785 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 786 _mm256_sub_epi8 (__m256i __A, __m256i __B)
 787 {
 788   return (__m256i)__builtin_ia32_psubb256 ((__v32qi)__A, (__v32qi)__B);
 789 }
 790
 791 extern __inline __m256i
 792 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 793 _mm256_sub_epi16 (__m256i __A, __m256i __B)
 794 {
 795   return (__m256i)__builtin_ia32_psubw256 ((__v16hi)__A, (__v16hi)__B);
 796 }
 797
 798 extern __inline __m256i
 799 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 800 _mm256_sub_epi32 (__m256i __A, __m256i __B)
 801 {
 802   return (__m256i)__builtin_ia32_psubd256 ((__v8si)__A, (__v8si)__B);
 803 }
 804
 805 extern __inline __m256i
 806 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 807 _mm256_sub_epi64 (__m256i __A, __m256i __B)
 808 {
 809   return (__m256i)__builtin_ia32_psubq256 ((__v4di)__A, (__v4di)__B);
 810 }
 811
 812 extern __inline __m256i
 813 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 814 _mm256_subs_epi8 (__m256i __A, __m256i __B)
 815 {
 816   return (__m256i)__builtin_ia32_psubsb256 ((__v32qi)__A, (__v32qi)__B);
 817 }
 818
 819 extern __inline __m256i
 820 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 821 _mm256_subs_epi16 (__m256i __A, __m256i __B)
 822 {
 823   return (__m256i)__builtin_ia32_psubsw256 ((__v16hi)__A, (__v16hi)__B);
 824 }
 825
 826 extern __inline __m256i
 827 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 828 _mm256_subs_epu8 (__m256i __A, __m256i __B)
 829 {
 830   return (__m256i)__builtin_ia32_psubusb256 ((__v32qi)__A, (__v32qi)__B);
 831 }
 832
 833 extern __inline __m256i
 834 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 835 _mm256_subs_epu16 (__m256i __A, __m256i __B)
 836 {
 837   return (__m256i)__builtin_ia32_psubusw256 ((__v16hi)__A, (__v16hi)__B);
 838 }
 839
 840 extern __inline __m256i
 841 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 842 _mm256_unpackhi_epi8 (__m256i __A, __m256i __B)
 843 {
 844   return (__m256i)__builtin_ia32_punpckhbw256 ((__v32qi)__A, (__v32qi)__B);
 845 }
 846
 847 extern __inline __m256i
 848 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 849 _mm256_unpackhi_epi16 (__m256i __A, __m256i __B)
 850 {
 851   return (__m256i)__builtin_ia32_punpckhwd256 ((__v16hi)__A, (__v16hi)__B);
 852 }
 853
 854 extern __inline __m256i
 855 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 856 _mm256_unpackhi_epi32 (__m256i __A, __m256i __B)
 857 {
 858   return (__m256i)__builtin_ia32_punpckhdq256 ((__v8si)__A, (__v8si)__B);
 859 }
 860
 861 extern __inline __m256i
 862 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 863 _mm256_unpackhi_epi64 (__m256i __A, __m256i __B)
 864 {
 865   return (__m256i)__builtin_ia32_punpckhqdq256 ((__v4di)__A, (__v4di)__B);
 866 }
 867
 868 extern __inline __m256i
 869 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 870 _mm256_unpacklo_epi8 (__m256i __A, __m256i __B)
 871 {
 872   return (__m256i)__builtin_ia32_punpcklbw256 ((__v32qi)__A, (__v32qi)__B);
 873 }
 874
 875 extern __inline __m256i
 876 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 877 _mm256_unpacklo_epi16 (__m256i __A, __m256i __B)
 878 {
 879   return (__m256i)__builtin_ia32_punpcklwd256 ((__v16hi)__A, (__v16hi)__B);
 880 }
 881
 882 extern __inline __m256i
 883 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 884 _mm256_unpacklo_epi32 (__m256i __A, __m256i __B)
 885 {
 886   return (__m256i)__builtin_ia32_punpckldq256 ((__v8si)__A, (__v8si)__B);
 887 }
 888
 889 extern __inline __m256i
 890 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 891 _mm256_unpacklo_epi64 (__m256i __A, __m256i __B)
 892 {
 893   return (__m256i)__builtin_ia32_punpcklqdq256 ((__v4di)__A, (__v4di)__B);
 894 }
 895
 896 extern __inline __m256i
 897 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 898 _mm256_xor_si256 (__m256i __A, __m256i __B)
 899 {
 900   return (__m256i)__builtin_ia32_pxor256 ((__v4di)__A, (__v4di)__B);
 901 }
 902
 903 extern __inline __m256i
 904 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 905 _mm256_stream_load_si256 (__m256i const *__X)
 906 {
 907   return (__m256i) __builtin_ia32_movntdqa256 ((__v4di *) __X);
 908 }
 909
 910 extern __inline __m128
 911 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 912 _mm_broadcastss_ps (__m128 __X)
 913 {
 914   return (__m128) __builtin_ia32_vbroadcastss_ps ((__v4sf)__X);
 915 }
 916
 917 extern __inline __m256
 918 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 919 _mm256_broadcastss_ps (__m128 __X)
 920 {
 921   return (__m256) __builtin_ia32_vbroadcastss_ps256 ((__v4sf)__X);
 922 }
 923
 924 extern __inline __m256d
 925 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 926 _mm256_broadcastsd_pd (__m128d __X)
 927 {
 928   return (__m256d) __builtin_ia32_vbroadcastsd_pd256 ((__v2df)__X);
 929 }
 930
 931 extern __inline __m256i
 932 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 933 _mm256_broadcastsi128_si256 (__m128i __X)
 934 {
 935   return (__m256i) __builtin_ia32_vbroadcastsi256 ((__v2di)__X);
 936 }
 937
 938 #ifdef __OPTIMIZE__
 939 extern __inline __m128i
 940 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 941 _mm_blend_epi32 (__m128i __X, __m128i __Y, const int __M)
 942 {
 943   return (__m128i) __builtin_ia32_pblendd128 ((__v4si)__X,
 944                                               (__v4si)__Y,
 945                                               __M);
 946 }
 947 #else
 948 #define _mm_blend_epi32(X, Y, M)                                        \
 949   ((__m128i) __builtin_ia32_pblendd128 ((__v4si)(__m128i)(X),           \
 950                                         (__v4si)(__m128i)(Y), (int)(M)))
 951 #endif
 952
 953 #ifdef __OPTIMIZE__
 954 extern __inline __m256i
 955 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 956 _mm256_blend_epi32 (__m256i __X, __m256i __Y, const int __M)
 957 {
 958   return (__m256i) __builtin_ia32_pblendd256 ((__v8si)__X,
 959                                               (__v8si)__Y,
 960                                               __M);
 961 }
 962 #else
 963 #define _mm256_blend_epi32(X, Y, M)                                     \
 964   ((__m256i) __builtin_ia32_pblendd256 ((__v8si)(__m256i)(X),           \
 965                                         (__v8si)(__m256i)(Y), (int)(M)))
 966 #endif
 967
 968 extern __inline __m256i
 969 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 970 _mm256_broadcastb_epi8 (__m128i __X)
 971 {
 972   return (__m256i) __builtin_ia32_pbroadcastb256 ((__v16qi)__X);
 973 }
 974
 975 extern __inline __m256i
 976 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 977 _mm256_broadcastw_epi16 (__m128i __X)
 978 {
 979   return (__m256i) __builtin_ia32_pbroadcastw256 ((__v8hi)__X);
 980 }
 981
 982 extern __inline __m256i
 983 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 984 _mm256_broadcastd_epi32 (__m128i __X)
 985 {
 986   return (__m256i) __builtin_ia32_pbroadcastd256 ((__v4si)__X);
 987 }
 988
 989 extern __inline __m256i
 990 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 991 _mm256_broadcastq_epi64 (__m128i __X)
 992 {
 993   return (__m256i) __builtin_ia32_pbroadcastq256 ((__v2di)__X);
 994 }
 995
 996 extern __inline __m128i
 997 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 998 _mm_broadcastb_epi8 (__m128i __X)
 999 {
1000   return (__m128i) __builtin_ia32_pbroadcastb128 ((__v16qi)__X);
1001 }
1002
1003 extern __inline __m128i
1004 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
1005 _mm_broadcastw_epi16 (__m128i __X)
1006 {
1007   return (__m128i) __builtin_ia32_pbroadcastw128 ((__v8hi)__X);
1008 }
1009
1010 extern __inline __m128i
1011 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
1012 _mm_broadcastd_epi32 (__m128i __X)
1013 {
1014   return (__m128i) __builtin_ia32_pbroadcastd128 ((__v4si)__X);
1015 }
1016
1017 extern __inline __m128i
1018 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
1019 _mm_broadcastq_epi64 (__m128i __X)
1020 {
1021   return (__m128i) __builtin_ia32_pbroadcastq128 ((__v2di)__X);
1022 }
1023
1024 extern __inline __m256i
1025 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
1026 _mm256_permutevar8x32_epi32 (__m256i __X, __m256i __Y)
1027 {
1028   return (__m256i) __builtin_ia32_permvarsi256 ((__v8si)__X, (__v8si)__Y);
1029 }
1030
1031 #ifdef __OPTIMIZE__
1032 extern __inline __m256d
1033 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
1034 _mm256_permute4x64_pd (__m256d __X, const int __M)
1035 {
1036   return (__m256d) __builtin_ia32_permdf256 ((__v4df)__X, __M);
1037 }
1038 #else
1039 #define _mm256_permute4x64_pd(X, M)                            \
1040   ((__m256d) __builtin_ia32_permdf256 ((__v4df)(__m256d)(X), (int)(M)))
1041 #endif
1042
1043 extern __inline __m256
1044 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
1045 _mm256_permutevar8x32_ps (__m256 __X, __m256i __Y)
1046 {
1047   return (__m256) __builtin_ia32_permvarsf256 ((__v8sf)__X, (__v8si)__Y);
1048 }
1049
1050 #ifdef __OPTIMIZE__
1051 extern __inline __m256i
1052 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
1053 _mm256_permute4x64_epi64 (__m256i __X, const int __M)
1054 {
1055   return (__m256i) __builtin_ia32_permdi256 ((__v4di)__X, __M);
1056 }
1057 #else
1058 #define _mm256_permute4x64_epi64(X, M)                         \
1059   ((__m256i) __builtin_ia32_permdi256 ((__v4di)(__m256i)(X), (int)(M)))
1060 #endif
1061
1062
1063 #ifdef __OPTIMIZE__
1064 extern __inline __m256i
1065 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
1066 _mm256_permute2x128_si256 (__m256i __X, __m256i __Y, const int __M)
1067 {
1068   return (__m256i) __builtin_ia32_permti256 ((__v4di)__X, (__v4di)__Y, __M);
1069 }
1070 #else
1071 #define _mm256_permute2x128_si256(X, Y, M)                              \
1072   ((__m256i) __builtin_ia32_permti256 ((__v4di)(__m256i)(X), (__v4di)(__m256i)(Y), (int)(M)))
1073 #endif
1074
1075 #ifdef __OPTIMIZE__
1076 extern __inline __m128i
1077 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
1078 _mm256_extracti128_si256 (__m256i __X, const int __M)
1079 {
1080   return (__m128i) __builtin_ia32_extract128i256 ((__v4di)__X, __M);
1081 }
1082 #else
1083 #define _mm256_extracti128_si256(X, M)                          \
1084   ((__m128i) __builtin_ia32_extract128i256 ((__v4di)(__m256i)(X), (int)(M)))
1085 #endif
1086
1087 #ifdef __OPTIMIZE__
1088 extern __inline __m256i
1089 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
1090 _mm256_inserti128_si256 (__m256i __X, __m128i __Y, const int __M)
1091 {
1092   return (__m256i) __builtin_ia32_insert128i256 ((__v4di)__X, (__v2di)__Y, __M);
1093 }
1094 #else
1095 #define _mm256_inserti128_si256(X, Y, M)                         \
1096   ((__m256i) __builtin_ia32_insert128i256 ((__v4di)(__m256i)(X), \
1097                                            (__v2di)(__m128i)(Y), \
1098                                            (int)(M)))
1099 #endif
1100
1101 extern __inline __m256i
1102 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
1103 _mm256_maskload_epi32 (int const *__X, __m256i __M )
1104 {
1105   return (__m256i) __builtin_ia32_maskloadd256 ((const __v8si *)__X,
1106                                                 (__v8si)__M);
1107 }
1108
1109 extern __inline __m256i
1110 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
1111 _mm256_maskload_epi64 (long long const *__X, __m256i __M )
1112 {
1113   return (__m256i) __builtin_ia32_maskloadq256 ((const __v4di *)__X,
1114                                                 (__v4di)__M);
1115 }
1116
1117 extern __inline __m128i
1118 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
1119 _mm_maskload_epi32 (int const *__X, __m128i __M )
1120 {
1121   return (__m128i) __builtin_ia32_maskloadd ((const __v4si *)__X,
1122                                              (__v4si)__M);
1123 }
1124
1125 extern __inline __m128i
1126 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
1127 _mm_maskload_epi64 (long long const *__X, __m128i __M )
1128 {
1129   return (__m128i) __builtin_ia32_maskloadq ((const __v2di *)__X,
1130                                              (__v2di)__M);
1131 }
1132
1133 extern __inline void
1134 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
1135 _mm256_maskstore_epi32 (int *__X, __m256i __M, __m256i __Y )
1136 {
1137   __builtin_ia32_maskstored256 ((__v8si *)__X, (__v8si)__M, (__v8si)__Y);
1138 }
1139
1140 extern __inline void
1141 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
1142 _mm256_maskstore_epi64 (long long *__X, __m256i __M, __m256i __Y )
1143 {
1144   __builtin_ia32_maskstoreq256 ((__v4di *)__X, (__v4di)__M, (__v4di)__Y);
1145 }
1146
1147 extern __inline void
1148 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
1149 _mm_maskstore_epi32 (int *__X, __m128i __M, __m128i __Y )
1150 {
1151   __builtin_ia32_maskstored ((__v4si *)__X, (__v4si)__M, (__v4si)__Y);
1152 }
1153
1154 extern __inline void
1155 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
1156 _mm_maskstore_epi64 (long long *__X, __m128i __M, __m128i __Y )
1157 {
1158   __builtin_ia32_maskstoreq (( __v2di *)__X, (__v2di)__M, (__v2di)__Y);
1159 }
1160
1161 extern __inline __m256i
1162 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
1163 _mm256_sllv_epi32 (__m256i __X, __m256i __Y)
1164 {
1165   return (__m256i) __builtin_ia32_psllv8si ((__v8si)__X, (__v8si)__Y);
1166 }
1167
1168 extern __inline __m128i
1169 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
1170 _mm_sllv_epi32 (__m128i __X, __m128i __Y)
1171 {
1172   return (__m128i) __builtin_ia32_psllv4si ((__v4si)__X, (__v4si)__Y);
1173 }
1174
1175 extern __inline __m256i
1176 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
1177 _mm256_sllv_epi64 (__m256i __X, __m256i __Y)
1178 {
1179   return (__m256i) __builtin_ia32_psllv4di ((__v4di)__X, (__v4di)__Y);
1180 }
1181
1182 extern __inline __m128i
1183 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
1184 _mm_sllv_epi64 (__m128i __X, __m128i __Y)
1185 {
1186   return (__m128i) __builtin_ia32_psllv2di ((__v2di)__X, (__v2di)__Y);
1187 }
1188
1189 extern __inline __m256i
1190 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
1191 _mm256_srav_epi32 (__m256i __X, __m256i __Y)
1192 {
1193   return (__m256i) __builtin_ia32_psrav8si ((__v8si)__X, (__v8si)__Y);
1194 }
1195
1196 extern __inline __m128i
1197 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
1198 _mm_srav_epi32 (__m128i __X, __m128i __Y)
1199 {
1200   return (__m128i) __builtin_ia32_psrav4si ((__v4si)__X, (__v4si)__Y);
1201 }
1202
1203 extern __inline __m256i
1204 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
1205 _mm256_srlv_epi32 (__m256i __X, __m256i __Y)
1206 {
1207   return (__m256i) __builtin_ia32_psrlv8si ((__v8si)__X, (__v8si)__Y);
1208 }
1209
1210 extern __inline __m128i
1211 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
1212 _mm_srlv_epi32 (__m128i __X, __m128i __Y)
1213 {
1214   return (__m128i) __builtin_ia32_psrlv4si ((__v4si)__X, (__v4si)__Y);
1215 }
1216
1217 extern __inline __m256i
1218 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
1219 _mm256_srlv_epi64 (__m256i __X, __m256i __Y)
1220 {
1221   return (__m256i) __builtin_ia32_psrlv4di ((__v4di)__X, (__v4di)__Y);
1222 }
1223
1224 extern __inline __m128i
1225 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
1226 _mm_srlv_epi64 (__m128i __X, __m128i __Y)
1227 {
1228   return (__m128i) __builtin_ia32_psrlv2di ((__v2di)__X, (__v2di)__Y);
1229 }
1230
1231 #ifdef __OPTIMIZE__
1232 extern __inline __m128d
1233 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
1234 _mm_i32gather_pd (double const *base, __m128i index, const int scale)
1235 {
1236   __v2df zero = _mm_setzero_pd ();
1237   __v2df mask = _mm_cmpeq_pd (zero, zero);
1238
1239   return (__m128d) __builtin_ia32_gathersiv2df (_mm_undefined_pd (),
1240                                                 base,
1241                                                 (__v4si)index,
1242                                                 mask,
1243                                                 scale);
1244 }
1245
1246 extern __inline __m128d
1247 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
1248 _mm_mask_i32gather_pd (__m128d src, double const *base, __m128i index,
1249                        __m128d mask, const int scale)
1250 {
1251   return (__m128d) __builtin_ia32_gathersiv2df ((__v2df)src,
1252                                                 base,
1253                                                 (__v4si)index,
1254                                                 (__v2df)mask,
1255                                                 scale);
1256 }
1257
1258 extern __inline __m256d
1259 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
1260 _mm256_i32gather_pd (double const *base, __m128i index, const int scale)
1261 {
1262   __v4df zero = _mm256_setzero_pd ();
1263   __v4df mask = _mm256_cmp_pd (zero, zero, _CMP_EQ_OQ);
1264
1265   return (__m256d) __builtin_ia32_gathersiv4df (_mm256_undefined_pd (),
1266                                                 base,
1267                                                 (__v4si)index,
1268                                                 mask,
1269                                                 scale);
1270 }
1271
1272 extern __inline __m256d
1273 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
1274 _mm256_mask_i32gather_pd (__m256d src, double const *base,
1275                           __m128i index, __m256d mask, const int scale)
1276 {
1277   return (__m256d) __builtin_ia32_gathersiv4df ((__v4df)src,
1278                                                 base,
1279                                                 (__v4si)index,
1280                                                 (__v4df)mask,
1281                                                 scale);
1282 }
1283
1284 extern __inline __m128d
1285 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
1286 _mm_i64gather_pd (double const *base, __m128i index, const int scale)
1287 {
1288   __v2df src = _mm_setzero_pd ();
1289   __v2df mask = _mm_cmpeq_pd (src, src);
1290
1291   return (__m128d) __builtin_ia32_gatherdiv2df (src,
1292                                                 base,
1293                                                 (__v2di)index,
1294                                                 mask,
1295                                                 scale);
1296 }
1297
1298 extern __inline __m128d
1299 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
1300 _mm_mask_i64gather_pd (__m128d src, double const *base, __m128i index,
1301                        __m128d mask, const int scale)
1302 {
1303   return (__m128d) __builtin_ia32_gatherdiv2df ((__v2df)src,
1304                                                 base,
1305                                                 (__v2di)index,
1306                                                 (__v2df)mask,
1307                                                 scale);
1308 }
1309
1310 extern __inline __m256d
1311 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
1312 _mm256_i64gather_pd (double const *base, __m256i index, const int scale)
1313 {
1314   __v4df src = _mm256_setzero_pd ();
1315   __v4df mask = _mm256_cmp_pd (src, src, _CMP_EQ_OQ);
1316
1317   return (__m256d) __builtin_ia32_gatherdiv4df (src,
1318                                                 base,
1319                                                 (__v4di)index,
1320                                                 mask,
1321                                                 scale);
1322 }
1323
1324 extern __inline __m256d
1325 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
1326 _mm256_mask_i64gather_pd (__m256d src, double const *base,
1327                           __m256i index, __m256d mask, const int scale)
1328 {
1329   return (__m256d) __builtin_ia32_gatherdiv4df ((__v4df)src,
1330                                                 base,
1331                                                 (__v4di)index,
1332                                                 (__v4df)mask,
1333                                                 scale);
1334 }
1335
1336 extern __inline __m128
1337 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
1338 _mm_i32gather_ps (float const *base, __m128i index, const int scale)
1339 {
1340   __v4sf src = _mm_setzero_ps ();
1341   __v4sf mask = _mm_cmpeq_ps (src, src);
1342
1343   return (__m128) __builtin_ia32_gathersiv4sf (src,
1344                                                base,
1345                                                (__v4si)index,
1346                                                mask,
1347                                                scale);
1348 }
1349
1350 extern __inline __m128
1351 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
1352 _mm_mask_i32gather_ps (__m128 src, float const *base, __m128i index,
1353                        __m128 mask, const int scale)
1354 {
1355   return (__m128) __builtin_ia32_gathersiv4sf ((__v4sf)src,
1356                                                base,
1357                                                (__v4si)index,
1358                                                (__v4sf)mask,
1359                                                scale);
1360 }
1361
1362 extern __inline __m256
1363 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
1364 _mm256_i32gather_ps (float const *base, __m256i index, const int scale)
1365 {
1366   __v8sf src = _mm256_setzero_ps ();
1367   __v8sf mask = _mm256_cmp_ps (src, src, _CMP_EQ_OQ);
1368
1369   return (__m256) __builtin_ia32_gathersiv8sf (src,
1370                                                base,
1371                                                (__v8si)index,
1372                                                mask,
1373                                                scale);
1374 }
1375
1376 extern __inline __m256
1377 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
1378 _mm256_mask_i32gather_ps (__m256 src, float const *base,
1379                           __m256i index, __m256 mask, const int scale)
1380 {
1381   return (__m256) __builtin_ia32_gathersiv8sf ((__v8sf)src,
1382                                                base,
1383                                                (__v8si)index,
1384                                                (__v8sf)mask,
1385                                                scale);
1386 }
1387
1388 extern __inline __m128
1389 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
1390 _mm_i64gather_ps (float const *base, __m128i index, const int scale)
1391 {
1392   __v4sf src = _mm_setzero_ps ();
1393   __v4sf mask = _mm_cmpeq_ps (src, src);
1394
1395   return (__m128) __builtin_ia32_gatherdiv4sf (src,
1396                                                base,
1397                                                (__v2di)index,
1398                                                mask,
1399                                                scale);
1400 }
1401
1402 extern __inline __m128
1403 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
1404 _mm_mask_i64gather_ps (__m128 src, float const *base, __m128i index,
1405                        __m128 mask, const int scale)
1406 {
1407   return (__m128) __builtin_ia32_gatherdiv4sf ((__v4sf)src,
1408                                                 base,
1409                                                 (__v2di)index,
1410                                                 (__v4sf)mask,
1411                                                 scale);
1412 }
1413
1414 extern __inline __m128
1415 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
1416 _mm256_i64gather_ps (float const *base, __m256i index, const int scale)
1417 {
1418   __v4sf src = _mm_setzero_ps ();
1419   __v4sf mask = _mm_cmpeq_ps (src, src);
1420
1421   return (__m128) __builtin_ia32_gatherdiv4sf256 (src,
1422                                                   base,
1423                                                   (__v4di)index,
1424                                                   mask,
1425                                                   scale);
1426 }
1427
1428 extern __inline __m128
1429 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
1430 _mm256_mask_i64gather_ps (__m128 src, float const *base,
1431                           __m256i index, __m128 mask, const int scale)
1432 {
1433   return (__m128) __builtin_ia32_gatherdiv4sf256 ((__v4sf)src,
1434                                                   base,
1435                                                   (__v4di)index,
1436                                                   (__v4sf)mask,
1437                                                   scale);
1438 }
1439
1440 extern __inline __m128i
1441 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
1442 _mm_i32gather_epi64 (long long int const *base,
1443                      __m128i index, const int scale)
1444 {
1445   __v2di src = __extension__ (__v2di){ 0, 0 };
1446   __v2di mask = __extension__ (__v2di){ ~0, ~0 };
1447
1448   return (__m128i) __builtin_ia32_gathersiv2di (src,
1449                                                 base,
1450                                                 (__v4si)index,
1451                                                 mask,
1452                                                 scale);
1453 }
1454
1455 extern __inline __m128i
1456 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
1457 _mm_mask_i32gather_epi64 (__m128i src, long long int const *base,
1458                           __m128i index, __m128i mask, const int scale)
1459 {
1460   return (__m128i) __builtin_ia32_gathersiv2di ((__v2di)src,
1461                                                 base,
1462                                                 (__v4si)index,
1463                                                 (__v2di)mask,
1464                                                 scale);
1465 }
1466
1467 extern __inline __m256i
1468 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
1469 _mm256_i32gather_epi64 (long long int const *base,
1470                         __m128i index, const int scale)
1471 {
1472   __v4di src = __extension__ (__v4di){ 0, 0, 0, 0 };
1473   __v4di mask = __extension__ (__v4di){ ~0, ~0, ~0, ~0 };
1474
1475   return (__m256i) __builtin_ia32_gathersiv4di (src,
1476                                                 base,
1477                                                 (__v4si)index,
1478                                                 mask,
1479                                                 scale);
1480 }
1481
1482 extern __inline __m256i
1483 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
1484 _mm256_mask_i32gather_epi64 (__m256i src, long long int const *base,
1485                              __m128i index, __m256i mask, const int scale)
1486 {
1487   return (__m256i) __builtin_ia32_gathersiv4di ((__v4di)src,
1488                                                 base,
1489                                                 (__v4si)index,
1490                                                 (__v4di)mask,
1491                                                 scale);
1492 }
1493
1494 extern __inline __m128i
1495 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
1496 _mm_i64gather_epi64 (long long int const *base,
1497                      __m128i index, const int scale)
1498 {
1499   __v2di src = __extension__ (__v2di){ 0, 0 };
1500   __v2di mask = __extension__ (__v2di){ ~0, ~0 };
1501
1502   return (__m128i) __builtin_ia32_gatherdiv2di (src,
1503                                                 base,
1504                                                 (__v2di)index,
1505                                                 mask,
1506                                                 scale);
1507 }
1508
1509 extern __inline __m128i
1510 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
1511 _mm_mask_i64gather_epi64 (__m128i src, long long int const *base, __m128i index,
1512                           __m128i mask, const int scale)
1513 {
1514   return (__m128i) __builtin_ia32_gatherdiv2di ((__v2di)src,
1515                                                 base,
1516                                                 (__v2di)index,
1517                                                 (__v2di)mask,
1518                                                 scale);
1519 }
1520
1521 extern __inline __m256i
1522 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
1523 _mm256_i64gather_epi64 (long long int const *base,
1524                         __m256i index, const int scale)
1525 {
1526   __v4di src = __extension__ (__v4di){ 0, 0, 0, 0 };
1527   __v4di mask = __extension__ (__v4di){ ~0, ~0, ~0, ~0 };
1528
1529   return (__m256i) __builtin_ia32_gatherdiv4di (src,
1530                                                 base,
1531                                                 (__v4di)index,
1532                                                 mask,
1533                                                 scale);
1534 }
1535
1536 extern __inline __m256i
1537 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
1538 _mm256_mask_i64gather_epi64 (__m256i src, long long int const *base,
1539                              __m256i index, __m256i mask, const int scale)
1540 {
1541   return (__m256i) __builtin_ia32_gatherdiv4di ((__v4di)src,
1542                                                 base,
1543                                                 (__v4di)index,
1544                                                 (__v4di)mask,
1545                                                 scale);
1546 }
1547
1548 extern __inline __m128i
1549 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
1550 _mm_i32gather_epi32 (int const *base, __m128i index, const int scale)
1551 {
1552   __v4si src = __extension__ (__v4si){ 0, 0, 0, 0 };
1553   __v4si mask = __extension__ (__v4si){ ~0, ~0, ~0, ~0 };
1554
1555   return (__m128i) __builtin_ia32_gathersiv4si (src,
1556                                                base,
1557                                                (__v4si)index,
1558                                                mask,
1559                                                scale);
1560 }
1561
1562 extern __inline __m128i
1563 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
1564 _mm_mask_i32gather_epi32 (__m128i src, int const *base, __m128i index,
1565                           __m128i mask, const int scale)
1566 {
1567   return (__m128i) __builtin_ia32_gathersiv4si ((__v4si)src,
1568                                                 base,
1569                                                 (__v4si)index,
1570                                                 (__v4si)mask,
1571                                                 scale);
1572 }
1573
1574 extern __inline __m256i
1575 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
1576 _mm256_i32gather_epi32 (int const *base, __m256i index, const int scale)
1577 {
1578   __v8si src = __extension__ (__v8si){ 0, 0, 0, 0, 0, 0, 0, 0 };
1579   __v8si mask = __extension__ (__v8si){ ~0, ~0, ~0, ~0, ~0, ~0, ~0, ~0 };
1580
1581   return (__m256i) __builtin_ia32_gathersiv8si (src,
1582                                                 base,
1583                                                 (__v8si)index,
1584                                                 mask,
1585                                                 scale);
1586 }
1587
1588 extern __inline __m256i
1589 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
1590 _mm256_mask_i32gather_epi32 (__m256i src, int const *base,
1591                              __m256i index, __m256i mask, const int scale)
1592 {
1593   return (__m256i) __builtin_ia32_gathersiv8si ((__v8si)src,
1594                                                 base,
1595                                                 (__v8si)index,
1596                                                 (__v8si)mask,
1597                                                 scale);
1598 }
1599
1600 extern __inline __m128i
1601 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
1602 _mm_i64gather_epi32 (int const *base, __m128i index, const int scale)
1603 {
1604   __v4si src = __extension__ (__v4si){ 0, 0, 0, 0 };
1605   __v4si mask = __extension__ (__v4si){ ~0, ~0, ~0, ~0 };
1606
1607   return (__m128i) __builtin_ia32_gatherdiv4si (src,
1608                                                 base,
1609                                                 (__v2di)index,
1610                                                 mask,
1611                                                 scale);
1612 }
1613
1614 extern __inline __m128i
1615 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
1616 _mm_mask_i64gather_epi32 (__m128i src, int const *base, __m128i index,
1617                           __m128i mask, const int scale)
1618 {
1619   return (__m128i) __builtin_ia32_gatherdiv4si ((__v4si)src,
1620                                                 base,
1621                                                 (__v2di)index,
1622                                                 (__v4si)mask,
1623                                                 scale);
1624 }
1625
1626 extern __inline __m128i
1627 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
1628 _mm256_i64gather_epi32 (int const *base, __m256i index, const int scale)
1629 {
1630   __v4si src = __extension__ (__v4si){ 0, 0, 0, 0 };
1631   __v4si mask = __extension__ (__v4si){ ~0, ~0, ~0, ~0 };
1632
1633   return (__m128i) __builtin_ia32_gatherdiv4si256 (src,
1634                                                   base,
1635                                                   (__v4di)index,
1636                                                   mask,
1637                                                   scale);
1638 }
1639
1640 extern __inline __m128i
1641 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
1642 _mm256_mask_i64gather_epi32 (__m128i src, int const *base,
1643                              __m256i index, __m128i mask, const int scale)
1644 {
1645   return (__m128i) __builtin_ia32_gatherdiv4si256 ((__v4si)src,
1646                                                    base,
1647                                                    (__v4di)index,
1648                                                    (__v4si)mask,
1649                                                    scale);
1650 }
1651 #else /* __OPTIMIZE__ */
1652 #define _mm_i32gather_pd(BASE, INDEX, SCALE)                            \
1653   (__m128d) __builtin_ia32_gathersiv2df ((__v2df) _mm_setzero_pd (),    \
1654                                          (double const *)BASE,          \
1655                                          (__v4si)(__m128i)INDEX,        \
1656                                          (__v2df)_mm_set1_pd(           \
1657                                            (double)(long long int) -1), \
1658                                          (int)SCALE)
1659
1660 #define _mm_mask_i32gather_pd(SRC, BASE, INDEX, MASK, SCALE)     \
1661   (__m128d) __builtin_ia32_gathersiv2df ((__v2df)(__m128d)SRC,   \
1662                                          (double const *)BASE,   \
1663                                          (__v4si)(__m128i)INDEX, \
1664                                          (__v2df)(__m128d)MASK,  \
1665                                          (int)SCALE)
1666
1667 #define _mm256_i32gather_pd(BASE, INDEX, SCALE)                         \
1668   (__m256d) __builtin_ia32_gathersiv4df ((__v4df) _mm256_setzero_pd (), \
1669                                          (double const *)BASE,          \
1670                                          (__v4si)(__m128i)INDEX,        \
1671                                          (__v4df)_mm256_set1_pd(        \
1672                                            (double)(long long int) -1), \
1673                                          (int)SCALE)
1674
1675 #define _mm256_mask_i32gather_pd(SRC, BASE, INDEX, MASK, SCALE)  \
1676   (__m256d) __builtin_ia32_gathersiv4df ((__v4df)(__m256d)SRC,   \
1677                                          (double const *)BASE,   \
1678                                          (__v4si)(__m128i)INDEX, \
1679                                          (__v4df)(__m256d)MASK,  \
1680                                          (int)SCALE)
1681
1682 #define _mm_i64gather_pd(BASE, INDEX, SCALE)                            \
1683   (__m128d) __builtin_ia32_gatherdiv2df ((__v2df) _mm_setzero_pd (),    \
1684                                          (double const *)BASE,          \
1685                                          (__v2di)(__m128i)INDEX,        \
1686                                          (__v2df)_mm_set1_pd(           \
1687                                            (double)(long long int) -1), \
1688                                          (int)SCALE)
1689
1690 #define _mm_mask_i64gather_pd(SRC, BASE, INDEX, MASK, SCALE)     \
1691   (__m128d) __builtin_ia32_gatherdiv2df ((__v2df)(__m128d)SRC,   \
1692                                          (double const *)BASE,   \
1693                                          (__v2di)(__m128i)INDEX, \
1694                                          (__v2df)(__m128d)MASK,  \
1695                                          (int)SCALE)
1696
1697 #define _mm256_i64gather_pd(BASE, INDEX, SCALE)                         \
1698   (__m256d) __builtin_ia32_gatherdiv4df ((__v4df) _mm256_setzero_pd (), \
1699                                          (double const *)BASE,          \
1700                                          (__v4di)(__m256i)INDEX,        \
1701                                          (__v4df)_mm256_set1_pd(        \
1702                                            (double)(long long int) -1), \
1703                                          (int)SCALE)
1704
1705 #define _mm256_mask_i64gather_pd(SRC, BASE, INDEX, MASK, SCALE)  \
1706   (__m256d) __builtin_ia32_gatherdiv4df ((__v4df)(__m256d)SRC,   \
1707                                          (double const *)BASE,   \
1708                                          (__v4di)(__m256i)INDEX, \
1709                                          (__v4df)(__m256d)MASK,  \
1710                                          (int)SCALE)
1711
1712 #define _mm_i32gather_ps(BASE, INDEX, SCALE)                            \
1713   (__m128) __builtin_ia32_gathersiv4sf ((__v4sf) _mm_setzero_ps (),     \
1714                                         (float const *)BASE,            \
1715                                         (__v4si)(__m128i)INDEX,         \
1716                                         _mm_set1_ps ((float)(int) -1),  \
1717                                         (int)SCALE)
1718
1719 #define _mm_mask_i32gather_ps(SRC, BASE, INDEX, MASK, SCALE)     \
1720   (__m128) __builtin_ia32_gathersiv4sf ((__v4sf)(__m128d)SRC,    \
1721                                         (float const *)BASE,     \
1722                                         (__v4si)(__m128i)INDEX,  \
1723                                         (__v4sf)(__m128d)MASK,   \
1724                                         (int)SCALE)
1725
1726 #define _mm256_i32gather_ps(BASE, INDEX, SCALE)                        \
1727   (__m256) __builtin_ia32_gathersiv8sf ((__v8sf) _mm256_setzero_ps (), \
1728                                         (float const *)BASE,           \
1729                                         (__v8si)(__m256i)INDEX,        \
1730                                         (__v8sf)_mm256_set1_ps (       \
1731                                           (float)(int) -1),            \
1732                                         (int)SCALE)
1733
1734 #define _mm256_mask_i32gather_ps(SRC, BASE, INDEX, MASK, SCALE) \
1735   (__m256) __builtin_ia32_gathersiv8sf ((__v8sf)(__m256)SRC,    \
1736                                         (float const *)BASE,    \
1737                                         (__v8si)(__m256i)INDEX, \
1738                                         (__v8sf)(__m256d)MASK,  \
1739                                         (int)SCALE)
1740
1741 #define _mm_i64gather_ps(BASE, INDEX, SCALE)                            \
1742   (__m128) __builtin_ia32_gatherdiv4sf ((__v4sf) _mm_setzero_pd (),     \
1743                                         (float const *)BASE,            \
1744                                         (__v2di)(__m128i)INDEX,         \
1745                                         (__v4sf)_mm_set1_ps (           \
1746                                           (float)(int) -1),             \
1747                                         (int)SCALE)
1748
1749 #define _mm_mask_i64gather_ps(SRC, BASE, INDEX, MASK, SCALE)     \
1750   (__m128) __builtin_ia32_gatherdiv4sf ((__v4sf)(__m128)SRC,     \
1751                                         (float const *)BASE,     \
1752                                         (__v2di)(__m128i)INDEX,  \
1753                                         (__v4sf)(__m128d)MASK,   \
1754                                         (int)SCALE)
1755
1756 #define _mm256_i64gather_ps(BASE, INDEX, SCALE)                         \
1757   (__m128) __builtin_ia32_gatherdiv4sf256 ((__v4sf) _mm_setzero_ps (),  \
1758                                            (float const *)BASE,         \
1759                                            (__v4di)(__m256i)INDEX,      \
1760                                            (__v4sf)_mm_set1_ps(         \
1761                                              (float)(int) -1),          \
1762                                            (int)SCALE)
1763
1764 #define _mm256_mask_i64gather_ps(SRC, BASE, INDEX, MASK, SCALE)    \
1765   (__m128) __builtin_ia32_gatherdiv4sf256 ((__v4sf)(__m128)SRC,    \
1766                                            (float const *)BASE,    \
1767                                            (__v4di)(__m256i)INDEX, \
1768                                            (__v4sf)(__m128)MASK,   \
1769                                            (int)SCALE)
1770
1771 #define _mm_i32gather_epi64(BASE, INDEX, SCALE)                         \
1772   (__m128i) __builtin_ia32_gathersiv2di ((__v2di) _mm_setzero_si128 (), \
1773                                          (long long const *)BASE,       \
1774                                          (__v4si)(__m128i)INDEX,        \
1775                                          (__v2di)_mm_set1_epi64x (-1),  \
1776                                          (int)SCALE)
1777
1778 #define _mm_mask_i32gather_epi64(SRC, BASE, INDEX, MASK, SCALE)   \
1779   (__m128i) __builtin_ia32_gathersiv2di ((__v2di)(__m128i)SRC,    \
1780                                          (long long const *)BASE, \
1781                                          (__v4si)(__m128i)INDEX,  \
1782                                          (__v2di)(__m128i)MASK,   \
1783                                          (int)SCALE)
1784
1785 #define _mm256_i32gather_epi64(BASE, INDEX, SCALE)                         \
1786   (__m256i) __builtin_ia32_gathersiv4di ((__v4di) _mm256_setzero_si256 (), \
1787                                          (long long const *)BASE,          \
1788                                          (__v4si)(__m128i)INDEX,           \
1789                                          (__v4di)_mm256_set1_epi64x (-1),  \
1790                                          (int)SCALE)
1791
1792 #define _mm256_mask_i32gather_epi64(SRC, BASE, INDEX, MASK, SCALE) \
1793   (__m256i) __builtin_ia32_gathersiv4di ((__v4di)(__m256i)SRC,     \
1794                                          (long long const *)BASE,  \
1795                                          (__v4si)(__m128i)INDEX,   \
1796                                          (__v4di)(__m256i)MASK,    \
1797                                          (int)SCALE)
1798
1799 #define _mm_i64gather_epi64(BASE, INDEX, SCALE)                         \
1800   (__m128i) __builtin_ia32_gatherdiv2di ((__v2di) _mm_setzero_si128 (), \
1801                                          (long long const *)BASE,       \
1802                                          (__v2di)(__m128i)INDEX,        \
1803                                          (__v2di)_mm_set1_epi64x (-1),  \
1804                                          (int)SCALE)
1805
1806 #define _mm_mask_i64gather_epi64(SRC, BASE, INDEX, MASK, SCALE)   \
1807   (__m128i) __builtin_ia32_gatherdiv2di ((__v2di)(__m128i)SRC,    \
1808                                          (long long const *)BASE, \
1809                                          (__v2di)(__m128i)INDEX,  \
1810                                          (__v2di)(__m128i)MASK,   \
1811                                          (int)SCALE)
1812
1813 #define _mm256_i64gather_epi64(BASE, INDEX, SCALE)                         \
1814   (__m256i) __builtin_ia32_gatherdiv4di ((__v4di) _mm256_setzero_si256 (), \
1815                                          (long long const *)BASE,          \
1816                                          (__v4di)(__m256i)INDEX,           \
1817                                          (__v4di)_mm256_set1_epi64x (-1),  \
1818                                          (int)SCALE)
1819
1820 #define _mm256_mask_i64gather_epi64(SRC, BASE, INDEX, MASK, SCALE) \
1821   (__m256i) __builtin_ia32_gatherdiv4di ((__v4di)(__m256i)SRC,     \
1822                                          (long long const *)BASE,  \
1823                                          (__v4di)(__m256i)INDEX,   \
1824                                          (__v4di)(__m256i)MASK,    \
1825                                          (int)SCALE)
1826
1827 #define _mm_i32gather_epi32(BASE, INDEX, SCALE)                         \
1828   (__m128i) __builtin_ia32_gathersiv4si ((__v4si) _mm_setzero_si128 (), \
1829                                          (int const *)BASE,             \
1830                                          (__v4si)(__m128i)INDEX,        \
1831                                          (__v4si)_mm_set1_epi32 (-1),   \
1832                                          (int)SCALE)
1833
1834 #define _mm_mask_i32gather_epi32(SRC, BASE, INDEX, MASK, SCALE) \
1835   (__m128i) __builtin_ia32_gathersiv4si ((__v4si)(__m128i)SRC,  \
1836                                         (int const *)BASE,      \
1837                                         (__v4si)(__m128i)INDEX, \
1838                                         (__v4si)(__m128i)MASK,  \
1839                                         (int)SCALE)
1840
1841 #define _mm256_i32gather_epi32(BASE, INDEX, SCALE)                         \
1842   (__m256i) __builtin_ia32_gathersiv8si ((__v8si) _mm256_setzero_si256 (), \
1843                                          (int const *)BASE,                \
1844                                          (__v8si)(__m256i)INDEX,           \
1845                                          (__v8si)_mm256_set1_epi32 (-1),   \
1846                                          (int)SCALE)
1847
1848 #define _mm256_mask_i32gather_epi32(SRC, BASE, INDEX, MASK, SCALE) \
1849   (__m256i) __builtin_ia32_gathersiv8si ((__v8si)(__m256i)SRC,     \
1850                                         (int const *)BASE,         \
1851                                         (__v8si)(__m256i)INDEX,    \
1852                                         (__v8si)(__m256i)MASK,     \
1853                                         (int)SCALE)
1854
1855 #define _mm_i64gather_epi32(BASE, INDEX, SCALE)                         \
1856   (__m128i) __builtin_ia32_gatherdiv4si ((__v4si) _mm_setzero_si128 (), \
1857                                          (int const *)BASE,             \
1858                                          (__v2di)(__m128i)INDEX,        \
1859                                          (__v4si)_mm_set1_epi32 (-1),   \
1860                                          (int)SCALE)
1861
1862 #define _mm_mask_i64gather_epi32(SRC, BASE, INDEX, MASK, SCALE) \
1863   (__m128i) __builtin_ia32_gatherdiv4si ((__v4si)(__m128i)SRC,  \
1864                                         (int const *)BASE,      \
1865                                         (__v2di)(__m128i)INDEX, \
1866                                         (__v4si)(__m128i)MASK,  \
1867                                         (int)SCALE)
1868
1869 #define _mm256_i64gather_epi32(BASE, INDEX, SCALE)                         \
1870   (__m128i) __builtin_ia32_gatherdiv4si256 ((__v4si) _mm_setzero_si128 (), \
1871                                             (int const *)BASE,             \
1872                                             (__v4di)(__m256i)INDEX,        \
1873                                             (__v4si)_mm_set1_epi32(-1),    \
1874                                             (int)SCALE)
1875
1876 #define _mm256_mask_i64gather_epi32(SRC, BASE, INDEX, MASK, SCALE) \
1877   (__m128i) __builtin_ia32_gatherdiv4si256 ((__v4si)(__m128i)SRC,  \
1878                                            (int const *)BASE,      \
1879                                            (__v4di)(__m256i)INDEX, \
1880                                            (__v4si)(__m128i)MASK,  \
1881                                            (int)SCALE)
1882 #endif  /* __OPTIMIZE__ */
1883
1884 #ifdef __DISABLE_AVX2__
1885 #undef __DISABLE_AVX2__
1886 #pragma GCC pop_options
1887 #endif /* __DISABLE_AVX2__ */
1888
1889 #endif /* _AVX2INTRIN_H_INCLUDED */