gcc/config/rs6000/tmmintrin.h

   1 /* Copyright (C) 2003-2024 Free Software Foundation, Inc.
   2
   3    This file is part of GCC.
   4
   5    GCC is free software; you can redistribute it and/or modify
   6    it under the terms of the GNU General Public License as published by
   7    the Free Software Foundation; either version 3, or (at your option)
   8    any later version.
   9
  10    GCC is distributed in the hope that it will be useful,
  11    but WITHOUT ANY WARRANTY; without even the implied warranty of
  12    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13    GNU General Public License for more details.
  14
  15    Under Section 7 of GPL version 3, you are granted additional
  16    permissions described in the GCC Runtime Library Exception, version
  17    3.1, as published by the Free Software Foundation.
  18
  19    You should have received a copy of the GNU General Public License and
  20    a copy of the GCC Runtime Library Exception along with this program;
  21    see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
  22    <http://www.gnu.org/licenses/>.  */
  23
  24 /* Implemented from the specification included in the Intel C++ Compiler
  25    User Guide and Reference, version 9.0.  */
  26
  27 #ifndef NO_WARN_X86_INTRINSICS
  28 /* This header is distributed to simplify porting x86_64 code that
  29    makes explicit use of Intel intrinsics to powerpc64le.
  30    It is the user's responsibility to determine if the results are
  31    acceptable and make additional changes as necessary.
  32    Note that much code that uses Intel intrinsics can be rewritten in
  33    standard C or GNU C extensions, which are more portable and better
  34    optimized across multiple targets.  */
  35 #endif
  36
  37 #ifndef TMMINTRIN_H_
  38 #define TMMINTRIN_H_
  39
  40 #include <altivec.h>
  41 #include <assert.h>
  42
  43 /* We need definitions from the SSE header files.  */
  44 #include <pmmintrin.h>
  45
  46 extern __inline __m128i
  47 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  48 _mm_abs_epi16 (__m128i __A)
  49 {
  50   return (__m128i) vec_abs ((__v8hi) __A);
  51 }
  52
  53 extern __inline __m128i
  54 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  55 _mm_abs_epi32 (__m128i __A)
  56 {
  57   return (__m128i) vec_abs ((__v4si) __A);
  58 }
  59
  60 extern __inline __m128i
  61 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  62 _mm_abs_epi8 (__m128i __A)
  63 {
  64   return (__m128i) vec_abs ((__v16qi) __A);
  65 }
  66
  67 extern __inline __m64
  68 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  69 _mm_abs_pi16 (__m64 __A)
  70 {
  71   __v8hi __B = (__v8hi) (__v2du) { __A, __A };
  72   return (__m64) ((__v2du) vec_abs (__B))[0];
  73 }
  74
  75 extern __inline __m64
  76 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  77 _mm_abs_pi32 (__m64 __A)
  78 {
  79   __v4si __B = (__v4si) (__v2du) { __A, __A };
  80   return (__m64) ((__v2du) vec_abs (__B))[0];
  81 }
  82
  83 extern __inline __m64
  84 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  85 _mm_abs_pi8 (__m64 __A)
  86 {
  87   __v16qi __B = (__v16qi) (__v2du) { __A, __A };
  88   return (__m64) ((__v2du) vec_abs (__B))[0];
  89 }
  90
  91 extern __inline __m128i
  92 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
  93 _mm_alignr_epi8 (__m128i __A, __m128i __B, const unsigned int __count)
  94 {
  95   if (__builtin_constant_p (__count) && __count < 16)
  96     {
  97 #ifdef __LITTLE_ENDIAN__
  98       __A = (__m128i) vec_reve ((__v16qu) __A);
  99       __B = (__m128i) vec_reve ((__v16qu) __B);
 100 #endif
 101       __A = (__m128i) vec_sld ((__v16qu) __B, (__v16qu) __A, __count);
 102 #ifdef __LITTLE_ENDIAN__
 103       __A = (__m128i) vec_reve ((__v16qu) __A);
 104 #endif
 105       return __A;
 106     }
 107
 108   if (__count == 0)
 109     return __B;
 110
 111   if (__count >= 16)
 112     {
 113       if (__count >= 32)
 114         {
 115           const __v16qu __zero = { 0 };
 116           return (__m128i) __zero;
 117         }
 118       else
 119         {
 120           const __v16qu __shift =
 121             vec_splats ((unsigned char) ((__count - 16) * 8));
 122 #ifdef __LITTLE_ENDIAN__
 123           return (__m128i) vec_sro ((__v16qu) __A, __shift);
 124 #else
 125           return (__m128i) vec_slo ((__v16qu) __A, __shift);
 126 #endif
 127         }
 128     }
 129   else
 130     {
 131       const __v16qu __shiftA =
 132         vec_splats ((unsigned char) ((16 - __count) * 8));
 133       const __v16qu __shiftB = vec_splats ((unsigned char) (__count * 8));
 134 #ifdef __LITTLE_ENDIAN__
 135       __A = (__m128i) vec_slo ((__v16qu) __A, __shiftA);
 136       __B = (__m128i) vec_sro ((__v16qu) __B, __shiftB);
 137 #else
 138       __A = (__m128i) vec_sro ((__v16qu) __A, __shiftA);
 139       __B = (__m128i) vec_slo ((__v16qu) __B, __shiftB);
 140 #endif
 141       return (__m128i) vec_or ((__v16qu) __A, (__v16qu) __B);
 142     }
 143 }
 144
 145 extern __inline __m64
 146 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 147 _mm_alignr_pi8 (__m64 __A, __m64 __B, unsigned int __count)
 148 {
 149   if (__count < 16)
 150     {
 151       __v2du __C = { __B, __A };
 152 #ifdef __LITTLE_ENDIAN__
 153       const __v4su __shift = { __count << 3, 0, 0, 0 };
 154       __C = (__v2du) vec_sro ((__v16qu) __C, (__v16qu) __shift);
 155 #else
 156       const __v4su __shift = { 0, 0, 0, __count << 3 };
 157       __C = (__v2du) vec_slo ((__v16qu) __C, (__v16qu) __shift);
 158 #endif
 159       return (__m64) __C[0];
 160     }
 161   else
 162     {
 163       const __m64 __zero = { 0 };
 164       return __zero;
 165     }
 166 }
 167
 168 extern __inline __m128i
 169 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 170 _mm_hadd_epi16 (__m128i __A, __m128i __B)
 171 {
 172   const __v16qu __P =
 173     {  0,  1,  4,  5,  8,  9, 12, 13, 16, 17, 20, 21, 24, 25, 28, 29 };
 174   const __v16qu __Q =
 175     {  2,  3,  6,  7, 10, 11, 14, 15, 18, 19, 22, 23, 26, 27, 30, 31 };
 176   __v8hi __C = vec_perm ((__v8hi) __A, (__v8hi) __B, __P);
 177   __v8hi __D = vec_perm ((__v8hi) __A, (__v8hi) __B, __Q);
 178   return (__m128i) vec_add (__C, __D);
 179 }
 180
 181 extern __inline __m128i
 182 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 183 _mm_hadd_epi32 (__m128i __A, __m128i __B)
 184 {
 185   const __v16qu __P =
 186     {  0,  1,  2,  3,  8,  9, 10, 11, 16, 17, 18, 19, 24, 25, 26, 27 };
 187   const __v16qu __Q =
 188     {  4,  5,  6,  7, 12, 13, 14, 15, 20, 21, 22, 23, 28, 29, 30, 31 };
 189   __v4si __C = vec_perm ((__v4si) __A, (__v4si) __B, __P);
 190   __v4si __D = vec_perm ((__v4si) __A, (__v4si) __B, __Q);
 191   return (__m128i) vec_add (__C, __D);
 192 }
 193
 194 extern __inline __m64
 195 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 196 _mm_hadd_pi16 (__m64 __A, __m64 __B)
 197 {
 198   __v8hi __C = (__v8hi) (__v2du) { __A, __B };
 199   const __v16qu __P =
 200     {  0,  1,  4,  5,  8,  9, 12, 13,  0,  1,  4,  5,  8,  9, 12, 13 };
 201   const __v16qu __Q =
 202     {  2,  3,  6,  7, 10, 11, 14, 15,  2,  3,  6,  7, 10, 11, 14, 15 };
 203   __v8hi __D = vec_perm (__C, __C, __Q);
 204   __C = vec_perm (__C, __C, __P);
 205   __C = vec_add (__C, __D);
 206   return (__m64) ((__v2du) __C)[1];
 207 }
 208
 209 extern __inline __m64
 210 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 211 _mm_hadd_pi32 (__m64 __A, __m64 __B)
 212 {
 213   __v4si __C = (__v4si) (__v2du) { __A, __B };
 214   const __v16qu __P =
 215     {  0,  1,  2,  3,  8,  9, 10, 11,  0,  1,  2,  3,  8,  9, 10, 11 };
 216   const __v16qu __Q =
 217     {  4,  5,  6,  7, 12, 13, 14, 15,  4,  5,  6,  7, 12, 13, 14, 15 };
 218   __v4si __D = vec_perm (__C, __C, __Q);
 219   __C = vec_perm (__C, __C, __P);
 220   __C = vec_add (__C, __D);
 221   return (__m64) ((__v2du) __C)[1];
 222 }
 223
 224 extern __inline __m128i
 225 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 226 _mm_hadds_epi16 (__m128i __A, __m128i __B)
 227 {
 228   __v4si __C = { 0 }, __D = { 0 };
 229   __C = vec_sum4s ((__v8hi) __A, __C);
 230   __D = vec_sum4s ((__v8hi) __B, __D);
 231   __C = (__v4si) vec_packs (__C, __D);
 232   return (__m128i) __C;
 233 }
 234
 235 extern __inline __m64
 236 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 237 _mm_hadds_pi16 (__m64 __A, __m64 __B)
 238 {
 239   const __v4si __zero = { 0 };
 240   __v8hi __C = (__v8hi) (__v2du) { __A, __B };
 241   __v4si __D = vec_sum4s (__C, __zero);
 242   __C = vec_packs (__D, __D);
 243   return (__m64) ((__v2du) __C)[1];
 244 }
 245
 246 extern __inline __m128i
 247 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 248 _mm_hsub_epi16 (__m128i __A, __m128i __B)
 249 {
 250   const __v16qu __P =
 251     {  0,  1,  4,  5,  8,  9, 12, 13, 16, 17, 20, 21, 24, 25, 28, 29 };
 252   const __v16qu __Q =
 253     {  2,  3,  6,  7, 10, 11, 14, 15, 18, 19, 22, 23, 26, 27, 30, 31 };
 254   __v8hi __C = vec_perm ((__v8hi) __A, (__v8hi) __B, __P);
 255   __v8hi __D = vec_perm ((__v8hi) __A, (__v8hi) __B, __Q);
 256   return (__m128i) vec_sub (__C, __D);
 257 }
 258
 259 extern __inline __m128i
 260 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 261 _mm_hsub_epi32 (__m128i __A, __m128i __B)
 262 {
 263   const __v16qu __P =
 264     {  0,  1,  2,  3,  8,  9, 10, 11, 16, 17, 18, 19, 24, 25, 26, 27 };
 265   const __v16qu __Q =
 266     {  4,  5,  6,  7, 12, 13, 14, 15, 20, 21, 22, 23, 28, 29, 30, 31 };
 267   __v4si __C = vec_perm ((__v4si) __A, (__v4si) __B, __P);
 268   __v4si __D = vec_perm ((__v4si) __A, (__v4si) __B, __Q);
 269   return (__m128i) vec_sub (__C, __D);
 270 }
 271
 272 extern __inline __m64
 273 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 274 _mm_hsub_pi16 (__m64 __A, __m64 __B)
 275 {
 276   const __v16qu __P =
 277     {  0,  1,  4,  5,  8,  9, 12, 13,  0,  1,  4,  5,  8,  9, 12, 13 };
 278   const __v16qu __Q =
 279     {  2,  3,  6,  7, 10, 11, 14, 15,  2,  3,  6,  7, 10, 11, 14, 15 };
 280   __v8hi __C = (__v8hi) (__v2du) { __A, __B };
 281   __v8hi __D = vec_perm (__C, __C, __Q);
 282   __C = vec_perm (__C, __C, __P);
 283   __C = vec_sub (__C, __D);
 284   return (__m64) ((__v2du) __C)[1];
 285 }
 286
 287 extern __inline __m64
 288 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 289 _mm_hsub_pi32 (__m64 __A, __m64 __B)
 290 {
 291   const __v16qu __P =
 292     {  0,  1,  2,  3,  8,  9, 10, 11,  0,  1,  2,  3,  8,  9, 10, 11 };
 293   const __v16qu __Q =
 294     {  4,  5,  6,  7, 12, 13, 14, 15,  4,  5,  6,  7, 12, 13, 14, 15 };
 295   __v4si __C = (__v4si) (__v2du) { __A, __B };
 296   __v4si __D = vec_perm (__C, __C, __Q);
 297   __C = vec_perm (__C, __C, __P);
 298   __C = vec_sub (__C, __D);
 299   return (__m64) ((__v2du) __C)[1];
 300 }
 301
 302 extern __inline __m128i
 303 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 304 _mm_hsubs_epi16 (__m128i __A, __m128i __B)
 305 {
 306   const __v16qu __P =
 307     {  0,  1,  4,  5,  8,  9, 12, 13, 16, 17, 20, 21, 24, 25, 28, 29 };
 308   const __v16qu __Q =
 309     {  2,  3,  6,  7, 10, 11, 14, 15, 18, 19, 22, 23, 26, 27, 30, 31 };
 310   __v8hi __C = vec_perm ((__v8hi) __A, (__v8hi) __B, __P);
 311   __v8hi __D = vec_perm ((__v8hi) __A, (__v8hi) __B, __Q);
 312   return (__m128i) vec_subs (__C, __D);
 313 }
 314
 315 extern __inline __m64
 316 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 317 _mm_hsubs_pi16 (__m64 __A, __m64 __B)
 318 {
 319   const __v16qu __P =
 320     {  0,  1,  4,  5,  8,  9, 12, 13,  0,  1,  4,  5,  8,  9, 12, 13 };
 321   const __v16qu __Q =
 322     {  2,  3,  6,  7, 10, 11, 14, 15,  2,  3,  6,  7, 10, 11, 14, 15 };
 323   __v8hi __C = (__v8hi) (__v2du) { __A, __B };
 324   __v8hi __D = vec_perm (__C, __C, __P);
 325   __v8hi __E = vec_perm (__C, __C, __Q);
 326   __C = vec_subs (__D, __E);
 327   return (__m64) ((__v2du) __C)[1];
 328 }
 329
 330 extern __inline __m128i
 331 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 332 _mm_shuffle_epi8 (__m128i __A, __m128i __B)
 333 {
 334   const __v16qi __zero = { 0 };
 335   __vector __bool char __select = vec_cmplt ((__v16qi) __B, __zero);
 336   __v16qi __C = vec_perm ((__v16qi) __A, (__v16qi) __A, (__v16qu) __B);
 337   return (__m128i) vec_sel (__C, __zero, __select);
 338 }
 339
 340 extern __inline __m64
 341 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 342 _mm_shuffle_pi8 (__m64 __A, __m64 __B)
 343 {
 344   const __v16qi __zero = { 0 };
 345   __v16qi __C = (__v16qi) (__v2du) { __A, __A };
 346   __v16qi __D = (__v16qi) (__v2du) { __B, __B };
 347   __vector __bool char __select = vec_cmplt ((__v16qi) __D, __zero);
 348   __C = vec_perm ((__v16qi) __C, (__v16qi) __C, (__v16qu) __D);
 349   __C = vec_sel (__C, __zero, __select);
 350   return (__m64) ((__v2du) (__C))[0];
 351 }
 352
 353 #ifdef _ARCH_PWR8
 354 extern __inline __m128i
 355 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 356 _mm_sign_epi8 (__m128i __A, __m128i __B)
 357 {
 358   const __v16qi __zero = { 0 };
 359   __v16qi __selectneg = (__v16qi) vec_cmplt ((__v16qi) __B, __zero);
 360   __v16qi __selectpos =
 361     (__v16qi) vec_neg ((__v16qi) vec_cmpgt ((__v16qi) __B, __zero));
 362   __v16qi __conv = vec_add (__selectneg, __selectpos);
 363   return (__m128i) vec_mul ((__v16qi) __A, (__v16qi) __conv);
 364 }
 365 #endif
 366
 367 #ifdef _ARCH_PWR8
 368 extern __inline __m128i
 369 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 370 _mm_sign_epi16 (__m128i __A, __m128i __B)
 371 {
 372   const __v8hi __zero = { 0 };
 373   __v8hi __selectneg = (__v8hi) vec_cmplt ((__v8hi) __B, __zero);
 374   __v8hi __selectpos =
 375     (__v8hi) vec_neg ((__v8hi) vec_cmpgt ((__v8hi) __B, __zero));
 376   __v8hi __conv = vec_add (__selectneg, __selectpos);
 377   return (__m128i) vec_mul ((__v8hi) __A, (__v8hi) __conv);
 378 }
 379 #endif
 380
 381 #ifdef _ARCH_PWR8
 382 extern __inline __m128i
 383 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 384 _mm_sign_epi32 (__m128i __A, __m128i __B)
 385 {
 386   const __v4si __zero = { 0 };
 387   __v4si __selectneg = (__v4si) vec_cmplt ((__v4si) __B, __zero);
 388   __v4si __selectpos =
 389     (__v4si) vec_neg ((__v4si) vec_cmpgt ((__v4si) __B, __zero));
 390   __v4si __conv = vec_add (__selectneg, __selectpos);
 391   return (__m128i) vec_mul ((__v4si) __A, (__v4si) __conv);
 392 }
 393 #endif
 394
 395 #ifdef _ARCH_PWR8
 396 extern __inline __m64
 397 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 398 _mm_sign_pi8 (__m64 __A, __m64 __B)
 399 {
 400   const __v16qi __zero = { 0 };
 401   __v16qi __C = (__v16qi) (__v2du) { __A, __A };
 402   __v16qi __D = (__v16qi) (__v2du) { __B, __B };
 403   __C = (__v16qi) _mm_sign_epi8 ((__m128i) __C, (__m128i) __D);
 404   return (__m64) ((__v2du) (__C))[0];
 405 }
 406 #endif
 407
 408 #ifdef _ARCH_PWR8
 409 extern __inline __m64
 410 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 411 _mm_sign_pi16 (__m64 __A, __m64 __B)
 412 {
 413   const __v8hi __zero = { 0 };
 414   __v8hi __C = (__v8hi) (__v2du) { __A, __A };
 415   __v8hi __D = (__v8hi) (__v2du) { __B, __B };
 416   __C = (__v8hi) _mm_sign_epi16 ((__m128i) __C, (__m128i) __D);
 417   return (__m64) ((__v2du) (__C))[0];
 418 }
 419 #endif
 420
 421 #ifdef _ARCH_PWR8
 422 extern __inline __m64
 423 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 424 _mm_sign_pi32 (__m64 __A, __m64 __B)
 425 {
 426   const __v4si __zero = { 0 };
 427   __v4si __C = (__v4si) (__v2du) { __A, __A };
 428   __v4si __D = (__v4si) (__v2du) { __B, __B };
 429   __C = (__v4si) _mm_sign_epi32 ((__m128i) __C, (__m128i) __D);
 430   return (__m64) ((__v2du) (__C))[0];
 431 }
 432 #endif
 433
 434 extern __inline __m128i
 435 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 436 _mm_maddubs_epi16 (__m128i __A, __m128i __B)
 437 {
 438   __v8hi __unsigned = vec_splats ((signed short) 0x00ff);
 439   __v8hi __C = vec_and (vec_unpackh ((__v16qi) __A), __unsigned);
 440   __v8hi __D = vec_and (vec_unpackl ((__v16qi) __A), __unsigned);
 441   __v8hi __E = vec_unpackh ((__v16qi) __B);
 442   __v8hi __F = vec_unpackl ((__v16qi) __B);
 443   __C = vec_mul (__C, __E);
 444   __D = vec_mul (__D, __F);
 445   const __v16qu __odds  =
 446     {  0,  1,  4,  5,  8,  9, 12, 13, 16, 17, 20, 21, 24, 25, 28, 29 };
 447   const __v16qu __evens =
 448     {  2,  3,  6,  7, 10, 11, 14, 15, 18, 19, 22, 23, 26, 27, 30, 31 };
 449   __E = vec_perm (__C, __D, __odds);
 450   __F = vec_perm (__C, __D, __evens);
 451   return (__m128i) vec_adds (__E, __F);
 452 }
 453
 454 extern __inline __m64
 455 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 456 _mm_maddubs_pi16 (__m64 __A, __m64 __B)
 457 {
 458   __v8hi __C = (__v8hi) (__v2du) { __A, __A };
 459   __C = vec_unpackl ((__v16qi) __C);
 460   const __v8hi __unsigned = vec_splats ((signed short) 0x00ff);
 461   __C = vec_and (__C, __unsigned);
 462   __v8hi __D = (__v8hi) (__v2du) { __B, __B };
 463   __D = vec_unpackl ((__v16qi) __D);
 464   __D = vec_mul (__C, __D);
 465   const __v16qu __odds  =
 466     {  0,  1,  4,  5,  8,  9, 12, 13, 16, 17, 20, 21, 24, 25, 28, 29 };
 467   const __v16qu __evens =
 468     {  2,  3,  6,  7, 10, 11, 14, 15, 18, 19, 22, 23, 26, 27, 30, 31 };
 469   __C = vec_perm (__D, __D, __odds);
 470   __D = vec_perm (__D, __D, __evens);
 471   __C = vec_adds (__C, __D);
 472   return (__m64) ((__v2du) (__C))[0];
 473 }
 474
 475 extern __inline __m128i
 476 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 477 _mm_mulhrs_epi16 (__m128i __A, __m128i __B)
 478 {
 479   __v4si __C = vec_unpackh ((__v8hi) __A);
 480   __v4si __D = vec_unpackh ((__v8hi) __B);
 481   __C = vec_mul (__C, __D);
 482   __D = vec_unpackl ((__v8hi) __A);
 483   __v4si __E = vec_unpackl ((__v8hi) __B);
 484   __D = vec_mul (__D, __E);
 485   const __v4su __shift = vec_splats ((unsigned int) 14);
 486   __C = vec_sr (__C, __shift);
 487   __D = vec_sr (__D, __shift);
 488   const __v4si __ones = vec_splats ((signed int) 1);
 489   __C = vec_add (__C, __ones);
 490   __C = vec_sr (__C, (__v4su) __ones);
 491   __D = vec_add (__D, __ones);
 492   __D = vec_sr (__D, (__v4su) __ones);
 493   return (__m128i) vec_pack (__C, __D);
 494 }
 495
 496 extern __inline __m64
 497 __attribute__((__gnu_inline__, __always_inline__, __artificial__))
 498 _mm_mulhrs_pi16 (__m64 __A, __m64 __B)
 499 {
 500   __v4si __C = (__v4si) (__v2du) { __A, __A };
 501   __C = vec_unpackh ((__v8hi) __C);
 502   __v4si __D = (__v4si) (__v2du) { __B, __B };
 503   __D = vec_unpackh ((__v8hi) __D);
 504   __C = vec_mul (__C, __D);
 505   const __v4su __shift = vec_splats ((unsigned int) 14);
 506   __C = vec_sr (__C, __shift);
 507   const __v4si __ones = vec_splats ((signed int) 1);
 508   __C = vec_add (__C, __ones);
 509   __C = vec_sr (__C, (__v4su) __ones);
 510   __v8hi __E = vec_pack (__C, __D);
 511   return (__m64) ((__v2du) (__E))[0];
 512 }
 513
 514 #endif